From a5d4ab9db7c4c0431476ca90842de27432c4765c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 15:52:49 -0700 Subject: [PATCH 001/102] feat: add runtime resource snapshots to the Rust core Applications can read a runtime's identity and status but not how much memory or CPU its adapter host tree uses. Issue #267 asks for a timestamped snapshot on the runtime API and a final snapshot on RunResult that reuses the same type. Add RuntimeResourceSnapshot and runtime_resource_snapshot(plan, runtime) with typed ResourceObservationUnavailable and ResourceObservationUnsupported errors. Linux reads /proc and macOS reads proc_pid_rusage, so no dependency is added; other platforms return Unsupported. The host PID lives in a map beside LOCAL_HOSTS, so a snapshot never waits for the host mutex a running invocation holds. Each local invocation's RunResult carries one final snapshot, and a failed read leaves it unset instead of failing the run. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/error.rs | 18 + crates/fabric-core/src/lib.rs | 6 +- crates/fabric-core/src/resource_snapshot.rs | 575 ++++++++++++++++++ crates/fabric-core/src/runtime.rs | 242 ++++++++ crates/fabric-core/src/schema.rs | 11 +- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 34 +- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/index.mdx | 3 + .../resource-snapshot/index.mdx | 21 + .../struct-runtimeresourcesnapshot.mdx | 117 ++++ .../runtime/fn-runtime-resource-snapshot.mdx | 16 + .../runtime/fn-start-runtime-with-service.mdx | 2 +- .../runtime/fn-start-runtime.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 3 +- .../runtime/struct-runresult.mdx | 6 +- .../schema/enum-schemaname.mdx | 9 +- .../nemo-fabric-core/schema/index.mdx | 2 +- schemas/SCHEMA.md | 2 + schemas/sdk/run-result.schema.json | 57 ++ .../sdk/runtime-resource-snapshot.schema.json | 48 ++ 27 files changed, 1172 insertions(+), 18 deletions(-) create mode 100644 crates/fabric-core/src/resource_snapshot.rs create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx create mode 100644 schemas/sdk/runtime-resource-snapshot.schema.json diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index a21340fb7..ce4682c28 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -277,6 +277,24 @@ pub enum FabricError { /// Adapter kind. adapter_kind: AdapterKind, }, + /// A runtime's resources cannot be read now: the runtime stopped, is + /// stopping, its adapter host process exited, or the process table cannot + /// be read. + #[error("resource observation is unavailable for runtime `{runtime_id}`: {reason}")] + ResourceObservationUnavailable { + /// Runtime handle id. + runtime_id: String, + /// Why the runtime cannot be observed. + reason: String, + }, + /// The runtime's environment provider or operating system has no resource reader. + #[error("resource observation is unsupported for runtime `{runtime_id}`: {reason}")] + ResourceObservationUnsupported { + /// Runtime handle id. + runtime_id: String, + /// What has no resource reader. + reason: String, + }, /// Process adapter settings were invalid. #[error("invalid process adapter settings for {path}: {source}")] InvalidProcessSettings { diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index 2dc251c9f..b20d70a09 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -9,6 +9,7 @@ pub mod agent_execution; pub mod config; pub mod doctor; pub mod error; +pub mod resource_snapshot; pub mod runtime; pub mod schema; @@ -38,6 +39,7 @@ pub use config::{ }; pub use doctor::{DoctorCheck, DoctorReport, DoctorStatus, doctor_plan}; pub use error::{FabricError, Result}; +pub use resource_snapshot::RuntimeResourceSnapshot; pub use runtime::{ AdapterInvocation, ArtifactManifest, ArtifactRef, EnvironmentHandle, ErrorInfo, ErrorStage, FabricEvent, InvocationHandle, OpenAiChatCompletionChunk, OpenAiChatCompletionChunkChoice, @@ -46,8 +48,8 @@ pub use runtime::{ OpenAiStreamSink, OpenAiStreamTransport, RunRequest, RunResult, RunStatus, RunUsage, RuntimeContext, RuntimeHandle, RuntimeTelemetryContext, ServiceHandle, ServiceOwnership, ServiceReference, TelemetryRef, attach_service, invoke_openai_stream, invoke_runtime, - prepare_environment, prepare_service, release_service, run_plan, start_runtime, - start_runtime_with_service, stop_runtime, + prepare_environment, prepare_service, release_service, run_plan, runtime_resource_snapshot, + start_runtime, start_runtime_with_service, stop_runtime, }; pub use schema::{ SchemaName, generate_all_schemas, generate_schema, generate_schema_json, write_schema_snapshots, diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs new file mode 100644 index 000000000..8401f096e --- /dev/null +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -0,0 +1,575 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Resource snapshots of a runtime's local adapter host and its descendants. +//! +//! A snapshot reads the process tree rooted at the adapter host process that +//! NeMo Fabric started for a runtime. Each read visits a parent before its +//! children. CPU time adds every process's own CPU time and the CPU time of the +//! children it has already waited for, so work done by finished descendants +//! still counts. A child reaped between the read of its parent and its own +//! read is missed by that one snapshot and is never counted twice. CPU time +//! can drop between snapshots when a descendant is reparented out of the tree, +//! and on macOS while an exited descendant is not yet waited for, because +//! macOS cannot list an unreaped process. +//! +//! Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. Neither +//! reads command lines, environment variables, or open files. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::error::{FabricError, Result}; + +const MILLIS_PER_DAY: i64 = 86_400_000; +const NANOS_PER_SECOND: f64 = 1_000_000_000.0; + +/// Timestamped resource usage of one runtime's local adapter host process tree. +/// +/// Memory, process count, and CPU time cover the adapter host process and its +/// live descendants. For a runtime started on a service, the shared service +/// process is not included. Values Fabric cannot read are returned as an error, +/// never as zero. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct RuntimeResourceSnapshot { + /// Runtime this snapshot describes. + pub runtime_id: String, + /// When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. + #[schemars(extend("format" = "date-time"))] + pub started_at: String, + /// When Fabric read these values, as an RFC 3339 UTC timestamp. + #[schemars(extend("format" = "date-time"))] + pub observed_at: String, + /// Sum of the resident set sizes of the live processes, in bytes. Pages + /// shared between processes count once per process. + pub resident_memory_bytes: u64, + /// Live processes in the tree, including the adapter host. Exited processes + /// that are not yet reaped are not counted. + pub process_count: u64, + /// CPU time consumed by the tree since the adapter host started, in seconds, + /// including descendants that have exited and been waited for. It can drop + /// when a descendant leaves the tree, and on macOS while an exited + /// descendant is not yet waited for. + #[schemars(range(min = 0.0))] + pub cpu_time_seconds: f64, +} + +/// The adapter host process of one runtime, as recorded when the runtime started. +#[derive(Debug, Clone)] +pub(crate) struct LocalHostProcess { + pid: u32, + /// Start time of the process at `pid`, which tells the host apart from a + /// later process that reuses its PID. + identity: Option, + started_at: String, +} + +impl LocalHostProcess { + /// Record the host `pid`, started by Fabric at `started_at`. + pub(crate) fn new(pid: u32, started_at: SystemTime) -> Self { + Self { + pid, + identity: platform::start_identity(pid), + started_at: rfc3339_millis(unix_millis(started_at)), + } + } +} + +/// Read a snapshot of `host`, or explain why none can be read. +/// +/// `host` is `None` when the runtime is not active in this process. +pub(crate) fn snapshot_local_host( + runtime_id: &str, + host: Option<&LocalHostProcess>, +) -> Result { + if !platform::SUPPORTED { + return Err(FabricError::ResourceObservationUnsupported { + runtime_id: runtime_id.to_string(), + reason: format!( + "no process reader for operating system `{}`", + std::env::consts::OS + ), + }); + } + let host = + host.ok_or_else(|| unavailable(runtime_id, "runtime is not active in this process"))?; + let read = host + .identity + .and_then(|identity| platform::read_tree(host.pid, identity)) + .ok_or_else(|| { + unavailable( + runtime_id, + "adapter host process has exited or the process table cannot be read", + ) + })?; + Ok(RuntimeResourceSnapshot { + runtime_id: runtime_id.to_string(), + started_at: host.started_at.clone(), + observed_at: rfc3339_millis(unix_millis(SystemTime::now())), + resident_memory_bytes: read.resident_bytes, + process_count: read.process_count, + cpu_time_seconds: read.cpu_nanos as f64 / NANOS_PER_SECOND, + }) +} + +fn unavailable(runtime_id: &str, reason: &str) -> FabricError { + FabricError::ResourceObservationUnavailable { + runtime_id: runtime_id.to_string(), + reason: reason.to_string(), + } +} + +/// One read of a live process tree. +#[cfg_attr( + not(any(target_os = "linux", target_os = "macos")), + allow( + dead_code, + reason = "only the Linux and macOS readers build a tree read" + ) +)] +#[derive(Debug, Default)] +struct TreeRead { + cpu_nanos: u64, + resident_bytes: u64, + process_count: u64, +} + +fn unix_millis(time: SystemTime) -> i64 { + match time.duration_since(UNIX_EPOCH) { + Ok(after) => i64::try_from(after.as_millis()).unwrap_or(i64::MAX), + Err(before) => -i64::try_from(before.duration().as_millis()).unwrap_or(i64::MAX), + } +} + +/// Format Unix milliseconds as RFC 3339 UTC with three fraction digits. +/// +/// Core has no date crate, so this converts days to a civil date with Howard +/// Hinnant's `civil_from_days` algorithm. +fn rfc3339_millis(unix_millis: i64) -> String { + let days = unix_millis.div_euclid(MILLIS_PER_DAY); + let millis_of_day = unix_millis.rem_euclid(MILLIS_PER_DAY); + let (year, month, day) = civil_from_days(days); + let seconds_of_day = millis_of_day / 1000; + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{:03}Z", + seconds_of_day / 3600, + seconds_of_day / 60 % 60, + seconds_of_day % 60, + millis_of_day % 1000, + ) +} + +fn civil_from_days(days: i64) -> (i64, i64, i64) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + (year, month, day) +} + +#[cfg(target_os = "linux")] +mod platform { + //! `/proc` reads: one pass lists every process's parent, then the tree is + //! read from the root down, each parent before its children. + + use std::collections::{BTreeMap, VecDeque}; + use std::sync::OnceLock; + + use super::TreeRead; + + pub(super) const SUPPORTED: bool = true; + + /// The fields of `/proc//stat` a snapshot needs. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) struct Stat { + pub zombie: bool, + pub ppid: u32, + /// `utime + stime + cutime + cstime`, in clock ticks. + pub cpu_ticks: u64, + /// `starttime`, in clock ticks since boot. + pub start: u64, + pub rss_pages: u64, + } + + /// Parse a `/proc//stat` line. Fields are counted after the last `)`, + /// because the command name can itself hold spaces and parentheses. + pub(super) fn parse_stat(text: &str) -> Option { + let (_, rest) = text.rsplit_once(')')?; + let fields: Vec<&str> = rest.split_whitespace().collect(); + let number = |index: usize| fields.get(index)?.parse::().ok(); + let ticks = [11, 12, 13, 14] + .into_iter() + .map(&number) + .sum::>()?; + Some(Stat { + zombie: fields.first() == Some(&"Z"), + ppid: u32::try_from(number(1)?).ok()?, + cpu_ticks: ticks, + start: number(19)?, + rss_pages: number(21)?, + }) + } + + fn read_stat(pid: u32) -> Option { + parse_stat(&std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?) + } + + pub(super) fn start_identity(pid: u32) -> Option { + read_stat(pid).map(|stat| stat.start) + } + + fn sysconf(name: libc::c_int) -> u64 { + // SAFETY: sysconf has no memory-safety preconditions. + u64::try_from(unsafe { libc::sysconf(name) }).unwrap_or(0) + } + + fn clock_ticks_per_second() -> u64 { + static TICKS: OnceLock = OnceLock::new(); + *TICKS.get_or_init(|| sysconf(libc::_SC_CLK_TCK).max(1)) + } + + fn page_size() -> u64 { + static PAGE: OnceLock = OnceLock::new(); + *PAGE.get_or_init(|| sysconf(libc::_SC_PAGESIZE)) + } + + /// Every process's children, each with its start time, or `None` when + /// `/proc` cannot be listed. + fn children_by_parent() -> Option>> { + let mut children: BTreeMap> = BTreeMap::new(); + for entry in std::fs::read_dir("/proc").ok()?.flatten() { + let Some(pid) = entry + .file_name() + .to_str() + .and_then(|name| name.parse().ok()) + else { + continue; + }; + if let Some(stat) = read_stat(pid) { + children + .entry(stat.ppid) + .or_default() + .push((pid, stat.start)); + } + } + Some(children) + } + + /// Read the tree under `root`, or `None` when the root has exited or its + /// PID now names a process with another start time than `identity`. + pub(super) fn read_tree(root: u32, identity: u64) -> Option { + let children = children_by_parent()?; + let mut read = TreeRead::default(); + let mut cpu_ticks: u64 = 0; + let mut rss_pages: u64 = 0; + let mut pending = VecDeque::from([(root, identity)]); + while let Some((pid, expected_start)) = pending.pop_front() { + let stat = read_stat(pid).filter(|stat| stat.start == expected_start); + if pid == root && stat.is_none_or(|stat| stat.zombie) { + return None; + } + let Some(stat) = stat else { + continue; + }; + cpu_ticks = cpu_ticks.saturating_add(stat.cpu_ticks); + if !stat.zombie { + rss_pages = rss_pages.saturating_add(stat.rss_pages); + read.process_count += 1; + } + // A child never starts before its parent; one that seems to reuses a PID. + let own = children.get(&pid).into_iter().flatten(); + pending.extend(own.filter(|(_, start)| *start >= stat.start).copied()); + } + read.cpu_nanos = ticks_to_nanos(cpu_ticks, clock_ticks_per_second()); + read.resident_bytes = rss_pages.saturating_mul(page_size()); + Some(read) + } + + pub(super) fn ticks_to_nanos(ticks: u64, ticks_per_second: u64) -> u64 { + let nanos = u128::from(ticks) * 1_000_000_000 / u128::from(ticks_per_second); + u64::try_from(nanos).unwrap_or(u64::MAX) + } +} + +#[cfg(target_os = "macos")] +mod platform { + //! `proc_pid_rusage` reads, from the root down, each parent before its children. + + use std::collections::{BTreeMap, VecDeque}; + use std::sync::OnceLock; + + use super::TreeRead; + + pub(super) const SUPPORTED: bool = true; + + fn rusage(pid: u32) -> Option { + let pid = libc::c_int::try_from(pid).ok()?; + // SAFETY: rusage_info_v2 is plain data; zero is a valid initial value. + let mut info: libc::rusage_info_v2 = unsafe { std::mem::zeroed() }; + // SAFETY: `info` is a writable rusage_info_v2, the layout RUSAGE_INFO_V2 names. + let status = unsafe { + libc::proc_pid_rusage( + pid, + libc::RUSAGE_INFO_V2, + (&raw mut info).cast::(), + ) + }; + (status == 0).then_some(info) + } + + /// `mach_timebase_info_data_t`. + #[repr(C)] + struct MachTimebaseInfo { + numer: u32, + denom: u32, + } + + // libc marks its binding deprecated in favour of the `mach2` crate; one + // system call does not justify a new dependency. + unsafe extern "C" { + fn mach_timebase_info(info: *mut MachTimebaseInfo) -> libc::c_int; + } + + /// Nanoseconds per Mach absolute time unit, as `(numer, denom)`, or `None` + /// when the kernel does not report it. + fn timebase() -> Option<(u64, u64)> { + static TIMEBASE: OnceLock> = OnceLock::new(); + *TIMEBASE.get_or_init(|| { + let mut info = MachTimebaseInfo { numer: 0, denom: 0 }; + // SAFETY: `info` is a writable mach_timebase_info_data_t. + let status = unsafe { mach_timebase_info(&mut info) }; + (status == 0 && info.denom != 0).then(|| (u64::from(info.numer), u64::from(info.denom))) + }) + } + + /// Convert Mach absolute time units, which `rusage_info` CPU times use on + /// Apple silicon, to nanoseconds. + pub(super) fn mach_to_nanos(units: u64, (numer, denom): (u64, u64)) -> u64 { + let nanos = u128::from(units) * u128::from(numer) / u128::from(denom); + u64::try_from(nanos).unwrap_or(u64::MAX) + } + + pub(super) fn start_identity(pid: u32) -> Option { + rusage(pid).map(|info| info.ri_proc_start_abstime) + } + + /// Every live process's children. One `proc_listallpids` call and one + /// `proc_pidinfo` per process: `proc_listchildpids` scans the whole + /// process table on each call, which costs too much for a wide tree. + /// `PROC_PIDTBSDINFO` fails for a zombie, so a zombie is never listed. + /// Returns `None` when the process table cannot be listed. + fn children_by_parent() -> Option>> { + let mut pids = vec![0 as libc::pid_t; 1024]; + let count = loop { + let bytes = libc::c_int::try_from(std::mem::size_of_val(pids.as_slice())).ok()?; + // SAFETY: the buffer is writable and sized in bytes as libproc requires. + let count = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) }; + let count = usize::try_from(count).ok().filter(|count| *count > 0)?; + if count < pids.len() { + break count; + } + pids.resize(pids.len() * 2, 0); + }; + let mut children: BTreeMap> = BTreeMap::new(); + for &pid in &pids[..count] { + if let (Ok(pid), Some(ppid)) = (u32::try_from(pid), parent_pid(pid)) { + children.entry(ppid).or_default().push(pid); + } + } + Some(children) + } + + /// The parent of a live process, or `None` for a zombie or a missing process. + fn parent_pid(pid: libc::pid_t) -> Option { + // SAFETY: proc_bsdinfo is plain data; zero is a valid initial value. + let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; + let size = libc::c_int::try_from(std::mem::size_of::()).ok()?; + // SAFETY: `info` is writable and `size` bytes long, as PROC_PIDTBSDINFO needs. + let written = unsafe { + libc::proc_pidinfo(pid, libc::PROC_PIDTBSDINFO, 0, (&raw mut info).cast(), size) + }; + (written == size).then_some(info.pbi_ppid) + } + + /// Read the tree under `root`, or `None` when the root has exited or its + /// PID now names a process with another start time than `identity`. A + /// child that seems to start before its parent reuses a PID and is left out. + pub(super) fn read_tree(root: u32, identity: u64) -> Option { + parent_pid(libc::pid_t::try_from(root).ok()?)?; + let root_info = rusage(root).filter(|info| info.ri_proc_start_abstime == identity)?; + let timebase = timebase()?; + let children = children_by_parent()?; + let mut read = TreeRead::default(); + let mut cpu_units: u64 = 0; + let mut resident: u64 = 0; + let mut pending = VecDeque::from([(root, Some(root_info))]); + while let Some((pid, known)) = pending.pop_front() { + let Some(info) = known.or_else(|| rusage(pid)) else { + continue; + }; + cpu_units = cpu_units + .saturating_add(info.ri_user_time) + .saturating_add(info.ri_system_time) + .saturating_add(info.ri_child_user_time) + .saturating_add(info.ri_child_system_time); + resident = resident.saturating_add(info.ri_resident_size); + read.process_count += 1; + let start = info.ri_proc_start_abstime; + let own = children.get(&pid).into_iter().flatten(); + pending.extend( + own.filter_map(|child| rusage(*child).map(|info| (*child, info))) + // Skip a child that exited after it was listed, and one + // that seems to start before its parent (a reused PID). + .filter(|(_, info)| { + info.ri_proc_exit_abstime == 0 && info.ri_proc_start_abstime >= start + }) + .map(|(child, info)| (child, Some(info))), + ); + } + read.cpu_nanos = mach_to_nanos(cpu_units, timebase); + read.resident_bytes = resident; + Some(read) + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +mod platform { + use super::TreeRead; + + pub(super) const SUPPORTED: bool = false; + + pub(super) fn start_identity(_pid: u32) -> Option { + None + } + + pub(super) fn read_tree(_root: u32, _identity: u64) -> Option { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rfc3339_millis_formats_fixed_vectors() { + assert_eq!(rfc3339_millis(0), "1970-01-01T00:00:00.000Z"); + assert_eq!( + rfc3339_millis(1_792_000_870_205), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!(rfc3339_millis(951_782_400_000), "2000-02-29T00:00:00.000Z"); + assert_eq!( + rfc3339_millis(4_107_542_399_999), + "2100-02-28T23:59:59.999Z" + ); + assert_eq!(rfc3339_millis(-1), "1969-12-31T23:59:59.999Z"); + } + + #[test] + fn snapshot_local_host_without_active_host_is_unavailable_or_unsupported() { + let error = snapshot_local_host("runtime-1", None).expect_err("no host"); + + if platform::SUPPORTED { + assert!(matches!( + error, + FabricError::ResourceObservationUnavailable { ref runtime_id, .. } + if runtime_id == "runtime-1" + )); + } else { + assert!(matches!( + error, + FabricError::ResourceObservationUnsupported { .. } + )); + } + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn snapshot_local_host_reads_this_process_tree() { + let host = LocalHostProcess::new(std::process::id(), SystemTime::now()); + + let snapshot = snapshot_local_host("runtime-1", Some(&host)).expect("snapshot"); + + assert_eq!(snapshot.runtime_id, "runtime-1"); + assert!(snapshot.process_count >= 1); + assert!(snapshot.resident_memory_bytes > 0); + // Linux counts CPU in 10 ms ticks, so a young test process can read zero. + assert!(snapshot.cpu_time_seconds.is_finite() && snapshot.cpu_time_seconds >= 0.0); + assert!(snapshot.observed_at >= snapshot.started_at); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn snapshot_local_host_with_reused_pid_identity_is_unavailable() { + let mut host = LocalHostProcess::new(std::process::id(), SystemTime::now()); + host.identity = host.identity.map(|identity| identity.wrapping_add(1)); + + let error = snapshot_local_host("runtime-1", Some(&host)).expect_err("other process"); + + assert!(matches!( + error, + FabricError::ResourceObservationUnavailable { .. } + )); + } + + /// A host that exited but is not yet reaped is not a live process. + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn snapshot_local_host_with_unreaped_exited_host_is_unavailable() { + let mut child = std::process::Command::new("sleep") + .arg("30") + .spawn() + .expect("spawn child"); + let host = LocalHostProcess::new(child.id(), SystemTime::now()); + child.kill().expect("kill child"); + std::thread::sleep(std::time::Duration::from_millis(200)); + + let error = snapshot_local_host("runtime-1", Some(&host)).expect_err("zombie host"); + child.wait().expect("reap child"); + + assert!(matches!( + error, + FabricError::ResourceObservationUnavailable { .. } + )); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_stat_parse_reads_cpu_with_waited_children_start_time_and_ticks() { + let stat = "42 (a (weird) name) S 7 42 42 0 -1 4194560 100 0 0 0 30 20 5 4 20 0 1 0 \ + 9001 1000000 250 18446744073709551615"; + + let parsed = platform::parse_stat(stat).expect("stat parses"); + + assert_eq!(parsed.ppid, 7); + assert_eq!(parsed.cpu_ticks, 30 + 20 + 5 + 4); + assert_eq!(parsed.start, 9001); + assert_eq!(parsed.rss_pages, 250); + assert!(!parsed.zombie); + assert_eq!(platform::ticks_to_nanos(150, 100), 1_500_000_000); + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_mach_units_convert_with_the_timebase() { + // Apple silicon's timebase: 125 ns per 3 units. + assert_eq!(platform::mach_to_nanos(24_000_000, (125, 3)), 1_000_000_000); + assert_eq!( + platform::mach_to_nanos(1_000_000_000, (1, 1)), + 1_000_000_000 + ); + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index ab16add07..9630f9f6e 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -31,6 +31,7 @@ use crate::config::{ validate_tool_definitions, validate_workflow, }; use crate::error::{FabricError, Result}; +use crate::resource_snapshot::{LocalHostProcess, RuntimeResourceSnapshot, snapshot_local_host}; static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; @@ -72,6 +73,10 @@ const DEFAULT_PYTHON: &str = "python.exe"; static TEST_STOPPED_AGENTS: Mutex> = Mutex::new(Vec::new()); static LOCAL_HOSTS: LazyLock>>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); +// Adapter host processes by runtime id. A host's own mutex stays locked for a +// whole invocation, so a resource snapshot reads the PID from here instead. +static LOCAL_HOST_PROCESSES: LazyLock>> = + LazyLock::new(|| Mutex::new(BTreeMap::new())); static LOCAL_SERVICES: LazyLock>>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); @@ -136,6 +141,10 @@ pub struct RunResult { /// Normalized invocation usage when reported by the adapter target. #[serde(default, skip_serializing_if = "Option::is_none")] pub usage: Option, + /// Resource snapshot of the runtime, read once after the adapter returned this + /// result. Absent when the runtime's resources could not be observed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resource_snapshot: Option, /// Artifacts produced or collected by this run. #[serde(default)] pub artifacts: ArtifactManifest, @@ -1163,6 +1172,46 @@ pub fn invoke_runtime( }) } +/// Read a timestamped resource snapshot of a started runtime. +/// +/// The snapshot covers the runtime's local adapter host process and its live +/// descendants. It does not wait for a running invocation. It returns +/// [`FabricError::ResourceObservationUnavailable`] when the runtime stopped, is +/// stopping, its host exited, or the process table cannot be read, and +/// [`FabricError::ResourceObservationUnsupported`] when the environment provider +/// or operating system has no resource reader. +pub fn runtime_resource_snapshot( + plan: &RunPlan, + runtime: &RuntimeHandle, +) -> Result { + validate_runtime_handle(plan, runtime)?; + if !uses_local_host(plan) { + return Err(FabricError::ResourceObservationUnsupported { + runtime_id: runtime.runtime_id.clone(), + reason: format!( + "adapter kind `{}` has no resource reader", + adapter_kind_name(runtime.adapter_kind) + ), + }); + } + local_runtime_resource_snapshot(runtime) +} + +fn local_runtime_resource_snapshot(runtime: &RuntimeHandle) -> Result { + if runtime.environment.provider != "local" { + return Err(FabricError::ResourceObservationUnsupported { + runtime_id: runtime.runtime_id.clone(), + reason: format!( + "environment provider `{}` has no resource reader", + runtime.environment.provider + ), + }); + } + // Clone the entry so the map lock is released before the process tree is read. + let host = local_host_processes().get(&runtime.runtime_id).cloned(); + snapshot_local_host(&runtime.runtime_id, host.as_ref()) +} + /// Invoke a started harness runtime and pass through native OpenAI chat-completion chunks. pub fn invoke_openai_stream( plan: &RunPlan, @@ -1523,6 +1572,7 @@ impl RuntimeAdapter for LocalHostAdapter { } fn stop(&self, runtime: &RuntimeHandle) -> Result> { + local_host_processes().remove(&runtime.runtime_id); let Some(host) = local_hosts().remove(&runtime.runtime_id) else { return Ok(vec![local_host_stop_event(runtime, true, false)]); }; @@ -1620,6 +1670,7 @@ fn start_local_runtime( )?; let environment_env = start.runtime_context.environment.env.clone(); let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Start(Box::new(start))); + let started_at = SystemTime::now(); let mut host = spawn_local_host(plan, &runtime, artifacts, relay_config, environment_env)?; if let Err(error) = exchange_lifecycle_message( &mut host, @@ -1631,6 +1682,10 @@ fn start_local_runtime( let _ = remove_local_host_files(&host); return Err(error); } + local_host_processes().insert( + runtime.runtime_id.clone(), + LocalHostProcess::new(host.child.id(), started_at), + ); local_hosts().insert(runtime.runtime_id.clone(), Arc::new(Mutex::new(host))); Ok(runtime) } @@ -1814,8 +1869,11 @@ fn run_local_host_invocation_with_timeout( ) { Ok(output) => { let stderr = take_local_host_stderr(&mut host_guard); + // Read while the host lock is held, so no later invocation is counted. + let resource_snapshot = local_runtime_resource_snapshot(runtime).ok(); Ok(( output, + resource_snapshot, stderr, host_guard.command.clone(), host_guard.child.id(), @@ -1838,6 +1896,7 @@ fn run_local_host_invocation_with_timeout( }; let ( adapter_output, + resource_snapshot, stderr, host_command, host_pid, @@ -1989,6 +2048,7 @@ fn run_local_host_invocation_with_timeout( output, error, usage: agent_result.usage.as_ref().map(run_usage), + resource_snapshot, artifacts, telemetry: telemetry_ref(plan, relay_config.as_ref()), events, @@ -2008,6 +2068,7 @@ fn invalidate_timed_out_local_host( .is_some_and(|host| Arc::ptr_eq(host, expected_host)) { hosts.remove(runtime_id); + local_host_processes().remove(runtime_id); } } @@ -2163,6 +2224,12 @@ fn local_hosts() -> std::sync::MutexGuard<'static, BTreeMap std::sync::MutexGuard<'static, BTreeMap> { + LOCAL_HOST_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) +} + fn local_services() -> std::sync::MutexGuard<'static, BTreeMap>>> { LOCAL_SERVICES @@ -3936,6 +4003,10 @@ for line in sys.stdin: "choices": [{"index": 0, "delta": {"content": "lo"}}], }, ]) + if MODE == "busy": + # A descendant that burns 1.2 s of CPU, then exits and is waited for. + subprocess.run([sys.executable, "-c", + "import time\nwhile time.process_time() < 1.2:\n pass"], check=True) output = { "host_pid": os.getpid(), "invocation_count": invocations, @@ -4670,6 +4741,177 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + fn assert_unavailable(result: &Result, runtime_id: &str) { + match result { + Err(FabricError::ResourceObservationUnavailable { + runtime_id: actual, .. + }) => assert_eq!(actual, runtime_id), + other => panic!("expected ResourceObservationUnavailable, got {other:?}"), + } + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn runtime_resource_snapshot_of_active_runtime_reads_host_tree() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + + let snapshot = runtime_resource_snapshot(&plan, &runtime).expect("snapshot"); + stop_runtime(&plan, &runtime).expect("stop local host"); + + assert_eq!(snapshot.runtime_id, runtime.runtime_id); + assert!(snapshot.process_count >= 1); + assert!(snapshot.resident_memory_bytes > 0); + assert!(snapshot.cpu_time_seconds >= 0.0); + assert!(snapshot.started_at.ends_with('Z')); + assert!(snapshot.started_at <= snapshot.observed_at); + let _ = fs::remove_dir_all(root); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn runtime_resource_snapshot_during_invocation_returns_before_invocation_ends() { + let (root, plan) = local_host_plan("busy"); + let runtime = start_runtime(&plan).expect("start local host"); + + let saw_descendant = thread::scope(|scope| { + let invocation = + scope.spawn(|| invoke_runtime(&plan, &runtime, RunRequest::text("work"))); + let mut saw_descendant = false; + while !saw_descendant && !invocation.is_finished() { + let snapshot = runtime_resource_snapshot(&plan, &runtime).expect("live snapshot"); + saw_descendant = snapshot.process_count >= 2 && !invocation.is_finished(); + thread::sleep(Duration::from_millis(20)); + } + invocation + .join() + .expect("invocation thread") + .expect("invoke local host"); + saw_descendant + }); + stop_runtime(&plan, &runtime).expect("stop local host"); + + assert!(saw_descendant, "no snapshot saw the running descendant"); + let _ = fs::remove_dir_all(root); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn invocation_result_carries_final_resource_snapshot_with_waited_descendant_cpu() { + let (root, plan) = local_host_plan("busy"); + let runtime = start_runtime(&plan).expect("start local host"); + + let result = + invoke_runtime(&plan, &runtime, RunRequest::text("work")).expect("invoke local host"); + stop_runtime(&plan, &runtime).expect("stop local host"); + + let snapshot = result.resource_snapshot.expect("final resource snapshot"); + assert_eq!(snapshot.runtime_id, result.runtime_id); + assert!(snapshot.process_count >= 1); + assert!( + snapshot.cpu_time_seconds >= 1.0, + "descendant CPU missing: {snapshot:?}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn runtime_resource_snapshot_after_stop_is_unavailable() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + stop_runtime(&plan, &runtime).expect("stop local host"); + + let result = runtime_resource_snapshot(&plan, &runtime); + + if cfg!(any(target_os = "linux", target_os = "macos")) { + assert_unavailable(&result, &runtime.runtime_id); + } else { + assert!(matches!( + result, + Err(FabricError::ResourceObservationUnsupported { .. }) + )); + } + let _ = fs::remove_dir_all(root); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn runtime_resource_snapshot_after_host_exit_is_unavailable() { + let (root, plan) = local_host_plan("crash_after_start"); + let runtime = start_runtime(&plan).expect("start local host"); + + let deadline = Instant::now() + Duration::from_secs(10); + let mut result = runtime_resource_snapshot(&plan, &runtime); + while result.is_ok() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(50)); + result = runtime_resource_snapshot(&plan, &runtime); + } + let _ = stop_runtime(&plan, &runtime); + + assert_unavailable(&result, &runtime.runtime_id); + let _ = fs::remove_dir_all(root); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn runtime_resource_snapshot_during_concurrent_stop_is_snapshot_or_unavailable() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + + let results = thread::scope(|scope| { + let snapshots = scope.spawn(|| { + let mut results = Vec::new(); + loop { + let result = runtime_resource_snapshot(&plan, &runtime); + let stopped = result.is_err(); + results.push(result); + if stopped { + return results; + } + } + }); + thread::sleep(Duration::from_millis(50)); + stop_runtime(&plan, &runtime).expect("stop local host"); + snapshots.join().expect("snapshot thread") + }); + + let (last, earlier) = results.split_last().expect("at least one snapshot"); + assert!(earlier.iter().all(Result::is_ok)); + assert_unavailable(last, &runtime.runtime_id); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn runtime_resource_snapshot_of_non_local_environment_is_unsupported() { + let (root, plan) = local_host_plan("success"); + let mut environment = prepare_environment(&plan).expect("prepare environment"); + environment.provider = "remote".to_string(); + let runtime_id = new_id("runtime"); + let runtime = RuntimeHandle { + runtime_binding: runtime_binding(&runtime_id, None, &plan, &environment) + .expect("runtime binding"), + runtime_id, + agent_name: plan.agent_name.clone(), + harness: harness(&plan), + adapter_kind: adapter_kind(&plan), + adapter_id: adapter_id(&plan), + service_id: None, + environment, + }; + + let result = runtime_resource_snapshot(&plan, &runtime); + + assert!( + matches!( + &result, + Err(FabricError::ResourceObservationUnsupported { runtime_id, reason }) + if *runtime_id == runtime.runtime_id && reason.contains("remote") + ), + "{result:?}" + ); + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_relay_config_is_runtime_scoped() { let (root, plan) = local_host_plan_with_relay("success", true); diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index ee15fda12..69589cfa9 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -13,6 +13,7 @@ use crate::config::{ AdapterDescriptor, AdapterTargetDescriptor, AgentConfig, FabricConfig, RunPlan, }; use crate::error::{FabricError, Result}; +use crate::resource_snapshot::RuntimeResourceSnapshot; use crate::runtime::{ AdapterInvocation, ArtifactManifest, EnvironmentHandle, ErrorInfo, FabricEvent, InvocationHandle, OpenAiStreamInvocation, OpenAiStreamRecord, RunRequest, RunResult, @@ -65,11 +66,13 @@ pub enum SchemaName { ErrorInfo, /// NeMo Fabric lifecycle event schema. FabricEvent, + /// Runtime resource snapshot schema. + RuntimeResourceSnapshot, } impl SchemaName { /// All public schemas in stable output order. - pub const ALL: [Self; 21] = [ + pub const ALL: [Self; 22] = [ Self::Agent, Self::AgentConfig, Self::AgentRunRequest, @@ -91,6 +94,7 @@ impl SchemaName { Self::ArtifactManifest, Self::ErrorInfo, Self::FabricEvent, + Self::RuntimeResourceSnapshot, ]; /// Stable file stem for this schema. @@ -117,6 +121,7 @@ impl SchemaName { Self::ArtifactManifest => "artifact-manifest", Self::ErrorInfo => "error-info", Self::FabricEvent => "fabric-event", + Self::RuntimeResourceSnapshot => "runtime-resource-snapshot", } } @@ -170,6 +175,9 @@ impl SchemaName { "artifact-manifest" | "artifact_manifest" => Ok(Self::ArtifactManifest), "error-info" | "error_info" => Ok(Self::ErrorInfo), "fabric-event" | "fabric_event" => Ok(Self::FabricEvent), + "runtime-resource-snapshot" | "runtime_resource_snapshot" => { + Ok(Self::RuntimeResourceSnapshot) + } _ => Err(FabricError::UnknownSchema { schema: value.to_string(), available: Self::ALL @@ -205,6 +213,7 @@ pub fn generate_schema(schema: SchemaName) -> Result { SchemaName::ArtifactManifest => to_value(schema_for!(ArtifactManifest)), SchemaName::ErrorInfo => to_value(schema_for!(ErrorInfo)), SchemaName::FabricEvent => to_value(schema_for!(FabricEvent)), + SchemaName::RuntimeResourceSnapshot => to_value(schema_for!(RuntimeResourceSnapshot)), } } diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 52a3b6f57..6c276ae6b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 125 +position: 127 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 26a286a0c..fa45b1f21 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 126 +position: 128 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 2c89a0e39..218c1dde6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 127 +position: 129 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index fa7255361..7a918b620 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 128 +position: 130 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index ea299ed94..f0f77cf94 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 129 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 95cf2f59a..4c5c5da33 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -497,6 +497,38 @@ Environment provider. Adapter kind. +### `ResourceObservationUnavailable` + +
+ +A runtime's resources cannot be read now: the runtime stopped, is stopping, its adapter host process exited, or the process table cannot be read. + +#### Fields + +##### `runtime_id: String` + +Runtime handle id. + +##### `reason: String` + +Why the runtime cannot be observed. + +### `ResourceObservationUnsupported` + +
+ +The runtime's environment provider or operating system has no resource reader. + +#### Fields + +##### `runtime_id: String` + +Runtime handle id. + +##### `reason: String` + +What has no resource reader. + ### `InvalidProcessSettings`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index cf7208b32..ca57e0275 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 130 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 8af97f312..6b00f4d18 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 133 +position: 136 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 6670ab4d8..045826ad4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -92,6 +92,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use doctor::doctor_plan;` - `pub use error::FabricError;` - `pub use error::Result;` +- `pub use resource_snapshot::RuntimeResourceSnapshot;` - `pub use runtime::AdapterInvocation;` - `pub use runtime::ArtifactManifest;` - `pub use runtime::ArtifactRef;` @@ -129,6 +130,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use runtime::prepare_service;` - `pub use runtime::release_service;` - `pub use runtime::run_plan;` +- `pub use runtime::runtime_resource_snapshot;` - `pub use runtime::start_runtime;` - `pub use runtime::start_runtime_with_service;` - `pub use runtime::stop_runtime;` @@ -146,6 +148,7 @@ Core config and runtime contract for NeMo Fabric. - [config](config/index.mdx): NeMo Fabric config models and loading helpers. - [doctor](doctor/index.mdx): Plan diagnostics for NeMo Fabric. - [error](error/index.mdx): Error types for NeMo Fabric core. +- [resource_snapshot](resource-snapshot/index.mdx): Resource snapshots of a runtime's local adapter host and its descendants. - [runtime](runtime/index.mdx): Runtime invocation helpers. - [schema](schema/index.mdx): JSON Schema generation for the public NeMo Fabric contract. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx new file mode 100644 index 000000000..7ff9eab7c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -0,0 +1,21 @@ +--- +title: "Module resource_snapshot" +sidebar-title: "resource_snapshot" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" +description: "Resource snapshots of a runtime's local adapter host and its descendants." +position: 133 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Resource snapshots of a runtime's local adapter host and its descendants. + +A snapshot reads the process tree rooted at the adapter host process that NeMo Fabric started for a runtime. Each read visits a parent before its children. CPU time adds every process's own CPU time and the CPU time of the children it has already waited for, so work done by finished descendants still counts. A child reaped between the read of its parent and its own read is missed by that one snapshot and is never counted twice. CPU time can drop between snapshots when a descendant is reparented out of the tree, and on macOS while an exited descendant is not yet waited for, because macOS cannot list an unreaped process. + +Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. Neither reads command lines, environment variables, or open files. + +## Structs + +- [RuntimeResourceSnapshot](struct-runtimeresourcesnapshot.mdx): Timestamped resource usage of one runtime's local adapter host process tree. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx new file mode 100644 index 000000000..82f8493e1 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx @@ -0,0 +1,117 @@ +--- +title: "Struct Runtime Resource Snapshot" +sidebar-title: "RuntimeResourceSnapshot" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot/struct-runtimeresourcesnapshot" +description: "Timestamped resource usage of one runtime's local adapter host process tree." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub started_at: String,\n    pub observed_at: String,\n    pub resident_memory_bytes: u64,\n    pub process_count: u64,\n    pub cpu_time_seconds: f64,\n}"}} />
+ +Timestamped resource usage of one runtime's local adapter host process tree. + +Memory, process count, and CPU time cover the adapter host process and its live descendants. For a runtime started on a service, the shared service process is not included. Values Fabric cannot read are returned as an error, never as zero. + +## Fields + +### `runtime_id: String` + +Runtime this snapshot describes. + +### `started_at: String` + +When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. + +### `observed_at: String` + +When Fabric read these values, as an RFC 3339 UTC timestamp. + +### `resident_memory_bytes: u64` + +Sum of the resident set sizes of the live processes, in bytes. Pages shared between processes count once per process. + +### `process_count: u64` + +Live processes in the tree, including the adapter host. Exited processes that are not yet reaped are not counted. + +### `cpu_time_seconds: f64` + +CPU time consumed by the tree since the adapter host started, in seconds, including descendants that have exited and been waited for. It can drop when a descendant leaves the tree, and on macOS while an exited descendant is not yet waited for. + +## Trait Implementations + +### `impl Clone for RuntimeResourceSnapshot` + +
Clone for RuntimeResourceSnapshot"}} />
+ +#### `clone` + +
clone(&self) -> RuntimeResourceSnapshot"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RuntimeResourceSnapshot` + +
Debug for RuntimeResourceSnapshot"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for RuntimeResourceSnapshot` + +
Deserialize<'de> for RuntimeResourceSnapshot"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RuntimeResourceSnapshot` + +
RuntimeResourceSnapshot"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RuntimeResourceSnapshot` + +
PartialEq for RuntimeResourceSnapshot"}} />
+ +#### `eq` + +
eq(&self, other: &RuntimeResourceSnapshot) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RuntimeResourceSnapshot` + +
Serialize for RuntimeResourceSnapshot"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for RuntimeResourceSnapshot` + +
StructuralPartialEq for RuntimeResourceSnapshot"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx new file mode 100644 index 000000000..cbe246229 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx @@ -0,0 +1,16 @@ +--- +title: "Function runtime_resource_snapshot" +sidebar-title: "runtime_resource_snapshot" +description: "Read a timestamped resource snapshot of a started runtime." +position: 38 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
RunPlan,\n    runtime: &RuntimeHandle,\n) -> Result<RuntimeResourceSnapshot>"}} />
+ +Read a timestamped resource snapshot of a started runtime. + +The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, or the process table cannot be read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the environment provider or operating system has no resource reader. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx index 547b47760..c61b166c3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_with_service" sidebar-title: "start_runtime_with_service" description: "Start a runtime connected to an active service." -position: 39 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx index 0f135dffc..04a3c9019 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime" sidebar-title: "start_runtime" description: "Start or connect to a harness runtime." -position: 38 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index 71febb037..a55e5bd91 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 40 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index e4ce802a1..309f520c4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 131 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -56,6 +56,7 @@ Runtime invocation helpers. - [prepare_service](fn-prepare-service.mdx): Prepare a Fabric-owned long-lived service for a resolved run plan. - [release_service](fn-release-service.mdx): Release a Fabric-owned service or detach from a caller-owned service. - [run_plan](fn-run-plan.mdx): Invoke a NeMo Fabric run plan. +- [runtime_resource_snapshot](fn-runtime-resource-snapshot.mdx): Read a timestamped resource snapshot of a started runtime. - [start_runtime](fn-start-runtime.mdx): Start or connect to a harness runtime. - [start_runtime_with_service](fn-start-runtime-with-service.mdx): Start a runtime connected to an active service. - [stop_runtime](fn-stop-runtime.mdx): Stop or detach from a harness runtime. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 1cb5bd052..3e05964a5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub resource_snapshot: Option<RuntimeResourceSnapshot>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
Result from a NeMo Fabric-managed harness invocation. @@ -59,6 +59,10 @@ Error metadata when applicable. Normalized invocation usage when reported by the adapter target. +### `resource_snapshot: Option` + +Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed. + ### `artifacts: ArtifactManifest` Artifacts produced or collected by this run. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index 9ae563e23..9b8e9dd87 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -32,6 +32,7 @@ pub enum SchemaName { ArtifactManifest, ErrorInfo, FabricEvent, + RuntimeResourceSnapshot, } ``` @@ -165,6 +166,12 @@ Structured error metadata schema. NeMo Fabric lifecycle event schema. +### `RuntimeResourceSnapshot` + +
+ +Runtime resource snapshot schema. + ## Implementations ### `impl SchemaName` @@ -173,7 +180,7 @@ NeMo Fabric lifecycle event schema. #### `ALL` -
21]"}} />
+
22]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 0569b34b2..a60dea2c9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 132 +position: 135 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 43867290c..37c9904c3 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -127,6 +127,8 @@ descriptor schema. - `sdk/artifact-manifest`: normalized artifact references. - `sdk/error-info`: structured runtime or adapter error metadata. - `sdk/fabric-event`: NeMo Fabric lifecycle or progress event. +- `sdk/runtime-resource-snapshot`: timestamped memory, process count, and CPU + time of a runtime's local adapter host and its descendants. ### Deferred Core Objects diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 034c17e82..e27c1d1cc 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -279,6 +279,52 @@ }, "type": "object" }, + "RuntimeResourceSnapshot": { + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", + "properties": { + "cpu_time_seconds": { + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding descendants that have exited and been waited for. It can drop\nwhen a descendant leaves the tree, and on macOS while an exited\ndescendant is not yet waited for.", + "format": "double", + "minimum": 0.0, + "type": "number" + }, + "observed_at": { + "description": "When Fabric read these values, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + }, + "process_count": { + "description": "Live processes in the tree, including the adapter host. Exited processes\nthat are not yet reaped are not counted.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "resident_memory_bytes": { + "description": "Sum of the resident set sizes of the live processes, in bytes. Pages\nshared between processes count once per process.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "runtime_id": { + "description": "Runtime this snapshot describes.", + "type": "string" + }, + "started_at": { + "description": "When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + } + }, + "required": [ + "runtime_id", + "started_at", + "observed_at", + "resident_memory_bytes", + "process_count", + "cpu_time_seconds" + ], + "type": "object" + }, "TelemetryRef": { "description": "Reference to telemetry emitted by Relay or another configured telemetry path.", "properties": { @@ -360,6 +406,17 @@ "description": "Request id.", "type": "string" }, + "resource_snapshot": { + "anyOf": [ + { + "$ref": "#/$defs/RuntimeResourceSnapshot" + }, + { + "type": "null" + } + ], + "description": "Resource snapshot of the runtime, read once after the adapter returned this\nresult. Absent when the runtime's resources could not be observed." + }, "runtime_id": { "description": "Runtime handle id.", "type": "string" diff --git a/schemas/sdk/runtime-resource-snapshot.schema.json b/schemas/sdk/runtime-resource-snapshot.schema.json new file mode 100644 index 000000000..86a36a5af --- /dev/null +++ b/schemas/sdk/runtime-resource-snapshot.schema.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", + "properties": { + "cpu_time_seconds": { + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding descendants that have exited and been waited for. It can drop\nwhen a descendant leaves the tree, and on macOS while an exited\ndescendant is not yet waited for.", + "format": "double", + "minimum": 0.0, + "type": "number" + }, + "observed_at": { + "description": "When Fabric read these values, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + }, + "process_count": { + "description": "Live processes in the tree, including the adapter host. Exited processes\nthat are not yet reaped are not counted.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "resident_memory_bytes": { + "description": "Sum of the resident set sizes of the live processes, in bytes. Pages\nshared between processes count once per process.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "runtime_id": { + "description": "Runtime this snapshot describes.", + "type": "string" + }, + "started_at": { + "description": "When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + } + }, + "required": [ + "runtime_id", + "started_at", + "observed_at", + "resident_memory_bytes", + "process_count", + "cpu_time_seconds" + ], + "title": "RuntimeResourceSnapshot", + "type": "object" +} \ No newline at end of file From 0e016fda139504419fe8ffa536ef312bb2d6eb3c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 15:52:50 -0700 Subject: [PATCH 002/102] feat: expose runtime resource snapshots in the Python SDK Python applications need the same snapshot as Rust callers, with the same typed failures, to monitor a runtime while an invocation runs. Add Runtime.resource_snapshot(), the RuntimeResourceSnapshot type, RunResult.resource_snapshot, and ResourceObservationUnavailableError and ResourceObservationUnsupportedError (FabricRuntimeError subclasses). The PyO3 binding raises dedicated native exceptions so the SDK maps them by type rather than by message text. Signed-off-by: mnajafian-nv --- crates/fabric-python/src/lib.rs | 53 ++++- .../api/python-library-reference/index.md | 3 + .../nemo_fabric.errors.md | 82 +++++++ .../nemo_fabric.runtime.md | 21 ++ .../nemo_fabric.types.md | 85 +++++++ .../src/nemo_fabric/__init__.py | 6 + .../src/nemo_fabric/_native.pyi | 4 + .../src/nemo_fabric/errors.py | 12 + .../src/nemo_fabric/runtime.py | 65 +++++- .../src/nemo_fabric/types.py | 75 +++++++ .../python/test_runtime_resource_snapshot.py | 211 ++++++++++++++++++ 11 files changed, 610 insertions(+), 7 deletions(-) create mode 100644 tests/python/test_runtime_resource_snapshot.py diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 2f9cb1f37..44587f33b 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -24,6 +24,18 @@ pyo3::create_exception!( PyRuntimeError, "Raised when a service release is rejected because runtimes are still active." ); +pyo3::create_exception!( + _native, + ResourceObservationUnavailableError, + PyRuntimeError, + "Raised when a runtime stopped, is stopping, its adapter host exited, or the process table cannot be read." +); +pyo3::create_exception!( + _native, + ResourceObservationUnsupportedError, + PyRuntimeError, + "Raised when a runtime's environment provider or operating system has no resource reader." +); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; const PYTHON_DATA_PATH_QUERY_TIMEOUT: Duration = Duration::from_secs(5); @@ -246,9 +258,32 @@ fn stop_runtime(py: Python<'_>, plan_json: String, runtime_json: String) -> PyRe to_json(&events) } +/// Read a resource snapshot of a started runtime and return RuntimeResourceSnapshot JSON. +#[pyfunction] +fn runtime_resource_snapshot( + py: Python<'_>, + plan_json: String, + runtime_json: String, +) -> PyResult { + let plan = parse_run_plan(plan_json)?; + let runtime = parse_runtime_handle(runtime_json)?; + let snapshot = py + .detach(|| nemo_fabric_core::runtime_resource_snapshot(&plan, &runtime)) + .map_err(to_py_error)?; + to_json(&snapshot) +} + #[pymodule] fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("ServiceInUseError", m.py().get_type::())?; + m.add( + "ResourceObservationUnavailableError", + m.py().get_type::(), + )?; + m.add( + "ResourceObservationUnsupportedError", + m.py().get_type::(), + )?; m.add_function(wrap_pyfunction!(version, m)?)?; m.add_function(wrap_pyfunction!(plan_config, m)?)?; m.add_function(wrap_pyfunction!(inspect_adapter_metadata, m)?)?; @@ -262,6 +297,7 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(invoke_runtime, m)?)?; m.add_function(wrap_pyfunction!(invoke_openai_stream, m)?)?; m.add_function(wrap_pyfunction!(stop_runtime, m)?)?; + m.add_function(wrap_pyfunction!(runtime_resource_snapshot, m)?)?; Ok(()) } @@ -273,13 +309,18 @@ where } fn to_py_error(error: nemo_fabric_core::FabricError) -> PyErr { - if matches!( - &error, - FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" - ) { - return PyTimeoutError::new_err(error.to_string()); + match &error { + FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" => { + PyTimeoutError::new_err(error.to_string()) + } + FabricError::ResourceObservationUnavailable { .. } => { + ResourceObservationUnavailableError::new_err(error.to_string()) + } + FabricError::ResourceObservationUnsupported { .. } => { + ResourceObservationUnsupportedError::new_err(error.to_string()) + } + _ => PyRuntimeError::new_err(error.to_string()), } - PyRuntimeError::new_err(error.to_string()) } fn resolve_context( diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index c1bdd6e0c..24fb1d403 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -75,6 +75,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`types.RunUsage`](./nemo_fabric.types.md#class-runusage): Normalized invocation-local usage reported by an adapter target. - [`types.RuntimeCapabilities`](./nemo_fabric.types.md#class-runtimecapabilities): Operations declared by the resolved runtime and adapter. - [`types.RuntimeHandle`](./nemo_fabric.types.md#class-runtimehandle): Opaque identity and binding for one started runtime. +- [`types.RuntimeResourceSnapshot`](./nemo_fabric.types.md#class-runtimeresourcesnapshot): Timestamped resource usage of one runtime's local adapter host process tree. - [`types.ServiceHandle`](./nemo_fabric.types.md#class-servicehandle): Opaque identity and binding for one prepared or attached service. - [`types.ServiceReference`](./nemo_fabric.types.md#class-servicereference): Reference to an already-running caller-owned service. - [`types.TelemetryRef`](./nemo_fabric.types.md#class-telemetryref): Reference to external or persisted telemetry for a run. @@ -84,6 +85,8 @@ SPDX-License-Identifier: Apache-2.0 --> - [`errors.FabricNativeUnavailableError`](./nemo_fabric.errors.md#class-fabricnativeunavailableerror): SDK call requires the PyO3 extension, but it is not installed or importable. - [`errors.FabricRuntimeError`](./nemo_fabric.errors.md#class-fabricruntimeerror): Failure while starting, invoking, stopping, or otherwise driving a runtime. - [`errors.FabricStateError`](./nemo_fabric.errors.md#class-fabricstateerror): Operation rejected because a local runtime is in the wrong state. +- [`errors.ResourceObservationUnavailableError`](./nemo_fabric.errors.md#class-resourceobservationunavailableerror): Runtime resources cannot be read now. +- [`errors.ResourceObservationUnsupportedError`](./nemo_fabric.errors.md#class-resourceobservationunsupportederror): The runtime's environment provider or operating system has no resource reader. ## Functions diff --git a/docs/reference/api/python-library-reference/nemo_fabric.errors.md b/docs/reference/api/python-library-reference/nemo_fabric.errors.md index 4b5559100..73b6ba7a8 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.errors.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.errors.md @@ -275,6 +275,88 @@ Initialize a structured NeMo Fabric exception. +--- + +## class `ResourceObservationUnavailableError` + +Runtime resources cannot be read now. + +The runtime stopped, is stopping, its adapter host process exited, or the process table cannot be read. + + +### Inheritance + +Direct base: `FabricRuntimeError`. + +### method `__init__` + +```python +def __init__( + message: str, + *, + stage: str | None = None, + code: str | None = None, + retryable: bool = False, + details: Mapping[str, Any] | None = None, +) -> None +``` + +Initialize a structured NeMo Fabric exception. + + + +**Args:** + + - `message`: Human-readable failure description. + - `stage`: Optional lifecycle stage that failed. + - `code`: Optional stable machine-readable error code. + - `retryable`: Whether callers may safely retry unchanged input. + - `details`: Optional structured diagnostic context. The exception stores a deep copy. + + + + + +--- + +## class `ResourceObservationUnsupportedError` + +The runtime's environment provider or operating system has no resource reader. + + +### Inheritance + +Direct base: `FabricRuntimeError`. + +### method `__init__` + +```python +def __init__( + message: str, + *, + stage: str | None = None, + code: str | None = None, + retryable: bool = False, + details: Mapping[str, Any] | None = None, +) -> None +``` + +Initialize a structured NeMo Fabric exception. + + + +**Args:** + + - `message`: Human-readable failure description. + - `stage`: Optional lifecycle stage that failed. + - `code`: Optional stable machine-readable error code. + - `retryable`: Whether callers may safely retry unchanged input. + - `details`: Optional structured diagnostic context. The exception stores a deep copy. + + + + + --- diff --git a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md index a9bde67df..b77596c57 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md @@ -183,6 +183,27 @@ Start one turn and stream raw NeMo Relay ATOF records as they arrive. --- +### method `resource_snapshot` + +```python +async def resource_snapshot() -> RuntimeResourceSnapshot +``` + +Return a timestamped resource snapshot of this runtime. + +The snapshot covers the local adapter host process and its live descendants. It does not wait for a running invocation, so it can be called while ``invoke()`` is in flight. It observes resources only and enforces no limits. + + + +**Raises:** + + - `ResourceObservationUnavailableError`: If the runtime stopped, is stopping, its adapter host process exited, or the process table cannot be read. + - `ResourceObservationUnsupportedError`: If the runtime's environment provider or operating system has no resource reader. + - `FabricNativeUnavailableError`: If the native extension is missing. + - `FabricRuntimeError`: If the native snapshot call fails otherwise. + +--- + ### method `stop` ```python diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 21a7ff445..528624f3e 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1244,6 +1244,89 @@ Return a detached, JSON-compatible mapping for serialization. --- +## class `RuntimeResourceSnapshot` + +Timestamped resource usage of one runtime's local adapter host process tree. + +Memory, process count, and CPU time cover the adapter host process and its live descendants, not a shared service process. Values that cannot be read raise an error instead of reading as zero. + + + +**Attributes:** + + - `runtime_id`: Runtime this snapshot describes. + - `started_at`: When Fabric started the adapter host, as RFC 3339 UTC. + - `observed_at`: When Fabric read these values, as RFC 3339 UTC. + - `resident_memory_bytes`: Sum of the live processes' resident set sizes. Pages shared between processes count once per process. + - `process_count`: Live processes in the tree, including the adapter host. + - `cpu_time_seconds`: CPU time used since the adapter host started, including descendants that have exited and been waited for. It can drop when a descendant leaves the tree. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `runtime_id` | `str` | +| `started_at` | `str` | +| `observed_at` | `str` | +| `resident_memory_bytes` | `int` | +| `process_count` | `int` | +| `cpu_time_seconds` | `float` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + ## class `RunResult` Normalized terminal result from one NeMo Fabric invocation. @@ -1265,6 +1348,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - `error`: Structured failure, or ``None`` on success. - `usage`: Normalized invocation usage, or ``None`` when unavailable. + - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. - `artifacts`: Normalized artifact manifest. - `telemetry`: Ordered telemetry references. - `events`: Ordered lifecycle and invocation events. @@ -1289,6 +1373,7 @@ The mapping exposes the following typed fields: | `output` | `RunOutput \| JSONValue` | | `error` | `ErrorInfo \| None` | | `usage` | `RunUsage \| None` | +| `resource_snapshot` | `RuntimeResourceSnapshot \| None` | | `artifacts` | `ArtifactManifest` | | `telemetry` | `Sequence[TelemetryRef]` | | `events` | `Sequence[FabricEvent]` | diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index aaa15bfe0..bfc60118f 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -12,6 +12,8 @@ from nemo_fabric.errors import FabricNativeUnavailableError from nemo_fabric.errors import FabricRuntimeError from nemo_fabric.errors import FabricStateError +from nemo_fabric.errors import ResourceObservationUnavailableError +from nemo_fabric.errors import ResourceObservationUnsupportedError from nemo_fabric.models import EnvironmentConfig from nemo_fabric.models import DiscoveryConfig from nemo_fabric.models import FabricBaseModel @@ -63,6 +65,7 @@ from nemo_fabric.types import RunUsage from nemo_fabric.types import RuntimeCapabilities from nemo_fabric.types import RuntimeHandle +from nemo_fabric.types import RuntimeResourceSnapshot from nemo_fabric.types import ServiceHandle from nemo_fabric.types import ServiceReference from nemo_fabric.types import TelemetryRef @@ -109,6 +112,8 @@ "FabricNativeUnavailableError", "FabricRuntimeError", "FabricStateError", + "ResourceObservationUnavailableError", + "ResourceObservationUnsupportedError", "RunOutput", "RunPlan", "RunRequest", @@ -116,6 +121,7 @@ "RunUsage", "RuntimeCapabilities", "RuntimeHandle", + "RuntimeResourceSnapshot", "RuntimeConfig", "Runtime", "RuntimeStatus", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi index 30322d181..9b969fc02 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi @@ -36,3 +36,7 @@ def invoke_openai_stream( transport_json: str, ) -> str: ... def stop_runtime(plan_json: str, runtime_json: str) -> str: ... +def runtime_resource_snapshot(plan_json: str, runtime_json: str) -> str: ... + +class ResourceObservationUnavailableError(RuntimeError): ... +class ResourceObservationUnsupportedError(RuntimeError): ... diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index 1ac906fcf..5edb2bbf4 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -79,3 +79,15 @@ class FabricCapabilityError(FabricRuntimeError): class FabricNativeUnavailableError(FabricRuntimeError): """SDK call requires the PyO3 extension, but it is not installed or importable.""" + + +class ResourceObservationUnavailableError(FabricRuntimeError): + """Runtime resources cannot be read now. + + The runtime stopped, is stopping, its adapter host process exited, or the + process table cannot be read. + """ + + +class ResourceObservationUnsupportedError(FabricRuntimeError): + """The runtime's environment provider or operating system has no resource reader.""" diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index b5a6e11eb..28be1b623 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -22,13 +22,21 @@ FabricCapabilityError, FabricConfigError, FabricError, + FabricRuntimeError, FabricStateError, + ResourceObservationUnavailableError, + ResourceObservationUnsupportedError, _runtime_error, ) from nemo_fabric.models import RunRequest from nemo_fabric.openai_streaming import OpenAIInvokeStream from nemo_fabric.streaming import InvokeStream -from nemo_fabric.types import RunPlan, RunResult, RuntimeHandle +from nemo_fabric.types import ( + RunPlan, + RunResult, + RuntimeHandle, + RuntimeResourceSnapshot, +) logger = logging.getLogger(__name__) @@ -731,6 +739,41 @@ def _ensure_invocable(self) -> None: if self._current_task is not None: raise FabricStateError("runtime is already running an invocation") + async def resource_snapshot(self) -> RuntimeResourceSnapshot: + """Return a timestamped resource snapshot of this runtime. + + The snapshot covers the local adapter host process and its live + descendants. It does not wait for a running invocation, so it can be + called while ``invoke()`` is in flight. It observes resources only and + enforces no limits. + + Raises: + ResourceObservationUnavailableError: If the runtime stopped, is + stopping, its adapter host process exited, or the process + table cannot be read. + ResourceObservationUnsupportedError: If the runtime's environment + provider or operating system has no resource reader. + FabricNativeUnavailableError: If the native extension is missing. + FabricRuntimeError: If the native snapshot call fails otherwise. + """ + + native = self._client._require_native_module("resource_snapshot") + + def snapshot() -> Any: + return json.loads( + native.runtime_resource_snapshot( + json.dumps(self._plan.to_mapping()), + json.dumps(self._runtime.to_mapping()), + ) + ) + + try: + return RuntimeResourceSnapshot.from_mapping(await _call_blocking(snapshot)) + except FabricError: + raise + except Exception as error: + raise _resource_snapshot_error(native, error) from error + async def stop(self) -> None: """Destroy an idle runtime exactly once. @@ -985,6 +1028,26 @@ def run() -> dict[str, Any]: raise _runtime_error(error, stage="run") from error +def _resource_snapshot_error(native: Any, error: Exception) -> FabricRuntimeError: + """Map the native resource-observation errors to their SDK types.""" + typed_errors: tuple[tuple[str, type[FabricRuntimeError], str], ...] = ( + ( + "ResourceObservationUnavailableError", + ResourceObservationUnavailableError, + "resource_observation_unavailable", + ), + ( + "ResourceObservationUnsupportedError", + ResourceObservationUnsupportedError, + "resource_observation_unsupported", + ), + ) + for native_name, sdk_type, code in typed_errors: + if isinstance(error, getattr(native, native_name, ())): + return sdk_type(str(error), stage="resource_snapshot", code=code) + return _runtime_error(error, stage="resource_snapshot") + + async def _call_blocking(func: Any) -> Any: worker = asyncio.create_task(asyncio.to_thread(func)) try: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 04504e4dc..07e22c7ac 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1834,6 +1834,71 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: return data +class RuntimeResourceSnapshot(FabricMapping): + """Timestamped resource usage of one runtime's local adapter host process tree. + + Memory, process count, and CPU time cover the adapter host process and its + live descendants, not a shared service process. Values that cannot be read + raise an error instead of reading as zero. + + Attributes: + runtime_id: Runtime this snapshot describes. + started_at: When Fabric started the adapter host, as RFC 3339 UTC. + observed_at: When Fabric read these values, as RFC 3339 UTC. + resident_memory_bytes: Sum of the live processes' resident set sizes. + Pages shared between processes count once per process. + process_count: Live processes in the tree, including the adapter host. + cpu_time_seconds: CPU time used since the adapter host started, + including descendants that have exited and been waited for. It + can drop when a descendant leaves the tree. + """ + + runtime_id: str + started_at: str + observed_at: str + resident_memory_bytes: int + process_count: int + cpu_time_seconds: float + _fields = frozenset( + { + "runtime_id", + "started_at", + "observed_at", + "resident_memory_bytes", + "process_count", + "cpu_time_seconds", + } + ) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + for field in ("runtime_id", "started_at", "observed_at"): + data[field] = _required_text(data.get(field), field.replace("_", " ")) + for field in ("resident_memory_bytes", "process_count"): + value = data.get(field) + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 0 <= value <= _UINT64_MAX + ): + raise FabricConfigError( + f"{field.replace('_', ' ')} must be a nonnegative integer " + f"no greater than {_UINT64_MAX}" + ) + cpu = data.get("cpu_time_seconds") + if ( + isinstance(cpu, bool) + or not isinstance(cpu, (int, float)) + or not math.isfinite(cpu) + or cpu < 0 + ): + raise FabricConfigError( + "cpu time seconds must be a finite nonnegative number" + ) + data["cpu_time_seconds"] = float(cpu) + return data + + class RunResult(FabricMapping): """Normalized terminal result from one NeMo Fabric invocation. @@ -1854,6 +1919,9 @@ class RunResult(FabricMapping): are preserved as-is. error: Structured failure, or ``None`` on success. usage: Normalized invocation usage, or ``None`` when unavailable. + resource_snapshot: Runtime resource snapshot read once after the + adapter returned this result, or ``None`` when the runtime's + resources could not be observed. artifacts: Normalized artifact manifest. telemetry: Ordered telemetry references. events: Ordered lifecycle and invocation events. @@ -1871,6 +1939,7 @@ class RunResult(FabricMapping): output: RunOutput | JSONValue error: ErrorInfo | None usage: RunUsage | None + resource_snapshot: RuntimeResourceSnapshot | None artifacts: ArtifactManifest telemetry: Sequence[TelemetryRef] events: Sequence[FabricEvent] @@ -1888,6 +1957,7 @@ class RunResult(FabricMapping): "output", "error", "usage", + "resource_snapshot", "artifacts", "telemetry", "events", @@ -1914,6 +1984,11 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: data["usage"] = ( None if data.get("usage") is None else RunUsage.from_mapping(data["usage"]) ) + data["resource_snapshot"] = ( + None + if data.get("resource_snapshot") is None + else RuntimeResourceSnapshot.from_mapping(data["resource_snapshot"]) + ) data["artifacts"] = ArtifactManifest.from_mapping( data.get("artifacts", {"artifacts": []}) ) diff --git a/tests/python/test_runtime_resource_snapshot.py b/tests/python/test_runtime_resource_snapshot.py new file mode 100644 index 000000000..db63e546a --- /dev/null +++ b/tests/python/test_runtime_resource_snapshot.py @@ -0,0 +1,211 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Contract tests for runtime resource snapshots.""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import nemo_fabric._native as native +import pytest +from nemo_fabric import Fabric +from nemo_fabric import FabricConfig +from nemo_fabric import FabricConfigError +from nemo_fabric import FabricRuntimeError +from nemo_fabric import ResourceObservationUnavailableError +from nemo_fabric import ResourceObservationUnsupportedError +from nemo_fabric import RunRequest +from nemo_fabric import RunResult +from nemo_fabric import Runtime +from nemo_fabric import RuntimeResourceSnapshot + +_SNAPSHOT = { + "runtime_id": "runtime-1", + "started_at": "2026-10-15T09:00:00.000Z", + "observed_at": "2026-10-15T09:00:05.250Z", + "resident_memory_bytes": 52_428_800, + "process_count": 3, + "cpu_time_seconds": 1.25, +} +_SUPPORTED_PLATFORM = sys.platform in {"linux", "darwin"} + + +def _shim_config() -> FabricConfig: + return FabricConfig.from_mapping( + { + "metadata": {"name": "resource-snapshot-probe"}, + "harness": { + "adapter_id": "test.fabric.hermes_shim", + "resolution": "preinstalled", + }, + "discovery": {"local_paths": ["adapters"]}, + } + ) + + +def _runtime_with_native(native_module: Any) -> Runtime: + client = Fabric() + client._native_module = lambda: native_module # type: ignore[method-assign] + return Runtime( + client=client, + plan={ + "agent_name": "demo", + "base_dir": ".", + "config": { + "metadata": {"name": "demo"}, + "harness": {"adapter_id": "test.fabric.shim"}, + }, + "adapter_descriptor": { + "descriptor": { + "adapter_id": "test.fabric.shim", + "harness": "hermes", + "adapter_kind": "python", + } + }, + "capabilities": { + "service": False, + "streaming": False, + "updates": False, + "cancellation": False, + }, + }, + runtime={ + "runtime_id": "runtime-1", + "runtime_binding": "fabric-runtime-binding-test", + "agent_name": "demo", + "harness": "hermes", + "adapter_kind": "python", + "adapter_id": "test.fabric.shim", + "environment": { + "environment_id": "environment-1", + "provider": "local", + "control_location": "external_control", + "ownership": "caller_owned", + }, + }, + ) + + +def test_runtime_resource_snapshot_with_invalid_field_raises_config_error(): + snapshot = RuntimeResourceSnapshot.from_mapping(_SNAPSHOT) + assert snapshot.cpu_time_seconds == 1.25 + assert snapshot.to_mapping() == _SNAPSHOT + + for field, value in ( + ("resident_memory_bytes", -1), + ("process_count", True), + ("cpu_time_seconds", float("nan")), + ("observed_at", ""), + ): + with pytest.raises(FabricConfigError): + RuntimeResourceSnapshot.from_mapping({**_SNAPSHOT, field: value}) + + +def test_run_result_resource_snapshot_is_typed_or_none(): + base = { + "agent_name": "demo", + "harness": "hermes", + "adapter_kind": "python", + "runtime_id": "runtime-1", + "invocation_id": "invocation-1", + "request_id": "request-1", + "status": "succeeded", + } + + with_snapshot = RunResult.from_mapping({**base, "resource_snapshot": _SNAPSHOT}) + without_snapshot = RunResult.from_mapping(base) + + assert isinstance(with_snapshot.resource_snapshot, RuntimeResourceSnapshot) + assert with_snapshot.resource_snapshot.process_count == 3 + assert without_snapshot.resource_snapshot is None + + +@pytest.mark.parametrize( + ("native_name", "sdk_type", "code"), + [ + ( + "ResourceObservationUnavailableError", + ResourceObservationUnavailableError, + "resource_observation_unavailable", + ), + ( + "ResourceObservationUnsupportedError", + ResourceObservationUnsupportedError, + "resource_observation_unsupported", + ), + ], +) +async def test_resource_snapshot_native_error_maps_to_typed_sdk_error( + native_name, sdk_type, code +): + native_error = getattr(native, native_name) + fake_native = MagicMock() + fake_native.ResourceObservationUnavailableError = ( + native.ResourceObservationUnavailableError + ) + fake_native.ResourceObservationUnsupportedError = ( + native.ResourceObservationUnsupportedError + ) + fake_native.runtime_resource_snapshot.side_effect = native_error("no reader") + runtime = _runtime_with_native(fake_native) + + with pytest.raises(sdk_type) as caught: + await runtime.resource_snapshot() + + assert isinstance(caught.value, FabricRuntimeError) + assert caught.value.code == code + assert caught.value.stage == "resource_snapshot" + + +@pytest.mark.skipif(not _SUPPORTED_PLATFORM, reason="needs a process reader") +async def test_resource_snapshot_reads_active_runtime_and_final_result( + hermes_shim_agent_dir: Path, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setenv("ADAPTER_PYTHON", sys.executable) + runtime = await Fabric().start_runtime( + _shim_config(), base_dir=hermes_shim_agent_dir + ) + try: + invocation = asyncio.create_task( + runtime.invoke( + request=RunRequest(input="slow", context={"delay_seconds": 2}) + ) + ) + await asyncio.sleep(0.3) + live = await runtime.resource_snapshot() + assert not invocation.done(), "snapshot waited for the invocation" + result = await invocation + finally: + await runtime.stop() + + assert live.runtime_id == runtime.runtime_id + assert live.process_count >= 1 + assert live.resident_memory_bytes > 0 + assert isinstance(result.resource_snapshot, RuntimeResourceSnapshot) + assert result.resource_snapshot.runtime_id == runtime.runtime_id + assert result.resource_snapshot.started_at == live.started_at + with pytest.raises(ResourceObservationUnavailableError): + await runtime.resource_snapshot() + + +@pytest.mark.skipif(_SUPPORTED_PLATFORM, reason="platform has a process reader") +async def test_resource_snapshot_on_unsupported_platform_raises_unsupported( + hermes_shim_agent_dir: Path, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setenv("ADAPTER_PYTHON", sys.executable) + runtime = await Fabric().start_runtime( + _shim_config(), base_dir=hermes_shim_agent_dir + ) + try: + with pytest.raises(ResourceObservationUnsupportedError): + await runtime.resource_snapshot() + result = await runtime.invoke(input="hello") + finally: + await runtime.stop() + + assert result.resource_snapshot is None From ce4f5f4d7272404aca6aa44f4c6233fd5b834495 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 15:52:50 -0700 Subject: [PATCH 003/102] docs: document runtime resource snapshots Issue #267 requires the fixed measurement scope to be documented and observation to be kept distinct from resource enforcement. Describe the fields, the adapter host plus descendants scope, the typed errors, the CPU caveats for per-invocation deltas, and that limits belong to the operating system or execution environment, in the SDK guide and the integration skill reference. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 56 +++++++++++++++++++ .../references/results-and-errors.md | 15 +++++ 2 files changed, 71 insertions(+) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 227e0bd44..0a383bcad 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -556,6 +556,7 @@ The following table summarizes the SDK entry points and runtime methods: | `Runtime.invoke(...)` | Yes | You need one turn on an existing runtime. | A runtime permits one active invocation at a time. | | `Runtime.invoke_openai_stream(...)` | No | You need adapter-native OpenAI Chat Completions chunks for one turn. | Requires descriptor `capabilities.streaming`. Returns an async `OpenAIInvokeStream`; await `stream.result()` for the separate terminal `RunResult`. | | `Runtime.invoke_stream(...)` | No | You need live ATOF records generated by NeMo Relay for one turn. | Returns an async `InvokeStream`; await `stream.result()` for the terminal `RunResult`. | +| `Runtime.resource_snapshot()` | Yes | You need the memory, process count, or CPU time of a started runtime. | Does not wait for an active invocation. Refer to [Runtime Resource Snapshots](#runtime-resource-snapshots). | | `Runtime.stop()` | Yes | You need to stop or detach from the runtime. | Called automatically when using `async with`. | | `Service.release()` | Yes | You need to stop a prepared service or detach from an attached service. | Fails while any connected runtime remains active. Called automatically when using `async with`. | @@ -678,6 +679,57 @@ on one live thread, Deep Agents invokes one compiled graph and checkpointer, Hermes Agent reuses one agent and session database, and Claude keeps one connected SDK client. Harness-native identifiers remain adapter-internal. +## Runtime Resource Snapshots + +`Runtime.resource_snapshot()` returns a timestamped `RuntimeResourceSnapshot` +for a started local runtime. It does not wait for an active invocation, so a +monitor can call it while `invoke()` runs: + +```python +snapshot = await runtime.resource_snapshot() +print(snapshot.observed_at, snapshot.resident_memory_bytes) +print(snapshot.process_count, snapshot.cpu_time_seconds) +``` + +| Field | Meaning | +| --- | --- | +| `runtime_id` | Runtime the snapshot describes. | +| `started_at` | When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. | +| `observed_at` | When NeMo Fabric read the values, as an RFC 3339 UTC timestamp. | +| `resident_memory_bytes` | Sum of the resident set sizes of the live processes. Pages shared between processes count once per process. | +| `process_count` | Live processes, including the adapter host. Exited processes that are not yet reaped are not counted. | +| `cpu_time_seconds` | CPU time used since the adapter host started, including descendants that have exited and been waited for. It can drop when a descendant leaves the tree. | + +The scope is fixed: the local adapter host process that NeMo Fabric started for +the runtime, plus its descendants. For a runtime started on a shared service, +the service process is not included. A descendant that starts a new session or +process group still counts while its parent chain leads to the host. A process +whose parent exited is reparented outside the tree and no longer counts. The +snapshot reads `/proc` on Linux and `proc_pid_rusage` on macOS. It uses only +process IDs, parent IDs, start times, CPU times, and resident sizes, never reads +command lines or environment variables, and reports only the runtime's tree. +Descendants that NeMo Fabric is not permitted to inspect are left out. + +When NeMo Fabric cannot read the values, it raises an error instead of +returning zeros: + +- `ResourceObservationUnavailableError` when the runtime stopped, is stopping, + its adapter host process exited, or the process table cannot be read. +- `ResourceObservationUnsupportedError` when the runtime's environment provider + or operating system has no resource reader. Windows raises this error. + +Each `RunResult` from a local runtime also carries `resource_snapshot`, read +once after the adapter returned the result. It is `None` when the read failed; +a failed read never fails the invocation. To estimate one invocation's CPU time, +take a snapshot before it and subtract its `cpu_time_seconds` from the +result's snapshot. CPU time can drop when a descendant is reparented out of the +tree, and on macOS while an exited descendant is not yet waited for, so treat a +difference as an estimate and a negative difference as unknown. + +Snapshots observe resources only. They do not enforce limits, stop runtimes, or +queue requests. Use the operating system or execution environment, such as +cgroups or container limits, to enforce resource limits. + ## Native OpenAI Streaming Use `Runtime.invoke_openai_stream()` when the selected adapter provides native @@ -908,6 +960,7 @@ Important fields: | `output` | Harness output normalized to the configured output schema. | | `error` | Structured failure metadata when available. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | +| `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. | | `artifacts` | Output files, logs, patches, native artifacts, and other materialized references. | | `telemetry` | References to NeMo Relay or other telemetry streams produced by the run. | | `events` | Ordered normalized lifecycle and invocation events. | @@ -1062,6 +1115,8 @@ All public SDK errors inherit from `FabricError`. | `FabricRuntimeError` | Runtime startup, invocation, or shutdown failed before a normalized result could be returned. | | `FabricStateError` | Invalid runtime state transition, such as invoking after stop or starting overlapping invocations. | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | +| `ResourceObservationUnavailableError` | `Runtime.resource_snapshot()` cannot read the runtime because it stopped, is stopping, its adapter host exited, or the process table cannot be read. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnsupportedError` | The runtime's environment provider or operating system has no resource reader. Subclass of `FabricRuntimeError`. | Consumers own job-level retries and rollout-level failure policy. Core host-operation deadlines (`host_timeout`) become Python `TimeoutError`, then @@ -1098,6 +1153,7 @@ generated API reference for exact Python signatures. | Request, result, and events | `schemas/sdk/run-request.schema.json`, `schemas/sdk/run-result.schema.json`, `schemas/sdk/fabric-event.schema.json` | | Runtime and invocation handles | `schemas/sdk/runtime-handle.schema.json`, `schemas/sdk/invocation-handle.schema.json` | | Artifacts and errors | `schemas/sdk/artifact-manifest.schema.json`, `schemas/sdk/error-info.schema.json` | +| Runtime resource snapshot | `schemas/sdk/runtime-resource-snapshot.schema.json` | ### Versioning diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index f2660d5e3..6ef4570d8 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -17,6 +17,7 @@ Every invocation that reaches the adapter boundary returns a normalized | `error` | Structured failure metadata when available. Branch on `status`, not on this field. | | `output` | Harness output normalized to the configured output schema. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | +| `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed. | | `artifacts` | Output files, logs, patches, and other materialized references. | | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | @@ -46,6 +47,18 @@ if result.usage is not None: Treat missing counters and cost as unknown, not zero. `cached_input_tokens` reports cached prompt tokens; `input_tokens_include_cache` declares whether `input_tokens` includes them (`True`), excludes them (`False`), or has unknown semantics (absent). Do not add cache to an inclusive input count. Adapters can put cache-write, reasoning, peak request, and per-model detail in the descriptor-declared `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` object. NeMo Fabric preserves that object at `result.usage["metadata"]["nemo_fabric.detailed_token_usage.v1"]`. Treat detailed input and output independently: any detailed accounting field for a dimension makes that dimension authoritative, with no fallback to its top-level counters. The views can diverge. Calculate an inclusive total only when the selected detailed counters provide enough information; otherwise leave the total unknown. Keep estimated costs separate in provider metadata; `cost_usd` is reported cost, not an estimate. +## Runtime Resource Snapshots + +`await runtime.resource_snapshot()` returns a `RuntimeResourceSnapshot` with +`runtime_id`, `started_at` and `observed_at` (RFC 3339 UTC), `resident_memory_bytes`, +`process_count`, and `cpu_time_seconds`. It covers the local adapter host process +and its descendants (not a shared service process), and it does not wait for +an active invocation. CPU time is cumulative since the host started, so +subtract an earlier snapshot to estimate one invocation; it can drop when a +descendant leaves the tree, so treat a negative difference as unknown. Values that cannot be read raise an error, never read as zero. +Snapshots observe only; enforce limits in the operating system or execution +environment. + ## Correlation IDs `runtime_id` identifies the runtime lifecycle, `invocation_id` identifies one @@ -69,6 +82,8 @@ Refer to the [errors reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/ | `FabricRuntimeError` | Startup, invocation, or shutdown failed before a normalized result. | | `FabricStateError` | Invalid runtime state transition (invoking after stop, overlapping invocations). | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | +| `ResourceObservationUnavailableError` | A resource snapshot cannot be read because the runtime stopped, is stopping, its adapter host exited, or the process table cannot be read. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnsupportedError` | The runtime's environment provider or operating system has no resource reader. Subclass of `FabricRuntimeError`. | ## Cleanup And Resilience From d5a09f9beb737b83ee12818a3a52e1456c90f2bd Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 16:02:56 -0700 Subject: [PATCH 004/102] docs: state which exited descendants count toward snapshot CPU time Linux counts an exited descendant's CPU before its parent waits for it, while macOS counts it only after the wait. The field docs said only waited-for descendants count, which understated Linux. Say both, and re-wrap the integration skill paragraph. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 6 +++--- .../python-library-reference/nemo_fabric.types.md | 2 +- .../struct-runtimeresourcesnapshot.mdx | 2 +- docs/sdk/python.mdx | 2 +- schemas/sdk/run-result.schema.json | 2 +- schemas/sdk/runtime-resource-snapshot.schema.json | 2 +- .../nemo-fabric-runtime/src/nemo_fabric/types.py | 4 ++-- .../references/results-and-errors.md | 13 +++++++------ 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 8401f096e..b76aa535d 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -49,9 +49,9 @@ pub struct RuntimeResourceSnapshot { /// that are not yet reaped are not counted. pub process_count: u64, /// CPU time consumed by the tree since the adapter host started, in seconds, - /// including descendants that have exited and been waited for. It can drop - /// when a descendant leaves the tree, and on macOS while an exited - /// descendant is not yet waited for. + /// including exited descendants (on macOS, only once they are waited for). + /// It can drop when a descendant leaves the tree, and on macOS while an + /// exited descendant is not yet waited for. #[schemars(range(min = 0.0))] pub cpu_time_seconds: f64, } diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 528624f3e..623c34593 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1259,7 +1259,7 @@ Memory, process count, and CPU time cover the adapter host process and its live - `observed_at`: When Fabric read these values, as RFC 3339 UTC. - `resident_memory_bytes`: Sum of the live processes' resident set sizes. Pages shared between processes count once per process. - `process_count`: Live processes in the tree, including the adapter host. - - `cpu_time_seconds`: CPU time used since the adapter host started, including descendants that have exited and been waited for. It can drop when a descendant leaves the tree. + - `cpu_time_seconds`: CPU time used since the adapter host started, including exited descendants (on macOS, only once they are waited for). It can drop when a descendant leaves the tree. ### Fields diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx index 82f8493e1..ba0517771 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx @@ -40,7 +40,7 @@ Live processes in the tree, including the adapter host. Exited processes that ar ### `cpu_time_seconds: f64` -CPU time consumed by the tree since the adapter host started, in seconds, including descendants that have exited and been waited for. It can drop when a descendant leaves the tree, and on macOS while an exited descendant is not yet waited for. +CPU time consumed by the tree since the adapter host started, in seconds, including exited descendants (on macOS, only once they are waited for). It can drop when a descendant leaves the tree, and on macOS while an exited descendant is not yet waited for. ## Trait Implementations diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 0a383bcad..6fbc919b8 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -698,7 +698,7 @@ print(snapshot.process_count, snapshot.cpu_time_seconds) | `observed_at` | When NeMo Fabric read the values, as an RFC 3339 UTC timestamp. | | `resident_memory_bytes` | Sum of the resident set sizes of the live processes. Pages shared between processes count once per process. | | `process_count` | Live processes, including the adapter host. Exited processes that are not yet reaped are not counted. | -| `cpu_time_seconds` | CPU time used since the adapter host started, including descendants that have exited and been waited for. It can drop when a descendant leaves the tree. | +| `cpu_time_seconds` | CPU time used since the adapter host started, including exited descendants (on macOS, only once they are waited for). It can drop when a descendant leaves the tree. | The scope is fixed: the local adapter host process that NeMo Fabric started for the runtime, plus its descendants. For a runtime started on a shared service, diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index e27c1d1cc..072be7097 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -283,7 +283,7 @@ "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", "properties": { "cpu_time_seconds": { - "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding descendants that have exited and been waited for. It can drop\nwhen a descendant leaves the tree, and on macOS while an exited\ndescendant is not yet waited for.", + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants (on macOS, only once they are waited for).\nIt can drop when a descendant leaves the tree, and on macOS while an\nexited descendant is not yet waited for.", "format": "double", "minimum": 0.0, "type": "number" diff --git a/schemas/sdk/runtime-resource-snapshot.schema.json b/schemas/sdk/runtime-resource-snapshot.schema.json index 86a36a5af..8a5bf94ad 100644 --- a/schemas/sdk/runtime-resource-snapshot.schema.json +++ b/schemas/sdk/runtime-resource-snapshot.schema.json @@ -3,7 +3,7 @@ "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", "properties": { "cpu_time_seconds": { - "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding descendants that have exited and been waited for. It can drop\nwhen a descendant leaves the tree, and on macOS while an exited\ndescendant is not yet waited for.", + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants (on macOS, only once they are waited for).\nIt can drop when a descendant leaves the tree, and on macOS while an\nexited descendant is not yet waited for.", "format": "double", "minimum": 0.0, "type": "number" diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 07e22c7ac..c1ddf0323 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1849,8 +1849,8 @@ class RuntimeResourceSnapshot(FabricMapping): Pages shared between processes count once per process. process_count: Live processes in the tree, including the adapter host. cpu_time_seconds: CPU time used since the adapter host started, - including descendants that have exited and been waited for. It - can drop when a descendant leaves the tree. + including exited descendants (on macOS, only once they are + waited for). It can drop when a descendant leaves the tree. """ runtime_id: str diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 6ef4570d8..094b75614 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -50,12 +50,13 @@ Treat missing counters and cost as unknown, not zero. `cached_input_tokens` repo ## Runtime Resource Snapshots `await runtime.resource_snapshot()` returns a `RuntimeResourceSnapshot` with -`runtime_id`, `started_at` and `observed_at` (RFC 3339 UTC), `resident_memory_bytes`, -`process_count`, and `cpu_time_seconds`. It covers the local adapter host process -and its descendants (not a shared service process), and it does not wait for -an active invocation. CPU time is cumulative since the host started, so -subtract an earlier snapshot to estimate one invocation; it can drop when a -descendant leaves the tree, so treat a negative difference as unknown. Values that cannot be read raise an error, never read as zero. +`runtime_id`, `started_at` and `observed_at` (RFC 3339 UTC), +`resident_memory_bytes`, `process_count`, and `cpu_time_seconds`. It covers the +local adapter host process and its descendants (not a shared service process), +and it does not wait for an active invocation. CPU time is cumulative since the +host started, so subtract an earlier snapshot to estimate one invocation; it +can drop when a descendant leaves the tree, so treat a negative difference as +unknown. Values that cannot be read raise an error, never read as zero. Snapshots observe only; enforce limits in the operating system or execution environment. From 85f1e5a851e2d94439faa71f7032456ae9a2dfc5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 22:47:18 -0700 Subject: [PATCH 005/102] refactor: queue macOS snapshot children with their usage Every entry the macOS reader queues already carries the rusage it was filtered on, so the optional value and the fallback re-read were dead paths. Queue the usage directly. Behavior is unchanged. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index b76aa535d..a1049aab1 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -414,11 +414,8 @@ mod platform { let mut read = TreeRead::default(); let mut cpu_units: u64 = 0; let mut resident: u64 = 0; - let mut pending = VecDeque::from([(root, Some(root_info))]); - while let Some((pid, known)) = pending.pop_front() { - let Some(info) = known.or_else(|| rusage(pid)) else { - continue; - }; + let mut pending = VecDeque::from([(root, root_info)]); + while let Some((pid, info)) = pending.pop_front() { cpu_units = cpu_units .saturating_add(info.ri_user_time) .saturating_add(info.ri_system_time) @@ -434,8 +431,7 @@ mod platform { // that seems to start before its parent (a reused PID). .filter(|(_, info)| { info.ri_proc_exit_abstime == 0 && info.ri_proc_start_abstime >= start - }) - .map(|(child, info)| (child, Some(info))), + }), ); } read.cpu_nanos = mach_to_nanos(cpu_units, timebase); From c846c7841c9150848356e6845badbedd027d8fae Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 00:36:54 -0700 Subject: [PATCH 006/102] fix: check each macOS snapshot child is still the listed process The macOS reader listed each child's PID and parent, then read the child's usage later by PID alone. A child that exited after the listing could have its PID reused by an unrelated process, whose usage was then counted in the runtime's snapshot. The listing now keeps each child's parent and start time, and the child's usage is counted only when the same parent and start time read back after the usage read. The host is read after the listing, so its listed children are known to be its own. The libproc calls sit behind a small trait so tests can script the reuse. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 340 +++++++++++++++----- 1 file changed, 264 insertions(+), 76 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index a1049aab1..043aa894c 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -305,7 +305,9 @@ mod platform { #[cfg(target_os = "macos")] mod platform { - //! `proc_pid_rusage` reads, from the root down, each parent before its children. + //! `libproc` reads. One pass lists every process's parent and start time, + //! then the tree is read from the root down, each parent before its + //! children. use std::collections::{BTreeMap, VecDeque}; use std::sync::OnceLock; @@ -314,19 +316,118 @@ mod platform { pub(super) const SUPPORTED: bool = true; - fn rusage(pid: u32) -> Option { - let pid = libc::c_int::try_from(pid).ok()?; - // SAFETY: rusage_info_v2 is plain data; zero is a valid initial value. - let mut info: libc::rusage_info_v2 = unsafe { std::mem::zeroed() }; - // SAFETY: `info` is a writable rusage_info_v2, the layout RUSAGE_INFO_V2 names. - let status = unsafe { - libc::proc_pid_rusage( - pid, - libc::RUSAGE_INFO_V2, - (&raw mut info).cast::(), + /// Why one process could not be read. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) enum ReadError { + /// The process exited, or is a zombie the call cannot describe. + Gone, + /// The process exists but this process may not read it. + Denied, + } + + /// A process's parent and start time, which together identify it. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) struct Parent { + pub ppid: u32, + /// `pbi_start_tvsec` and `pbi_start_tvusec`. + pub started: (u64, u64), + } + + /// The `proc_pid_rusage` fields a snapshot needs, CPU times in Mach units. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) struct Usage { + pub cpu_units: u64, + pub resident_bytes: u64, + /// `ri_proc_start_abstime`. + pub start: u64, + pub exited: bool, + } + + /// The process reads a tree read makes. Tests script them. + pub(super) trait Libproc { + /// Every PID, or `None` when the process table cannot be listed. + fn pids(&self) -> Option>; + fn parent(&self, pid: u32) -> Result; + fn usage(&self, pid: u32) -> Result; + } + + struct Live; + + impl Libproc for Live { + fn pids(&self) -> Option> { + let mut pids = vec![0 as libc::pid_t; 1024]; + let count = loop { + let bytes = libc::c_int::try_from(std::mem::size_of_val(pids.as_slice())).ok()?; + // SAFETY: the buffer is writable and sized in bytes as libproc requires. + let count = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) }; + let count = usize::try_from(count).ok().filter(|count| *count > 0)?; + if count < pids.len() { + break count; + } + pids.resize(pids.len() * 2, 0); + }; + Some( + pids[..count] + .iter() + .filter_map(|pid| u32::try_from(*pid).ok()) + .collect(), ) - }; - (status == 0).then_some(info) + } + + /// `PROC_PIDTBSDINFO` fails for a zombie, so a zombie reads as gone. + fn parent(&self, pid: u32) -> Result { + let pid = libc::pid_t::try_from(pid).map_err(|_| ReadError::Gone)?; + // SAFETY: proc_bsdinfo is plain data; zero is a valid initial value. + let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; + let size = libc::c_int::try_from(std::mem::size_of::()) + .map_err(|_| ReadError::Denied)?; + // SAFETY: `info` is writable and `size` bytes long, as PROC_PIDTBSDINFO needs. + let written = unsafe { + libc::proc_pidinfo(pid, libc::PROC_PIDTBSDINFO, 0, (&raw mut info).cast(), size) + }; + if written == size { + return Ok(Parent { + ppid: info.pbi_ppid, + started: (info.pbi_start_tvsec, info.pbi_start_tvusec), + }); + } + Err(last_read_error()) + } + + fn usage(&self, pid: u32) -> Result { + let pid = libc::c_int::try_from(pid).map_err(|_| ReadError::Gone)?; + // SAFETY: rusage_info_v2 is plain data; zero is a valid initial value. + let mut info: libc::rusage_info_v2 = unsafe { std::mem::zeroed() }; + // SAFETY: `info` is a writable rusage_info_v2, the layout RUSAGE_INFO_V2 names. + let status = unsafe { + libc::proc_pid_rusage( + pid, + libc::RUSAGE_INFO_V2, + (&raw mut info).cast::(), + ) + }; + if status != 0 { + return Err(last_read_error()); + } + Ok(Usage { + cpu_units: info + .ri_user_time + .saturating_add(info.ri_system_time) + .saturating_add(info.ri_child_user_time) + .saturating_add(info.ri_child_system_time), + resident_bytes: info.ri_resident_size, + start: info.ri_proc_start_abstime, + exited: info.ri_proc_exit_abstime != 0, + }) + } + } + + /// Classify the error of the libproc call that just failed. + fn last_read_error() -> ReadError { + match std::io::Error::last_os_error().raw_os_error() { + Some(libc::ESRCH) => ReadError::Gone, + _ => ReadError::Denied, + } } /// `mach_timebase_info_data_t`. @@ -362,82 +463,67 @@ mod platform { } pub(super) fn start_identity(pid: u32) -> Option { - rusage(pid).map(|info| info.ri_proc_start_abstime) - } - - /// Every live process's children. One `proc_listallpids` call and one - /// `proc_pidinfo` per process: `proc_listchildpids` scans the whole - /// process table on each call, which costs too much for a wide tree. - /// `PROC_PIDTBSDINFO` fails for a zombie, so a zombie is never listed. - /// Returns `None` when the process table cannot be listed. - fn children_by_parent() -> Option>> { - let mut pids = vec![0 as libc::pid_t; 1024]; - let count = loop { - let bytes = libc::c_int::try_from(std::mem::size_of_val(pids.as_slice())).ok()?; - // SAFETY: the buffer is writable and sized in bytes as libproc requires. - let count = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) }; - let count = usize::try_from(count).ok().filter(|count| *count > 0)?; - if count < pids.len() { - break count; - } - pids.resize(pids.len() * 2, 0); - }; - let mut children: BTreeMap> = BTreeMap::new(); - for &pid in &pids[..count] { - if let (Ok(pid), Some(ppid)) = (u32::try_from(pid), parent_pid(pid)) { - children.entry(ppid).or_default().push(pid); + Live.usage(pid).ok().map(|usage| usage.start) + } + + pub(super) fn read_tree(root: u32, identity: u64) -> Option { + read_tree_with(&Live, root, identity, timebase()?) + } + + /// Every live process's children, each with its parent record. One + /// `proc_listallpids` call and one `proc_pidinfo` per process: + /// `proc_listchildpids` scans the whole process table on each call, which + /// costs too much for a wide tree. + fn children_by_parent(procs: &impl Libproc) -> Option>> { + let mut children: BTreeMap> = BTreeMap::new(); + for pid in procs.pids()? { + if let Ok(parent) = procs.parent(pid) { + children.entry(parent.ppid).or_default().push((pid, parent)); } } Some(children) } - /// The parent of a live process, or `None` for a zombie or a missing process. - fn parent_pid(pid: libc::pid_t) -> Option { - // SAFETY: proc_bsdinfo is plain data; zero is a valid initial value. - let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; - let size = libc::c_int::try_from(std::mem::size_of::()).ok()?; - // SAFETY: `info` is writable and `size` bytes long, as PROC_PIDTBSDINFO needs. - let written = unsafe { - libc::proc_pidinfo(pid, libc::PROC_PIDTBSDINFO, 0, (&raw mut info).cast(), size) - }; - (written == size).then_some(info.pbi_ppid) - } - /// Read the tree under `root`, or `None` when the root has exited or its - /// PID now names a process with another start time than `identity`. A - /// child that seems to start before its parent reuses a PID and is left out. - pub(super) fn read_tree(root: u32, identity: u64) -> Option { - parent_pid(libc::pid_t::try_from(root).ok()?)?; - let root_info = rusage(root).filter(|info| info.ri_proc_start_abstime == identity)?; - let timebase = timebase()?; - let children = children_by_parent()?; + /// PID now names a process with another start time than `identity`. + pub(super) fn read_tree_with( + procs: &impl Libproc, + root: u32, + identity: u64, + timebase: (u64, u64), + ) -> Option { + let children = children_by_parent(procs)?; + // Read after the listing: a root that is still the same live process + // was alive during the whole listing, so its listed children are its own. + let root_usage = procs + .usage(root) + .ok() + .filter(|usage| usage.start == identity && !usage.exited)?; let mut read = TreeRead::default(); let mut cpu_units: u64 = 0; - let mut resident: u64 = 0; - let mut pending = VecDeque::from([(root, root_info)]); - while let Some((pid, info)) = pending.pop_front() { - cpu_units = cpu_units - .saturating_add(info.ri_user_time) - .saturating_add(info.ri_system_time) - .saturating_add(info.ri_child_user_time) - .saturating_add(info.ri_child_system_time); - resident = resident.saturating_add(info.ri_resident_size); + let mut pending = VecDeque::from([(root, root_usage)]); + while let Some((pid, usage)) = pending.pop_front() { + cpu_units = cpu_units.saturating_add(usage.cpu_units); + read.resident_bytes = read.resident_bytes.saturating_add(usage.resident_bytes); read.process_count += 1; - let start = info.ri_proc_start_abstime; - let own = children.get(&pid).into_iter().flatten(); - pending.extend( - own.filter_map(|child| rusage(*child).map(|info| (*child, info))) - // Skip a child that exited after it was listed, and one - // that seems to start before its parent (a reused PID). - .filter(|(_, info)| { - info.ri_proc_exit_abstime == 0 && info.ri_proc_start_abstime >= start - }), - ); + for &(child, listed) in children.get(&pid).into_iter().flatten() { + if let Some(usage) = listed_child_usage(procs, child, listed) { + pending.push_back((child, usage)); + } + } } read.cpu_nanos = mach_to_nanos(cpu_units, timebase); - read.resident_bytes = resident; Some(read) } + + /// The usage of `child`, or `None` when it exited or its PID no longer + /// names the listed process. A process that still has the listed parent + /// and start time after the usage read was the listed one throughout, so + /// the usage is its own. + fn listed_child_usage(procs: &impl Libproc, child: u32, listed: Parent) -> Option { + let usage = procs.usage(child).ok().filter(|usage| !usage.exited)?; + (procs.parent(child).ok()? == listed).then_some(usage) + } } #[cfg(not(any(target_os = "linux", target_os = "macos")))] @@ -558,6 +644,108 @@ mod tests { assert_eq!(platform::ticks_to_nanos(150, 100), 1_500_000_000); } + /// Scripted `libproc` answers for the macOS tree walk. + #[cfg(target_os = "macos")] + #[derive(Default)] + struct ScriptedLibproc { + pids: Option>, + /// Answers to `parent`, first the listing's, then the later checks'. + parents: std::collections::BTreeMap< + u32, + std::cell::RefCell>>, + >, + usages: std::collections::BTreeMap< + u32, + std::result::Result, + >, + } + + #[cfg(target_os = "macos")] + impl ScriptedLibproc { + fn process( + mut self, + pid: u32, + parents: Vec>, + usage: std::result::Result, + ) -> Self { + self.pids.get_or_insert_with(Vec::new).push(pid); + self.parents.insert(pid, std::cell::RefCell::new(parents)); + self.usages.insert(pid, usage); + self + } + } + + #[cfg(target_os = "macos")] + impl platform::Libproc for ScriptedLibproc { + fn pids(&self) -> Option> { + self.pids.clone() + } + + fn parent(&self, pid: u32) -> std::result::Result { + let mut answers = self.parents.get(&pid).expect("scripted pid").borrow_mut(); + if answers.len() > 1 { + answers.remove(0) + } else { + answers[0] + } + } + + fn usage(&self, pid: u32) -> std::result::Result { + self.usages[&pid] + } + } + + #[cfg(target_os = "macos")] + fn mac_parent(ppid: u32, started: u64) -> platform::Parent { + platform::Parent { + ppid, + started: (started, 0), + } + } + + #[cfg(target_os = "macos")] + fn mac_usage(start: u64, cpu_units: u64) -> platform::Usage { + platform::Usage { + cpu_units, + resident_bytes: 1000, + start, + exited: false, + } + } + + /// The host (PID 10) has one listed child (PID 11). After the listing the + /// child exits and an unrelated, newer process takes PID 11. + #[cfg(target_os = "macos")] + #[test] + fn macos_tree_read_skips_child_pid_reused_after_listing() { + let procs = ScriptedLibproc::default() + .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) + .process( + 11, + vec![Ok(mac_parent(10, 200)), Ok(mac_parent(1, 300))], + Ok(mac_usage(9, 1_000_000)), + ); + + let read = platform::read_tree_with(&procs, 10, 5, (1, 1)).expect("tree read"); + + assert_eq!(read.process_count, 1); + assert_eq!(read.cpu_nanos, 7); + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_tree_read_counts_child_that_is_still_the_listed_process() { + let procs = ScriptedLibproc::default() + .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) + .process(11, vec![Ok(mac_parent(10, 200))], Ok(mac_usage(9, 3))); + + let read = platform::read_tree_with(&procs, 10, 5, (1, 1)).expect("tree read"); + + assert_eq!(read.process_count, 2); + assert_eq!(read.cpu_nanos, 10); + assert_eq!(read.resident_bytes, 2000); + } + #[cfg(target_os = "macos")] #[test] fn macos_mach_units_convert_with_the_timebase() { From ba3404ae69804ee5c89b3891d8dbf42d4309bd80 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 00:41:35 -0700 Subject: [PATCH 007/102] fix: report a snapshot as unavailable when a runtime process cannot be read Both readers turned every failed read into "skip this process", so a child that had exited and a live child that could not be inspected gave the same successful, smaller snapshot. #267 asks for an inaccessible runtime to return ResourceObservationUnavailable and for unavailable measurements never to read as zero. Each read now reports whether the process is gone or unreadable. A gone or replaced child is still skipped. A live process in the tree that cannot be read, or a process table that cannot be listed in full, now returns ResourceObservationUnavailable with the reason. On macOS the listing falls back to the short BSD record for another user's process, so its parent is still known, and such a process inside the tree makes the snapshot unavailable. The Linux reads move behind the same kind of trait as macOS so tests can script denied, reaped and replaced processes. A new test covers a snapshot after an invocation timeout. The SDK guide, error docstrings and skill reference now name the new condition, and say that discovery reads other processes' parent IDs and start times while the snapshot exposes only its own fields. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/error.rs | 4 +- crates/fabric-core/src/resource_snapshot.rs | 536 +++++++++++++++--- crates/fabric-core/src/runtime.rs | 26 +- crates/fabric-python/src/lib.rs | 2 +- docs/sdk/python.mdx | 16 +- .../src/nemo_fabric/errors.py | 4 +- .../src/nemo_fabric/runtime.py | 4 +- .../references/results-and-errors.md | 2 +- 8 files changed, 505 insertions(+), 89 deletions(-) diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index ce4682c28..fb4075af0 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -278,8 +278,8 @@ pub enum FabricError { adapter_kind: AdapterKind, }, /// A runtime's resources cannot be read now: the runtime stopped, is - /// stopping, its adapter host process exited, or the process table cannot - /// be read. + /// stopping, its adapter host process exited, or a live process in its + /// tree or the process table cannot be read. #[error("resource observation is unavailable for runtime `{runtime_id}`: {reason}")] ResourceObservationUnavailable { /// Runtime handle id. diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 043aa894c..579381e15 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -13,8 +13,14 @@ //! and on macOS while an exited descendant is not yet waited for, because //! macOS cannot list an unreaped process. //! -//! Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. Neither -//! reads command lines, environment variables, or open files. +//! A live process in the tree that cannot be read, or a process table that +//! cannot be listed in full, makes the snapshot unavailable instead of +//! leaving the process out. +//! +//! Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. To find +//! the tree, the readers also read other processes' parent IDs and start +//! times. A snapshot exposes only its own fields: never command lines, +//! environment variables, open files, or processes outside the tree. use std::time::{SystemTime, UNIX_EPOCH}; @@ -26,6 +32,18 @@ use crate::error::{FabricError, Result}; const MILLIS_PER_DAY: i64 = 86_400_000; const NANOS_PER_SECOND: f64 = 1_000_000_000.0; +/// Reasons a tree read returns when the snapshot is unavailable. +#[cfg_attr( + not(any(target_os = "linux", target_os = "macos")), + allow(dead_code, reason = "only the Linux and macOS readers read a tree") +)] +mod reason { + pub(super) const HOST_EXITED: &str = "adapter host process has exited"; + pub(super) const PROCESS_UNREADABLE: &str = + "a live process in the runtime's process tree cannot be read"; + pub(super) const TABLE_UNREADABLE: &str = "the process table cannot be read in full"; +} + /// Timestamped resource usage of one runtime's local adapter host process tree. /// /// Memory, process count, and CPU time cover the adapter host process and its @@ -95,15 +113,14 @@ pub(crate) fn snapshot_local_host( } let host = host.ok_or_else(|| unavailable(runtime_id, "runtime is not active in this process"))?; - let read = host - .identity - .and_then(|identity| platform::read_tree(host.pid, identity)) - .ok_or_else(|| { - unavailable( - runtime_id, - "adapter host process has exited or the process table cannot be read", - ) - })?; + let identity = host.identity.ok_or_else(|| { + unavailable( + runtime_id, + "adapter host process could not be read when the runtime started", + ) + })?; + let read = platform::read_tree(host.pid, identity) + .map_err(|reason| unavailable(runtime_id, reason))?; Ok(RuntimeResourceSnapshot { runtime_id: runtime_id.to_string(), started_at: host.started_at.clone(), @@ -187,10 +204,19 @@ mod platform { use std::collections::{BTreeMap, VecDeque}; use std::sync::OnceLock; - use super::TreeRead; + use super::{TreeRead, reason}; pub(super) const SUPPORTED: bool = true; + /// Why one process could not be read. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) enum ReadError { + /// The process exited. + Gone, + /// The process exists but cannot be read, or its record cannot be parsed. + Unreadable, + } + /// The fields of `/proc//stat` a snapshot needs. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct Stat { @@ -222,12 +248,46 @@ mod platform { }) } - fn read_stat(pid: u32) -> Option { - parse_stat(&std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?) + /// The `/proc` reads a tree read makes. Tests script them. + pub(super) trait Procfs { + /// Every PID in `/proc`. + fn pids(&self) -> Result, ReadError>; + fn stat(&self, pid: u32) -> Result; + } + + struct Live; + + impl Procfs for Live { + fn pids(&self) -> Result, ReadError> { + let mut pids = Vec::new(); + for entry in std::fs::read_dir("/proc").map_err(read_error)? { + let name = entry.map_err(read_error)?.file_name(); + if let Some(pid) = name.to_str().and_then(|name| name.parse().ok()) { + pids.push(pid); + } + } + Ok(pids) + } + + fn stat(&self, pid: u32) -> Result { + let text = std::fs::read_to_string(format!("/proc/{pid}/stat")).map_err(read_error)?; + parse_stat(&text).ok_or(ReadError::Unreadable) + } + } + + /// `ENOENT` and `ESRCH` mean the process exited; anything else means it + /// cannot be read. + fn read_error(error: std::io::Error) -> ReadError { + if error.kind() == std::io::ErrorKind::NotFound || error.raw_os_error() == Some(libc::ESRCH) + { + ReadError::Gone + } else { + ReadError::Unreadable + } } pub(super) fn start_identity(pid: u32) -> Option { - read_stat(pid).map(|stat| stat.start) + Live.stat(pid).ok().map(|stat| stat.start) } fn sysconf(name: libc::c_int) -> u64 { @@ -245,43 +305,54 @@ mod platform { *PAGE.get_or_init(|| sysconf(libc::_SC_PAGESIZE)) } - /// Every process's children, each with its start time, or `None` when - /// `/proc` cannot be listed. - fn children_by_parent() -> Option>> { - let mut children: BTreeMap> = BTreeMap::new(); - for entry in std::fs::read_dir("/proc").ok()?.flatten() { - let Some(pid) = entry - .file_name() - .to_str() - .and_then(|name| name.parse().ok()) - else { - continue; - }; - if let Some(stat) = read_stat(pid) { - children + pub(super) fn read_tree(root: u32, identity: u64) -> Result { + read_tree_with(&Live, root, identity, clock_ticks_per_second(), page_size()) + } + + /// Child PIDs and start times by parent PID. + type ChildrenByParent = BTreeMap>; + + /// Every process's children, each with its start time. A process that + /// exited during the pass is left out; one that cannot be read leaves the + /// listing incomplete. + fn children_by_parent(procfs: &impl Procfs) -> Result { + let pids = procfs.pids().map_err(|_| reason::TABLE_UNREADABLE)?; + let mut children = ChildrenByParent::new(); + for pid in pids { + match procfs.stat(pid) { + Ok(stat) => children .entry(stat.ppid) .or_default() - .push((pid, stat.start)); + .push((pid, stat.start)), + Err(ReadError::Gone) => {} + Err(ReadError::Unreadable) => return Err(reason::TABLE_UNREADABLE), } } - Some(children) + Ok(children) } - /// Read the tree under `root`, or `None` when the root has exited or its - /// PID now names a process with another start time than `identity`. - pub(super) fn read_tree(root: u32, identity: u64) -> Option { - let children = children_by_parent()?; + /// Read the tree under `root`. Fails when the root exited or its PID now + /// names a process with another start time than `identity`, or when a + /// live process in the tree cannot be read. + pub(super) fn read_tree_with( + procfs: &impl Procfs, + root: u32, + identity: u64, + ticks_per_second: u64, + page_size: u64, + ) -> Result { + let children = children_by_parent(procfs)?; let mut read = TreeRead::default(); let mut cpu_ticks: u64 = 0; let mut rss_pages: u64 = 0; let mut pending = VecDeque::from([(root, identity)]); while let Some((pid, expected_start)) = pending.pop_front() { - let stat = read_stat(pid).filter(|stat| stat.start == expected_start); - if pid == root && stat.is_none_or(|stat| stat.zombie) { - return None; - } - let Some(stat) = stat else { - continue; + let stat = match procfs.stat(pid) { + Ok(stat) if stat.start == expected_start && !(pid == root && stat.zombie) => stat, + // An exited child, or one whose PID now names another process. + Ok(_) | Err(ReadError::Gone) if pid != root => continue, + Ok(_) | Err(ReadError::Gone) => return Err(reason::HOST_EXITED), + Err(ReadError::Unreadable) => return Err(reason::PROCESS_UNREADABLE), }; cpu_ticks = cpu_ticks.saturating_add(stat.cpu_ticks); if !stat.zombie { @@ -292,9 +363,9 @@ mod platform { let own = children.get(&pid).into_iter().flatten(); pending.extend(own.filter(|(_, start)| *start >= stat.start).copied()); } - read.cpu_nanos = ticks_to_nanos(cpu_ticks, clock_ticks_per_second()); - read.resident_bytes = rss_pages.saturating_mul(page_size()); - Some(read) + read.cpu_nanos = ticks_to_nanos(cpu_ticks, ticks_per_second); + read.resident_bytes = rss_pages.saturating_mul(page_size); + Ok(read) } pub(super) fn ticks_to_nanos(ticks: u64, ticks_per_second: u64) -> u64 { @@ -312,7 +383,7 @@ mod platform { use std::collections::{BTreeMap, VecDeque}; use std::sync::OnceLock; - use super::TreeRead; + use super::{TreeRead, reason}; pub(super) const SUPPORTED: bool = true; @@ -329,10 +400,23 @@ mod platform { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct Parent { pub ppid: u32, - /// `pbi_start_tvsec` and `pbi_start_tvusec`. - pub started: (u64, u64), + /// `pbi_start_tvsec` and `pbi_start_tvusec`, or `None` for another + /// user's process, whose parent only the short record shows. + pub started: Option<(u64, u64)>, + } + + /// `struct proc_bsdshortinfo` from ``, which the `libc` + /// crate version in use does not bind: PID, parent PID, then 56 bytes of + /// group, status, name, flags, and user and group IDs that the reader ignores. + #[repr(C)] + struct ProcBsdShortInfo { + pid: u32, + ppid: u32, + rest: [u32; 14], } + const PROC_PIDT_SHORTBSDINFO: libc::c_int = 13; + /// The `proc_pid_rusage` fields a snapshot needs, CPU times in Mach units. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct Usage { @@ -375,6 +459,8 @@ mod platform { } /// `PROC_PIDTBSDINFO` fails for a zombie, so a zombie reads as gone. + /// It is denied for another user's process, whose parent the short + /// record still shows. fn parent(&self, pid: u32) -> Result { let pid = libc::pid_t::try_from(pid).map_err(|_| ReadError::Gone)?; // SAFETY: proc_bsdinfo is plain data; zero is a valid initial value. @@ -388,10 +474,13 @@ mod platform { if written == size { return Ok(Parent { ppid: info.pbi_ppid, - started: (info.pbi_start_tvsec, info.pbi_start_tvusec), + started: Some((info.pbi_start_tvsec, info.pbi_start_tvusec)), }); } - Err(last_read_error()) + match last_read_error() { + ReadError::Denied => short_parent(pid), + gone => Err(gone), + } } fn usage(&self, pid: u32) -> Result { @@ -422,6 +511,27 @@ mod platform { } } + fn short_parent(pid: libc::pid_t) -> Result { + let mut info = ProcBsdShortInfo { + pid: 0, + ppid: 0, + rest: [0; 14], + }; + let size = libc::c_int::try_from(std::mem::size_of::()) + .map_err(|_| ReadError::Denied)?; + // SAFETY: `info` is writable and `size` bytes long, as PROC_PIDT_SHORTBSDINFO needs. + let written = unsafe { + libc::proc_pidinfo(pid, PROC_PIDT_SHORTBSDINFO, 0, (&raw mut info).cast(), size) + }; + if written == size { + return Ok(Parent { + ppid: info.ppid, + started: None, + }); + } + Err(last_read_error()) + } + /// Classify the error of the libproc call that just failed. fn last_read_error() -> ReadError { match std::io::Error::last_os_error().raw_os_error() { @@ -466,39 +576,48 @@ mod platform { Live.usage(pid).ok().map(|usage| usage.start) } - pub(super) fn read_tree(root: u32, identity: u64) -> Option { - read_tree_with(&Live, root, identity, timebase()?) + pub(super) fn read_tree(root: u32, identity: u64) -> Result { + let timebase = timebase().ok_or(reason::PROCESS_UNREADABLE)?; + read_tree_with(&Live, root, identity, timebase) } /// Every live process's children, each with its parent record. One /// `proc_listallpids` call and one `proc_pidinfo` per process: /// `proc_listchildpids` scans the whole process table on each call, which - /// costs too much for a wide tree. - fn children_by_parent(procs: &impl Libproc) -> Option>> { + /// costs too much for a wide tree. A process that exited during the pass + /// is left out; one that cannot be read leaves the listing incomplete. + fn children_by_parent( + procs: &impl Libproc, + ) -> Result>, &'static str> { + let pids = procs.pids().ok_or(reason::TABLE_UNREADABLE)?; let mut children: BTreeMap> = BTreeMap::new(); - for pid in procs.pids()? { - if let Ok(parent) = procs.parent(pid) { - children.entry(parent.ppid).or_default().push((pid, parent)); + for pid in pids { + match procs.parent(pid) { + Ok(parent) => children.entry(parent.ppid).or_default().push((pid, parent)), + Err(ReadError::Gone) => {} + Err(ReadError::Denied) => return Err(reason::TABLE_UNREADABLE), } } - Some(children) + Ok(children) } - /// Read the tree under `root`, or `None` when the root has exited or its - /// PID now names a process with another start time than `identity`. + /// Read the tree under `root`. Fails when the root exited or its PID now + /// names a process with another start time than `identity`, or when a + /// live process in the tree cannot be read. pub(super) fn read_tree_with( procs: &impl Libproc, root: u32, identity: u64, timebase: (u64, u64), - ) -> Option { + ) -> Result { let children = children_by_parent(procs)?; // Read after the listing: a root that is still the same live process // was alive during the whole listing, so its listed children are its own. - let root_usage = procs - .usage(root) - .ok() - .filter(|usage| usage.start == identity && !usage.exited)?; + let root_usage = match procs.usage(root) { + Ok(usage) if usage.start == identity && !usage.exited => usage, + Ok(_) | Err(ReadError::Gone) => return Err(reason::HOST_EXITED), + Err(ReadError::Denied) => return Err(reason::PROCESS_UNREADABLE), + }; let mut read = TreeRead::default(); let mut cpu_units: u64 = 0; let mut pending = VecDeque::from([(root, root_usage)]); @@ -507,22 +626,40 @@ mod platform { read.resident_bytes = read.resident_bytes.saturating_add(usage.resident_bytes); read.process_count += 1; for &(child, listed) in children.get(&pid).into_iter().flatten() { - if let Some(usage) = listed_child_usage(procs, child, listed) { + if let Some(usage) = listed_child_usage(procs, child, listed)? { pending.push_back((child, usage)); } } } read.cpu_nanos = mach_to_nanos(cpu_units, timebase); - Some(read) + Ok(read) } /// The usage of `child`, or `None` when it exited or its PID no longer /// names the listed process. A process that still has the listed parent /// and start time after the usage read was the listed one throughout, so - /// the usage is its own. - fn listed_child_usage(procs: &impl Libproc, child: u32, listed: Parent) -> Option { - let usage = procs.usage(child).ok().filter(|usage| !usage.exited)?; - (procs.parent(child).ok()? == listed).then_some(usage) + /// the usage is its own. Fails when the child is live but cannot be read. + fn listed_child_usage( + procs: &impl Libproc, + child: u32, + listed: Parent, + ) -> Result, &'static str> { + if listed.started.is_none() { + return Err(reason::PROCESS_UNREADABLE); + } + let usage = match procs.usage(child) { + Ok(usage) if !usage.exited => usage, + Ok(_) | Err(ReadError::Gone) => return Ok(None), + Err(ReadError::Denied) => return Err(reason::PROCESS_UNREADABLE), + }; + match procs.parent(child) { + Ok(now) if now == listed => Ok(Some(usage)), + Ok(now) if now.ppid == listed.ppid && now.started.is_none() => { + Err(reason::PROCESS_UNREADABLE) + } + Ok(_) | Err(ReadError::Gone) => Ok(None), + Err(ReadError::Denied) => Err(reason::PROCESS_UNREADABLE), + } } } @@ -536,8 +673,8 @@ mod platform { None } - pub(super) fn read_tree(_root: u32, _identity: u64) -> Option { - None + pub(super) fn read_tree(_root: u32, _identity: u64) -> Result { + Err("no process reader for this operating system") } } @@ -699,7 +836,7 @@ mod tests { fn mac_parent(ppid: u32, started: u64) -> platform::Parent { platform::Parent { ppid, - started: (started, 0), + started: Some((started, 0)), } } @@ -746,6 +883,259 @@ mod tests { assert_eq!(read.resident_bytes, 2000); } + #[cfg(target_os = "macos")] + fn mac_tree_with_child( + child_parents: Vec>, + child_usage: std::result::Result, + ) -> std::result::Result { + let procs = ScriptedLibproc::default() + .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) + .process(11, child_parents, child_usage); + platform::read_tree_with(&procs, 10, 5, (1, 1)) + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_tree_read_with_unreadable_live_child_is_unavailable() { + let denied_usage = mac_tree_with_child( + vec![Ok(mac_parent(10, 200))], + Err(platform::ReadError::Denied), + ); + let other_user = mac_tree_with_child( + vec![Ok(platform::Parent { + ppid: 10, + started: None, + })], + Ok(mac_usage(9, 3)), + ); + let denied_recheck = mac_tree_with_child( + vec![Ok(mac_parent(10, 200)), Err(platform::ReadError::Denied)], + Ok(mac_usage(9, 3)), + ); + + assert_eq!(denied_usage.unwrap_err(), reason::PROCESS_UNREADABLE); + assert_eq!(other_user.unwrap_err(), reason::PROCESS_UNREADABLE); + assert_eq!(denied_recheck.unwrap_err(), reason::PROCESS_UNREADABLE); + } + + /// The child is listed, then exits before its usage read or before the + /// identity check that follows it. + #[cfg(target_os = "macos")] + #[test] + fn macos_tree_read_skips_child_reaped_between_reads() { + let before_usage = mac_tree_with_child( + vec![Ok(mac_parent(10, 200))], + Err(platform::ReadError::Gone), + ) + .expect("tree read"); + let before_check = mac_tree_with_child( + vec![Ok(mac_parent(10, 200)), Err(platform::ReadError::Gone)], + Ok(mac_usage(9, 3)), + ) + .expect("tree read"); + let exited = mac_tree_with_child( + vec![Ok(mac_parent(10, 200))], + Ok(platform::Usage { + exited: true, + ..mac_usage(9, 3) + }), + ) + .expect("tree read"); + + for read in [before_usage, before_check, exited] { + assert_eq!(read.process_count, 1); + assert_eq!(read.cpu_nanos, 7); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_tree_read_with_incomplete_process_table_is_unavailable() { + let unlisted = ScriptedLibproc::default(); + let unreadable_other = ScriptedLibproc::default() + .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) + .process( + 50, + vec![Err(platform::ReadError::Denied)], + Err(platform::ReadError::Denied), + ); + + assert_eq!( + platform::read_tree_with(&unlisted, 10, 5, (1, 1)).unwrap_err(), + reason::TABLE_UNREADABLE + ); + assert_eq!( + platform::read_tree_with(&unreadable_other, 10, 5, (1, 1)).unwrap_err(), + reason::TABLE_UNREADABLE + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_tree_read_of_exited_or_unreadable_host_is_unavailable() { + let host = |usage| { + let procs = ScriptedLibproc::default().process(10, vec![Ok(mac_parent(1, 100))], usage); + platform::read_tree_with(&procs, 10, 5, (1, 1)).unwrap_err() + }; + + assert_eq!(host(Err(platform::ReadError::Gone)), reason::HOST_EXITED); + assert_eq!(host(Ok(mac_usage(6, 7))), reason::HOST_EXITED); + assert_eq!( + host(Ok(platform::Usage { + exited: true, + ..mac_usage(5, 7) + })), + reason::HOST_EXITED + ); + assert_eq!( + host(Err(platform::ReadError::Denied)), + reason::PROCESS_UNREADABLE + ); + } + + /// Scripted `/proc` answers for the Linux tree walk. + #[cfg(target_os = "linux")] + #[derive(Default)] + struct ScriptedProcfs { + pids: Vec, + unlisted: bool, + /// Answers to `stat`, first the listing's, then the walk's. + stats: std::collections::BTreeMap< + u32, + std::cell::RefCell>>, + >, + } + + #[cfg(target_os = "linux")] + impl ScriptedProcfs { + fn process( + mut self, + pid: u32, + stats: Vec>, + ) -> Self { + self.pids.push(pid); + self.stats.insert(pid, std::cell::RefCell::new(stats)); + self + } + } + + #[cfg(target_os = "linux")] + impl platform::Procfs for ScriptedProcfs { + fn pids(&self) -> std::result::Result, platform::ReadError> { + if self.unlisted { + return Err(platform::ReadError::Unreadable); + } + Ok(self.pids.clone()) + } + + fn stat(&self, pid: u32) -> std::result::Result { + let mut answers = self.stats.get(&pid).expect("scripted pid").borrow_mut(); + if answers.len() > 1 { + answers.remove(0) + } else { + answers[0] + } + } + } + + #[cfg(target_os = "linux")] + fn linux_stat(ppid: u32, start: u64, cpu_ticks: u64) -> platform::Stat { + platform::Stat { + zombie: false, + ppid, + cpu_ticks, + start, + rss_pages: 1, + } + } + + /// The host is PID 10 started at tick 5; its child is PID 11. + #[cfg(target_os = "linux")] + fn linux_tree_with_child( + child: Vec>, + ) -> std::result::Result { + let procfs = ScriptedProcfs::default() + .process(10, vec![Ok(linux_stat(1, 5, 7))]) + .process(11, child); + platform::read_tree_with(&procfs, 10, 5, 1, 1) + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_with_unreadable_live_child_is_unavailable() { + let read = linux_tree_with_child(vec![ + Ok(linux_stat(10, 9, 3)), + Err(platform::ReadError::Unreadable), + ]); + + assert_eq!(read.unwrap_err(), reason::PROCESS_UNREADABLE); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_skips_child_reaped_or_replaced_between_reads() { + let reaped = linux_tree_with_child(vec![ + Ok(linux_stat(10, 9, 3)), + Err(platform::ReadError::Gone), + ]) + .expect("tree read"); + let replaced = + linux_tree_with_child(vec![Ok(linux_stat(10, 9, 3)), Ok(linux_stat(1, 12, 900))]) + .expect("tree read"); + let counted = linux_tree_with_child(vec![Ok(linux_stat(10, 9, 3))]).expect("tree read"); + + for read in [reaped, replaced] { + assert_eq!(read.process_count, 1); + assert_eq!(read.cpu_nanos, 7_000_000_000); + } + assert_eq!(counted.process_count, 2); + assert_eq!(counted.cpu_nanos, 10_000_000_000); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_with_incomplete_process_table_is_unavailable() { + let unlisted = ScriptedProcfs { + unlisted: true, + ..ScriptedProcfs::default() + }; + let unreadable_other = ScriptedProcfs::default() + .process(10, vec![Ok(linux_stat(1, 5, 7))]) + .process(50, vec![Err(platform::ReadError::Unreadable)]); + + assert_eq!( + platform::read_tree_with(&unlisted, 10, 5, 1, 1).unwrap_err(), + reason::TABLE_UNREADABLE + ); + assert_eq!( + platform::read_tree_with(&unreadable_other, 10, 5, 1, 1).unwrap_err(), + reason::TABLE_UNREADABLE + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_of_exited_or_unreadable_host_is_unavailable() { + let host = |walk| { + let procfs = ScriptedProcfs::default().process(10, vec![Ok(linux_stat(1, 5, 7)), walk]); + platform::read_tree_with(&procfs, 10, 5, 1, 1).unwrap_err() + }; + + assert_eq!(host(Err(platform::ReadError::Gone)), reason::HOST_EXITED); + assert_eq!(host(Ok(linux_stat(1, 6, 7))), reason::HOST_EXITED); + assert_eq!( + host(Ok(platform::Stat { + zombie: true, + ..linux_stat(1, 5, 7) + })), + reason::HOST_EXITED + ); + assert_eq!( + host(Err(platform::ReadError::Unreadable)), + reason::PROCESS_UNREADABLE + ); + } + #[cfg(target_os = "macos")] #[test] fn macos_mach_units_convert_with_the_timebase() { diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 9630f9f6e..589cdbd78 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1177,7 +1177,8 @@ pub fn invoke_runtime( /// The snapshot covers the runtime's local adapter host process and its live /// descendants. It does not wait for a running invocation. It returns /// [`FabricError::ResourceObservationUnavailable`] when the runtime stopped, is -/// stopping, its host exited, or the process table cannot be read, and +/// stopping, its host exited, or a live process in its tree or the process +/// table cannot be read, and /// [`FabricError::ResourceObservationUnsupported`] when the environment provider /// or operating system has no resource reader. pub fn runtime_resource_snapshot( @@ -4881,6 +4882,29 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn runtime_resource_snapshot_after_invocation_timeout_is_unavailable() { + let (root, plan) = local_host_plan("invoke_timeout"); + let runtime = start_runtime(&plan).expect("start local host"); + runtime_resource_snapshot(&plan, &runtime).expect("snapshot before the timeout"); + + run_local_host_adapter_with_timeout( + &plan, + &runtime, + RunRequest::text("hang"), + Duration::from_millis(100), + ) + .expect_err("unresponsive host must time out"); + + assert_unavailable( + &runtime_resource_snapshot(&plan, &runtime), + &runtime.runtime_id, + ); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } + #[test] fn runtime_resource_snapshot_of_non_local_environment_is_unsupported() { let (root, plan) = local_host_plan("success"); diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 44587f33b..c068a184a 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -28,7 +28,7 @@ pyo3::create_exception!( _native, ResourceObservationUnavailableError, PyRuntimeError, - "Raised when a runtime stopped, is stopping, its adapter host exited, or the process table cannot be read." + "Raised when a runtime stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read." ); pyo3::create_exception!( _native, diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 6fbc919b8..999a90775 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -705,16 +705,18 @@ the runtime, plus its descendants. For a runtime started on a shared service, the service process is not included. A descendant that starts a new session or process group still counts while its parent chain leads to the host. A process whose parent exited is reparented outside the tree and no longer counts. The -snapshot reads `/proc` on Linux and `proc_pid_rusage` on macOS. It uses only -process IDs, parent IDs, start times, CPU times, and resident sizes, never reads -command lines or environment variables, and reports only the runtime's tree. -Descendants that NeMo Fabric is not permitted to inspect are left out. +snapshot reads `/proc` on Linux and `proc_pid_rusage` on macOS. To find the +tree, it reads the parent ID and start time of every process on the machine. +The snapshot exposes only the fields above, never command lines, environment +variables, credentials, or processes outside the runtime's tree. When NeMo Fabric cannot read the values, it raises an error instead of -returning zeros: +returning zeros or leaving a process out: - `ResourceObservationUnavailableError` when the runtime stopped, is stopping, - its adapter host process exited, or the process table cannot be read. + its adapter host process exited, or a live process in its tree or the + process table cannot be read, for example a macOS descendant that runs as + another user. - `ResourceObservationUnsupportedError` when the runtime's environment provider or operating system has no resource reader. Windows raises this error. @@ -1115,7 +1117,7 @@ All public SDK errors inherit from `FabricError`. | `FabricRuntimeError` | Runtime startup, invocation, or shutdown failed before a normalized result could be returned. | | `FabricStateError` | Invalid runtime state transition, such as invoking after stop or starting overlapping invocations. | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | -| `ResourceObservationUnavailableError` | `Runtime.resource_snapshot()` cannot read the runtime because it stopped, is stopping, its adapter host exited, or the process table cannot be read. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnavailableError` | `Runtime.resource_snapshot()` cannot read the runtime because it stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read. Subclass of `FabricRuntimeError`. | | `ResourceObservationUnsupportedError` | The runtime's environment provider or operating system has no resource reader. Subclass of `FabricRuntimeError`. | Consumers own job-level retries and rollout-level failure policy. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index 5edb2bbf4..6f6b8e0c9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -84,8 +84,8 @@ class FabricNativeUnavailableError(FabricRuntimeError): class ResourceObservationUnavailableError(FabricRuntimeError): """Runtime resources cannot be read now. - The runtime stopped, is stopping, its adapter host process exited, or the - process table cannot be read. + The runtime stopped, is stopping, its adapter host process exited, or a + live process in its tree or the process table cannot be read. """ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 28be1b623..a421eb3ff 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -749,8 +749,8 @@ async def resource_snapshot(self) -> RuntimeResourceSnapshot: Raises: ResourceObservationUnavailableError: If the runtime stopped, is - stopping, its adapter host process exited, or the process - table cannot be read. + stopping, its adapter host process exited, or a live process + in its tree or the process table cannot be read. ResourceObservationUnsupportedError: If the runtime's environment provider or operating system has no resource reader. FabricNativeUnavailableError: If the native extension is missing. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 094b75614..2500fa488 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -83,7 +83,7 @@ Refer to the [errors reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/ | `FabricRuntimeError` | Startup, invocation, or shutdown failed before a normalized result. | | `FabricStateError` | Invalid runtime state transition (invoking after stop, overlapping invocations). | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | -| `ResourceObservationUnavailableError` | A resource snapshot cannot be read because the runtime stopped, is stopping, its adapter host exited, or the process table cannot be read. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnavailableError` | A resource snapshot cannot be read because the runtime stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read. Subclass of `FabricRuntimeError`. | | `ResourceObservationUnsupportedError` | The runtime's environment provider or operating system has no resource reader. Subclass of `FabricRuntimeError`. | ## Cleanup And Resilience From 5cf94dc011d84b61c5cb3792a281d69551b4e9da Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 00:50:42 -0700 Subject: [PATCH 008/102] perf: read only the runtime's own processes on Linux Each local invocation reads a snapshot while it holds its host's lock. The Linux reader parsed /proc//stat for every process on the machine, so the cost grew with the machine's process count. In a release build in Docker (rust:1.94, aarch64), the median over 60 reads was 0.66 ms with 108 processes, 5.8 to 7.1 ms with 1,008, 35 to 38 ms with 5,008 and 72 to 74 ms with 10,008. The reader now lists each process's children from /proc//task//children and reads only the runtime's tree. The same benchmark reads 0.027 to 0.028 ms at every process count. A child counts only when its stat, read after the listing, still names the expected parent. The kernel can skip a live child when a sibling is reaped during the read, so the children list is read until two reads agree, and the snapshot is unavailable after four reads that differ. Kernels without children files keep the full scan. Also name the macOS children map type, which clippy flagged. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 214 +++++++++++++++++--- docs/sdk/python.mdx | 7 +- 2 files changed, 189 insertions(+), 32 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 579381e15..c6d004828 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -18,9 +18,10 @@ //! leaving the process out. //! //! Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. To find -//! the tree, the readers also read other processes' parent IDs and start -//! times. A snapshot exposes only its own fields: never command lines, -//! environment variables, open files, or processes outside the tree. +//! the tree, the macOS reader, and the Linux reader on a kernel without +//! `/proc//task//children`, also read other processes' parent IDs. +//! A snapshot exposes only its own fields: never command lines, environment +//! variables, open files, or processes outside the tree. use std::time::{SystemTime, UNIX_EPOCH}; @@ -198,8 +199,10 @@ fn civil_from_days(days: i64) -> (i64, i64, i64) { #[cfg(target_os = "linux")] mod platform { - //! `/proc` reads: one pass lists every process's parent, then the tree is - //! read from the root down, each parent before its children. + //! `/proc` reads, from the root down, each parent before its children. + //! Each process's children come from `/proc//task//children`, + //! so a read visits only the runtime's own processes. A kernel built + //! without those files falls back to one pass over every process's parent. use std::collections::{BTreeMap, VecDeque}; use std::sync::OnceLock; @@ -248,8 +251,15 @@ mod platform { }) } + /// Times a process's children are read while waiting for two reads that agree. + const MAX_CHILDREN_READS: usize = 4; + /// The `/proc` reads a tree read makes. Tests script them. pub(super) trait Procfs { + /// Whether the kernel provides `/proc//task//children`. + fn has_children_files(&self) -> bool; + /// The children of every thread of `pid`, sorted. + fn children(&self, pid: u32) -> Result, ReadError>; /// Every PID in `/proc`. fn pids(&self) -> Result, ReadError>; fn stat(&self, pid: u32) -> Result; @@ -258,6 +268,18 @@ mod platform { struct Live; impl Procfs for Live { + fn has_children_files(&self) -> bool { + static PRESENT: OnceLock = OnceLock::new(); + *PRESENT.get_or_init(|| { + let pid = std::process::id(); + std::path::Path::new(&format!("/proc/{pid}/task/{pid}/children")).exists() + }) + } + + fn children(&self, pid: u32) -> Result, ReadError> { + read_until_stable(|| children_once(pid)) + } + fn pids(&self) -> Result, ReadError> { let mut pids = Vec::new(); for entry in std::fs::read_dir("/proc").map_err(read_error)? { @@ -275,6 +297,43 @@ mod platform { } } + /// One read of the children files of every thread of `pid`. A thread that + /// exited during the read is skipped; its children move to another thread. + fn children_once(pid: u32) -> Result, ReadError> { + let mut children = Vec::new(); + for entry in std::fs::read_dir(format!("/proc/{pid}/task")).map_err(read_error)? { + let tid = entry.map_err(read_error)?.file_name(); + let path = format!("/proc/{pid}/task/{}/children", tid.to_string_lossy()); + let text = match std::fs::read_to_string(path).map_err(read_error) { + Ok(text) => text, + Err(ReadError::Gone) => continue, + Err(unreadable) => return Err(unreadable), + }; + for child in text.split_whitespace() { + children.push(child.parse().map_err(|_| ReadError::Unreadable)?); + } + } + children.sort_unstable(); + Ok(children) + } + + /// Read a process's children until two reads in a row agree. The kernel + /// can skip a live child when a sibling is reaped during the read, and a + /// thread's children move when it exits; either changes the next read. + pub(super) fn read_until_stable( + mut read: impl FnMut() -> Result, ReadError>, + ) -> Result, ReadError> { + let mut last = read()?; + for _ in 1..MAX_CHILDREN_READS { + let next = read()?; + if next == last { + return Ok(next); + } + last = next; + } + Err(ReadError::Unreadable) + } + /// `ENOENT` and `ESRCH` mean the process exited; anything else means it /// cannot be read. fn read_error(error: std::io::Error) -> ReadError { @@ -309,21 +368,15 @@ mod platform { read_tree_with(&Live, root, identity, clock_ticks_per_second(), page_size()) } - /// Child PIDs and start times by parent PID. - type ChildrenByParent = BTreeMap>; - - /// Every process's children, each with its start time. A process that - /// exited during the pass is left out; one that cannot be read leaves the - /// listing incomplete. - fn children_by_parent(procfs: &impl Procfs) -> Result { + /// Every process's children, for a kernel without children files. A + /// process that exited during the pass is left out; one that cannot be + /// read leaves the listing incomplete. + fn children_by_parent(procfs: &impl Procfs) -> Result>, &'static str> { let pids = procfs.pids().map_err(|_| reason::TABLE_UNREADABLE)?; - let mut children = ChildrenByParent::new(); + let mut children: BTreeMap> = BTreeMap::new(); for pid in pids { match procfs.stat(pid) { - Ok(stat) => children - .entry(stat.ppid) - .or_default() - .push((pid, stat.start)), + Ok(stat) => children.entry(stat.ppid).or_default().push(pid), Err(ReadError::Gone) => {} Err(ReadError::Unreadable) => return Err(reason::TABLE_UNREADABLE), } @@ -341,16 +394,32 @@ mod platform { ticks_per_second: u64, page_size: u64, ) -> Result { - let children = children_by_parent(procfs)?; + let listing = if procfs.has_children_files() { + None + } else { + Some(children_by_parent(procfs)?) + }; let mut read = TreeRead::default(); let mut cpu_ticks: u64 = 0; let mut rss_pages: u64 = 0; - let mut pending = VecDeque::from([(root, identity)]); - while let Some((pid, expected_start)) = pending.pop_front() { - let stat = match procfs.stat(pid) { - Ok(stat) if stat.start == expected_start && !(pid == root && stat.zombie) => stat, + // Each entry is a PID and the parent it was listed under (none for the root). + let mut pending = VecDeque::from([(root, None)]); + while let Some((pid, parent)) = pending.pop_front() { + // List the children before the stat read: a stat that still names + // the expected process shows the listed children were its own. + let children = match &listing { + Some(listing) => Ok(listing.get(&pid).cloned().unwrap_or_default()), + None => procfs.children(pid), + }; + let found = children.and_then(|children| Ok((procfs.stat(pid)?, children))); + let expected = |stat: &Stat| match parent { + None => stat.start == identity && !stat.zombie, + Some(parent) => stat.ppid == parent, + }; + let (stat, children) = match found { + Ok((stat, children)) if expected(&stat) => (stat, children), // An exited child, or one whose PID now names another process. - Ok(_) | Err(ReadError::Gone) if pid != root => continue, + Ok(_) | Err(ReadError::Gone) if parent.is_some() => continue, Ok(_) | Err(ReadError::Gone) => return Err(reason::HOST_EXITED), Err(ReadError::Unreadable) => return Err(reason::PROCESS_UNREADABLE), }; @@ -359,9 +428,7 @@ mod platform { rss_pages = rss_pages.saturating_add(stat.rss_pages); read.process_count += 1; } - // A child never starts before its parent; one that seems to reuses a PID. - let own = children.get(&pid).into_iter().flatten(); - pending.extend(own.filter(|(_, start)| *start >= stat.start).copied()); + pending.extend(children.into_iter().map(|child| (child, Some(pid)))); } read.cpu_nanos = ticks_to_nanos(cpu_ticks, ticks_per_second); read.resident_bytes = rss_pages.saturating_mul(page_size); @@ -581,16 +648,17 @@ mod platform { read_tree_with(&Live, root, identity, timebase) } + /// Child PIDs, each with its parent record, by parent PID. + type ChildrenByParent = BTreeMap>; + /// Every live process's children, each with its parent record. One /// `proc_listallpids` call and one `proc_pidinfo` per process: /// `proc_listchildpids` scans the whole process table on each call, which /// costs too much for a wide tree. A process that exited during the pass /// is left out; one that cannot be read leaves the listing incomplete. - fn children_by_parent( - procs: &impl Libproc, - ) -> Result>, &'static str> { + fn children_by_parent(procs: &impl Libproc) -> Result { let pids = procs.pids().ok_or(reason::TABLE_UNREADABLE)?; - let mut children: BTreeMap> = BTreeMap::new(); + let mut children = ChildrenByParent::new(); for pid in pids { match procs.parent(pid) { Ok(parent) => children.entry(parent.ppid).or_default().push((pid, parent)), @@ -997,6 +1065,11 @@ mod tests { #[cfg(target_os = "linux")] #[derive(Default)] struct ScriptedProcfs { + /// Children by parent from children files, or `None` for a kernel + /// without them. + children: Option< + std::collections::BTreeMap, platform::ReadError>>, + >, pids: Vec, unlisted: bool, /// Answers to `stat`, first the listing's, then the walk's. @@ -1021,6 +1094,15 @@ mod tests { #[cfg(target_os = "linux")] impl platform::Procfs for ScriptedProcfs { + fn has_children_files(&self) -> bool { + self.children.is_some() + } + + fn children(&self, pid: u32) -> std::result::Result, platform::ReadError> { + let children = self.children.as_ref().expect("children files"); + children.get(&pid).cloned().unwrap_or(Ok(Vec::new())) + } + fn pids(&self) -> std::result::Result, platform::ReadError> { if self.unlisted { return Err(platform::ReadError::Unreadable); @@ -1136,6 +1218,78 @@ mod tests { ); } + /// A kernel with children files: the host is PID 10 started at tick 5 and + /// lists child PID 11. The process table is never listed. + #[cfg(target_os = "linux")] + fn linux_children_tree( + host_children: std::result::Result, platform::ReadError>, + child: std::result::Result, + ) -> std::result::Result { + let mut procfs = ScriptedProcfs { + children: Some(std::collections::BTreeMap::from([(10, host_children)])), + unlisted: true, + ..ScriptedProcfs::default() + } + .process(10, vec![Ok(linux_stat(1, 5, 7))]) + .process(11, vec![child]); + procfs.pids.clear(); + platform::read_tree_with(&procfs, 10, 5, 1, 1) + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_with_children_files_reads_only_the_tree() { + let read = linux_children_tree(Ok(vec![11]), Ok(linux_stat(10, 9, 3))).expect("tree read"); + + assert_eq!(read.process_count, 2); + assert_eq!(read.cpu_nanos, 10_000_000_000); + assert_eq!(read.resident_bytes, 2); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_with_children_files_skips_reaped_or_replaced_child() { + let reaped = + linux_children_tree(Ok(vec![11]), Err(platform::ReadError::Gone)).expect("tree read"); + let replaced = + linux_children_tree(Ok(vec![11]), Ok(linux_stat(1, 12, 900))).expect("tree read"); + + for read in [reaped, replaced] { + assert_eq!(read.process_count, 1); + assert_eq!(read.cpu_nanos, 7_000_000_000); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_with_children_files_and_unreadable_process_is_unavailable() { + let child = linux_children_tree(Ok(vec![11]), Err(platform::ReadError::Unreadable)); + let host_children = linux_children_tree( + Err(platform::ReadError::Unreadable), + Ok(linux_stat(10, 9, 3)), + ); + let host_gone = + linux_children_tree(Err(platform::ReadError::Gone), Ok(linux_stat(10, 9, 3))); + + assert_eq!(child.unwrap_err(), reason::PROCESS_UNREADABLE); + assert_eq!(host_children.unwrap_err(), reason::PROCESS_UNREADABLE); + assert_eq!(host_gone.unwrap_err(), reason::HOST_EXITED); + } + + /// The second read finds a child the first skipped; the third agrees. + #[cfg(target_os = "linux")] + #[test] + fn linux_children_read_waits_for_two_reads_that_agree() { + let mut reads = vec![vec![11, 13], vec![11, 12, 13], vec![11, 12, 13]].into_iter(); + let mut changing = (0..).map(|n| vec![n]); + + let stable = platform::read_until_stable(|| Ok(reads.next().expect("read"))); + let unstable = platform::read_until_stable(|| Ok(changing.next().expect("read"))); + + assert_eq!(stable, Ok(vec![11, 12, 13])); + assert_eq!(unstable, Err(platform::ReadError::Unreadable)); + } + #[cfg(target_os = "macos")] #[test] fn macos_mach_units_convert_with_the_timebase() { diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 999a90775..ae9926b55 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -705,8 +705,11 @@ the runtime, plus its descendants. For a runtime started on a shared service, the service process is not included. A descendant that starts a new session or process group still counts while its parent chain leads to the host. A process whose parent exited is reparented outside the tree and no longer counts. The -snapshot reads `/proc` on Linux and `proc_pid_rusage` on macOS. To find the -tree, it reads the parent ID and start time of every process on the machine. +snapshot reads `/proc` on Linux and `proc_pid_rusage` on macOS. On Linux it +finds the tree from each process's `/proc//task//children` list, so it +reads only the runtime's own processes. On macOS, and on a Linux kernel without +those lists, it reads the parent ID of every process on the machine to find the +tree, and macOS also reads each process's start time to detect a reused PID. The snapshot exposes only the fields above, never command lines, environment variables, credentials, or processes outside the runtime's tree. From 9c54d0a619e5a3c9b62b5b1d3a098e46831ba23a Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 00:51:42 -0700 Subject: [PATCH 009/102] docs: regenerate API references for unavailable snapshot reads The FabricError, runtime_resource_snapshot, resource_snapshot module and Python error docstrings now name an unreadable live process as a cause of ResourceObservationUnavailable, and the module docs describe how each reader finds the tree. Regenerated with Rust 1.94.0 and the locked docs tools; only the changed sentences are applied. Signed-off-by: mnajafian-nv --- .../api/python-library-reference/nemo_fabric.errors.md | 2 +- .../api/python-library-reference/nemo_fabric.runtime.md | 2 +- .../nemo-fabric-core/error/enum-fabricerror.mdx | 2 +- .../nemo-fabric-core/resource-snapshot/index.mdx | 4 +++- .../nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/reference/api/python-library-reference/nemo_fabric.errors.md b/docs/reference/api/python-library-reference/nemo_fabric.errors.md index 73b6ba7a8..e3acafc12 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.errors.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.errors.md @@ -281,7 +281,7 @@ Initialize a structured NeMo Fabric exception. Runtime resources cannot be read now. -The runtime stopped, is stopping, its adapter host process exited, or the process table cannot be read. +The runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. ### Inheritance diff --git a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md index b77596c57..98ed49750 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md @@ -197,7 +197,7 @@ The snapshot covers the local adapter host process and its live descendants. It **Raises:** - - `ResourceObservationUnavailableError`: If the runtime stopped, is stopping, its adapter host process exited, or the process table cannot be read. + - `ResourceObservationUnavailableError`: If the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. - `ResourceObservationUnsupportedError`: If the runtime's environment provider or operating system has no resource reader. - `FabricNativeUnavailableError`: If the native extension is missing. - `FabricRuntimeError`: If the native snapshot call fails otherwise. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 4c5c5da33..d4082937d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -501,7 +501,7 @@ Adapter kind.
-A runtime's resources cannot be read now: the runtime stopped, is stopping, its adapter host process exited, or the process table cannot be read. +A runtime's resources cannot be read now: the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. #### Fields diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index 7ff9eab7c..eefcd4bb4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -14,7 +14,9 @@ Resource snapshots of a runtime's local adapter host and its descendants. A snapshot reads the process tree rooted at the adapter host process that NeMo Fabric started for a runtime. Each read visits a parent before its children. CPU time adds every process's own CPU time and the CPU time of the children it has already waited for, so work done by finished descendants still counts. A child reaped between the read of its parent and its own read is missed by that one snapshot and is never counted twice. CPU time can drop between snapshots when a descendant is reparented out of the tree, and on macOS while an exited descendant is not yet waited for, because macOS cannot list an unreaped process. -Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. Neither reads command lines, environment variables, or open files. +A live process in the tree that cannot be read, or a process table that cannot be listed in full, makes the snapshot unavailable instead of leaving the process out. + +Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. To find the tree, the macOS reader, and the Linux reader on a kernel without `/proc//task//children`, also read other processes' parent IDs. A snapshot exposes only its own fields: never command lines, environment variables, open files, or processes outside the tree. ## Structs diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx index cbe246229..745457bd3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx @@ -13,4 +13,4 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Read a timestamped resource snapshot of a started runtime. -The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, or the process table cannot be read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the environment provider or operating system has no resource reader. +The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, or a live process in its tree or the process table cannot be read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the environment provider or operating system has no resource reader. From 888230bfc51da32c6c08059de89ffe820d0f8e08 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:08:54 -0700 Subject: [PATCH 010/102] fix: read macOS snapshots from each process's children The macOS reader listed every process on the machine and read each one's parent to find the runtime's tree. That needed a short BSD record fallback for other users' processes, made a snapshot unavailable when any process on the machine refused both records, and dropped an exited child that its parent had not yet waited for out of CPU time, because PROC_PIDTBSDINFO fails on a zombie. The reader now lists each process's children with proc_listchildpids, as the Linux reader does with /proc children files. proc_listchildpids lists zombies and proc_pid_rusage reads them, so an unreaped child's CPU time counts on macOS the same way it does on Linux. A child counts only when it started before its parent's children were listed, which proves its PID was not reused after the listing. The walk now lives outside the platform modules behind one ProcessTable trait, so its scripted tests run on Linux CI for both identity rules. The Linux reader reports a children list that never settles as a changing tree instead of an unreadable process. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 1317 +++++++---------- docs/sdk/python.mdx | 20 +- schemas/sdk/run-result.schema.json | 2 +- .../sdk/runtime-resource-snapshot.schema.json | 2 +- .../src/nemo_fabric/types.py | 4 +- 5 files changed, 568 insertions(+), 777 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index c6d004828..3c4582cb6 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -4,24 +4,23 @@ //! Resource snapshots of a runtime's local adapter host and its descendants. //! //! A snapshot reads the process tree rooted at the adapter host process that -//! NeMo Fabric started for a runtime. Each read visits a parent before its -//! children. CPU time adds every process's own CPU time and the CPU time of the -//! children it has already waited for, so work done by finished descendants -//! still counts. A child reaped between the read of its parent and its own -//! read is missed by that one snapshot and is never counted twice. CPU time -//! can drop between snapshots when a descendant is reparented out of the tree, -//! and on macOS while an exited descendant is not yet waited for, because -//! macOS cannot list an unreaped process. +//! NeMo Fabric started for a runtime, each parent before its children. CPU +//! time adds every process's own CPU time and the CPU time of the children it +//! has already waited for. An exited child that is not yet waited for counts +//! through its own CPU time, so work done by finished descendants still +//! counts. A child reaped between the read of its parent and its own read is +//! missed by that one snapshot and is never counted twice. CPU time can drop +//! between snapshots when a descendant is reparented out of the tree. //! -//! A live process in the tree that cannot be read, or a process table that -//! cannot be listed in full, makes the snapshot unavailable instead of -//! leaving the process out. +//! Linux reads `/proc//task//children` and `/proc//stat`. On a +//! kernel without those children files, it reads every process's parent ID +//! once per snapshot to find the tree. macOS reads `proc_listchildpids` and +//! `proc_pid_rusage`. A snapshot exposes only its own fields: never command +//! lines, environment variables, open files, or processes outside the tree. //! -//! Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. To find -//! the tree, the macOS reader, and the Linux reader on a kernel without -//! `/proc//task//children`, also read other processes' parent IDs. -//! A snapshot exposes only its own fields: never command lines, environment -//! variables, open files, or processes outside the tree. +//! A live process in the tree that cannot be read, or a process table that +//! the Linux fallback cannot read in full, makes the snapshot unavailable +//! instead of leaving the process out. use std::time::{SystemTime, UNIX_EPOCH}; @@ -33,18 +32,6 @@ use crate::error::{FabricError, Result}; const MILLIS_PER_DAY: i64 = 86_400_000; const NANOS_PER_SECOND: f64 = 1_000_000_000.0; -/// Reasons a tree read returns when the snapshot is unavailable. -#[cfg_attr( - not(any(target_os = "linux", target_os = "macos")), - allow(dead_code, reason = "only the Linux and macOS readers read a tree") -)] -mod reason { - pub(super) const HOST_EXITED: &str = "adapter host process has exited"; - pub(super) const PROCESS_UNREADABLE: &str = - "a live process in the runtime's process tree cannot be read"; - pub(super) const TABLE_UNREADABLE: &str = "the process table cannot be read in full"; -} - /// Timestamped resource usage of one runtime's local adapter host process tree. /// /// Memory, process count, and CPU time cover the adapter host process and its @@ -68,9 +55,8 @@ pub struct RuntimeResourceSnapshot { /// that are not yet reaped are not counted. pub process_count: u64, /// CPU time consumed by the tree since the adapter host started, in seconds, - /// including exited descendants (on macOS, only once they are waited for). - /// It can drop when a descendant leaves the tree, and on macOS while an - /// exited descendant is not yet waited for. + /// including exited descendants. It can drop when a descendant leaves the + /// tree. #[schemars(range(min = 0.0))] pub cpu_time_seconds: f64, } @@ -154,6 +140,119 @@ struct TreeRead { process_count: u64, } +/// The tree walk the Linux and macOS readers share. It is platform-neutral so +/// its tests run on every platform. +#[cfg_attr( + not(any(target_os = "linux", target_os = "macos")), + allow(dead_code, reason = "only the Linux and macOS readers walk a tree") +)] +mod tree { + use std::collections::VecDeque; + + use super::TreeRead; + + pub(super) const HOST_EXITED: &str = "adapter host process has exited"; + pub(super) const PROCESS_UNREADABLE: &str = + "a live process in the runtime's process tree cannot be read"; + + /// Why a process, or the list of its children, could not be read. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) enum ReadError { + /// The process exited and was reaped. + Gone, + /// The process exists but cannot be read, for this reason. + Unavailable(&'static str), + } + + /// One process as a tree read sees it. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) struct Process { + /// Parent PID, when the platform reads it in the same record as the + /// rest of these fields. + pub ppid: Option, + /// Start time, in the clock that [`Children::listed_at`] uses. + pub start: u64, + /// Whether the process exited and is not yet reaped. + pub exited: bool, + /// The process's own CPU time plus that of the children it waited for. + pub cpu_nanos: u64, + pub resident_bytes: u64, + } + + /// The children of one process, listed at one instant. + #[derive(Debug, Clone, PartialEq, Eq)] + pub(super) struct Children { + pub pids: Vec, + /// When the list was read, in the clock that [`Process::start`] uses. + /// Only a process read without its parent PID is checked against it. + pub listed_at: u64, + } + + /// The reads a tree read makes. Each platform implements them; tests + /// script them. + pub(super) trait ProcessTable { + fn children(&self, pid: u32) -> Result; + fn read(&self, pid: u32) -> Result; + } + + /// Whether `process`, listed as a child of `parent` at `listed_at` and + /// read afterwards, is still the listed process. A parent PID read with + /// the record shows it directly. Without one, a process that started + /// before the list was read already held its PID then, so it is the + /// listed process. + fn is_listed_child(process: &Process, parent: u32, listed_at: u64) -> bool { + match process.ppid { + Some(ppid) => ppid == parent, + None => process.start < listed_at, + } + } + + /// Read the tree under `root`. Fails when the root exited or its PID now + /// names a process with another start time than `identity`, or when a + /// live process in the tree cannot be read. + pub(super) fn read_tree_with( + table: &impl ProcessTable, + root: u32, + identity: u64, + ) -> Result { + let mut read = TreeRead::default(); + // Each entry is a PID and, for a child, its parent and the time the + // parent's children were listed. + let mut pending = VecDeque::from([(root, None)]); + while let Some((pid, listed)) = pending.pop_front() { + // List the children before reading the process: a read that still + // shows the expected process proves the listed children were its own. + let found = table + .children(pid) + .and_then(|children| Ok((table.read(pid)?, children))); + let expected = |process: &Process| match listed { + None => process.start == identity && !process.exited, + Some((parent, listed_at)) => is_listed_child(process, parent, listed_at), + }; + let (process, children) = match found { + Ok((process, children)) if expected(&process) => (process, children), + // A reaped child, or one whose PID now names another process. + Ok(_) | Err(ReadError::Gone) if listed.is_some() => continue, + Ok(_) | Err(ReadError::Gone) => return Err(HOST_EXITED), + Err(ReadError::Unavailable(reason)) => return Err(reason), + }; + read.cpu_nanos = read.cpu_nanos.saturating_add(process.cpu_nanos); + if !process.exited { + read.resident_bytes = read.resident_bytes.saturating_add(process.resident_bytes); + read.process_count += 1; + } + let listed_at = children.listed_at; + pending.extend( + children + .pids + .into_iter() + .map(|child| (child, Some((pid, listed_at)))), + ); + } + Ok(read) + } +} + fn unix_millis(time: SystemTime) -> i64 { match time.duration_since(UNIX_EPOCH) { Ok(after) => i64::try_from(after.as_millis()).unwrap_or(i64::MAX), @@ -199,26 +298,25 @@ fn civil_from_days(days: i64) -> (i64, i64, i64) { #[cfg(target_os = "linux")] mod platform { - //! `/proc` reads, from the root down, each parent before its children. - //! Each process's children come from `/proc//task//children`, - //! so a read visits only the runtime's own processes. A kernel built - //! without those files falls back to one pass over every process's parent. + //! `/proc` reads. Each process's children come from + //! `/proc//task//children`, so a read visits only the runtime's + //! own processes. A kernel built without those files falls back to one + //! pass over every process's parent per snapshot. - use std::collections::{BTreeMap, VecDeque}; + use std::collections::BTreeMap; use std::sync::OnceLock; - use super::{TreeRead, reason}; + use super::TreeRead; + use super::tree::{ + Children, PROCESS_UNREADABLE, Process, ProcessTable, ReadError, read_tree_with, + }; pub(super) const SUPPORTED: bool = true; - - /// Why one process could not be read. - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) enum ReadError { - /// The process exited. - Gone, - /// The process exists but cannot be read, or its record cannot be parsed. - Unreadable, - } + pub(super) const TABLE_UNREADABLE: &str = "the process table cannot be read in full"; + pub(super) const TREE_CHANGING: &str = + "the runtime's process tree kept changing during the read"; + /// Times a process's children are read while waiting for two reads that agree. + const MAX_CHILDREN_READS: usize = 4; /// The fields of `/proc//stat` a snapshot needs. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -251,50 +349,45 @@ mod platform { }) } - /// Times a process's children are read while waiting for two reads that agree. - const MAX_CHILDREN_READS: usize = 4; - - /// The `/proc` reads a tree read makes. Tests script them. - pub(super) trait Procfs { - /// Whether the kernel provides `/proc//task//children`. - fn has_children_files(&self) -> bool; - /// The children of every thread of `pid`, sorted. - fn children(&self, pid: u32) -> Result, ReadError>; - /// Every PID in `/proc`. - fn pids(&self) -> Result, ReadError>; - fn stat(&self, pid: u32) -> Result; + /// Where a tree read finds each process's children. + pub(super) enum ChildSource { + /// `/proc//task//children`. + ChildrenFiles, + /// Children by parent PID, from one pass over every process. + Scanned(BTreeMap>), } - struct Live; - - impl Procfs for Live { - fn has_children_files(&self) -> bool { - static PRESENT: OnceLock = OnceLock::new(); - *PRESENT.get_or_init(|| { - let pid = std::process::id(); - std::path::Path::new(&format!("/proc/{pid}/task/{pid}/children")).exists() - }) - } + struct Live { + children: ChildSource, + ticks_per_second: u64, + page_size: u64, + } - fn children(&self, pid: u32) -> Result, ReadError> { - read_until_stable(|| children_once(pid)) + impl ProcessTable for Live { + fn children(&self, pid: u32) -> Result { + let pids = match &self.children { + ChildSource::ChildrenFiles => read_until_stable(|| children_once(pid))?, + ChildSource::Scanned(by_parent) => by_parent.get(&pid).cloned().unwrap_or_default(), + }; + // Each read checks the parent PID in `stat`, so the list time is unused. + Ok(Children { pids, listed_at: 0 }) } - fn pids(&self) -> Result, ReadError> { - let mut pids = Vec::new(); - for entry in std::fs::read_dir("/proc").map_err(read_error)? { - let name = entry.map_err(read_error)?.file_name(); - if let Some(pid) = name.to_str().and_then(|name| name.parse().ok()) { - pids.push(pid); - } - } - Ok(pids) + fn read(&self, pid: u32) -> Result { + let stat = stat(pid)?; + Ok(Process { + ppid: Some(stat.ppid), + start: stat.start, + exited: stat.zombie, + cpu_nanos: ticks_to_nanos(stat.cpu_ticks, self.ticks_per_second), + resident_bytes: stat.rss_pages.saturating_mul(self.page_size), + }) } + } - fn stat(&self, pid: u32) -> Result { - let text = std::fs::read_to_string(format!("/proc/{pid}/stat")).map_err(read_error)?; - parse_stat(&text).ok_or(ReadError::Unreadable) - } + fn stat(pid: u32) -> Result { + let text = std::fs::read_to_string(format!("/proc/{pid}/stat")).map_err(read_error)?; + parse_stat(&text).ok_or(ReadError::Unavailable(PROCESS_UNREADABLE)) } /// One read of the children files of every thread of `pid`. A thread that @@ -310,7 +403,11 @@ mod platform { Err(unreadable) => return Err(unreadable), }; for child in text.split_whitespace() { - children.push(child.parse().map_err(|_| ReadError::Unreadable)?); + children.push( + child + .parse() + .map_err(|_| ReadError::Unavailable(PROCESS_UNREADABLE))?, + ); } } children.sort_unstable(); @@ -331,7 +428,37 @@ mod platform { } last = next; } - Err(ReadError::Unreadable) + Err(ReadError::Unavailable(TREE_CHANGING)) + } + + /// Every process's children, for a kernel without children files. A + /// process that exited during the pass is left out; one whose parent + /// cannot be read leaves the listing incomplete. + pub(super) fn children_by_parent( + pids: Vec, + parent_of: impl Fn(u32) -> Result, + ) -> Result>, &'static str> { + let mut children: BTreeMap> = BTreeMap::new(); + for pid in pids { + match parent_of(pid) { + Ok(ppid) => children.entry(ppid).or_default().push(pid), + Err(ReadError::Gone) => {} + Err(ReadError::Unavailable(_)) => return Err(TABLE_UNREADABLE), + } + } + Ok(children) + } + + fn all_pids() -> Result, &'static str> { + let entries = std::fs::read_dir("/proc").map_err(|_| TABLE_UNREADABLE)?; + let mut pids = Vec::new(); + for entry in entries { + let name = entry.map_err(|_| TABLE_UNREADABLE)?.file_name(); + if let Some(pid) = name.to_str().and_then(|name| name.parse().ok()) { + pids.push(pid); + } + } + Ok(pids) } /// `ENOENT` and `ESRCH` mean the process exited; anything else means it @@ -341,12 +468,12 @@ mod platform { { ReadError::Gone } else { - ReadError::Unreadable + ReadError::Unavailable(PROCESS_UNREADABLE) } } pub(super) fn start_identity(pid: u32) -> Option { - Live.stat(pid).ok().map(|stat| stat.start) + stat(pid).ok().map(|stat| stat.start) } fn sysconf(name: libc::c_int) -> u64 { @@ -354,85 +481,42 @@ mod platform { u64::try_from(unsafe { libc::sysconf(name) }).unwrap_or(0) } - fn clock_ticks_per_second() -> u64 { - static TICKS: OnceLock = OnceLock::new(); - *TICKS.get_or_init(|| sysconf(libc::_SC_CLK_TCK).max(1)) + fn has_children_files() -> bool { + static PRESENT: OnceLock = OnceLock::new(); + *PRESENT.get_or_init(|| { + let pid = std::process::id(); + std::path::Path::new(&format!("/proc/{pid}/task/{pid}/children")).exists() + }) } - fn page_size() -> u64 { - static PAGE: OnceLock = OnceLock::new(); - *PAGE.get_or_init(|| sysconf(libc::_SC_PAGESIZE)) + /// Children by parent PID, from one pass over every process. + pub(super) fn scan_children() -> Result { + let by_parent = children_by_parent(all_pids()?, |pid| stat(pid).map(|stat| stat.ppid))?; + Ok(ChildSource::Scanned(by_parent)) } pub(super) fn read_tree(root: u32, identity: u64) -> Result { - read_tree_with(&Live, root, identity, clock_ticks_per_second(), page_size()) - } - - /// Every process's children, for a kernel without children files. A - /// process that exited during the pass is left out; one that cannot be - /// read leaves the listing incomplete. - fn children_by_parent(procfs: &impl Procfs) -> Result>, &'static str> { - let pids = procfs.pids().map_err(|_| reason::TABLE_UNREADABLE)?; - let mut children: BTreeMap> = BTreeMap::new(); - for pid in pids { - match procfs.stat(pid) { - Ok(stat) => children.entry(stat.ppid).or_default().push(pid), - Err(ReadError::Gone) => {} - Err(ReadError::Unreadable) => return Err(reason::TABLE_UNREADABLE), - } - } - Ok(children) + let children = if has_children_files() { + ChildSource::ChildrenFiles + } else { + scan_children()? + }; + read_tree_from(children, root, identity) } - /// Read the tree under `root`. Fails when the root exited or its PID now - /// names a process with another start time than `identity`, or when a - /// live process in the tree cannot be read. - pub(super) fn read_tree_with( - procfs: &impl Procfs, + pub(super) fn read_tree_from( + children: ChildSource, root: u32, identity: u64, - ticks_per_second: u64, - page_size: u64, ) -> Result { - let listing = if procfs.has_children_files() { - None - } else { - Some(children_by_parent(procfs)?) + static TICKS: OnceLock = OnceLock::new(); + static PAGE: OnceLock = OnceLock::new(); + let live = Live { + children, + ticks_per_second: *TICKS.get_or_init(|| sysconf(libc::_SC_CLK_TCK).max(1)), + page_size: *PAGE.get_or_init(|| sysconf(libc::_SC_PAGESIZE)), }; - let mut read = TreeRead::default(); - let mut cpu_ticks: u64 = 0; - let mut rss_pages: u64 = 0; - // Each entry is a PID and the parent it was listed under (none for the root). - let mut pending = VecDeque::from([(root, None)]); - while let Some((pid, parent)) = pending.pop_front() { - // List the children before the stat read: a stat that still names - // the expected process shows the listed children were its own. - let children = match &listing { - Some(listing) => Ok(listing.get(&pid).cloned().unwrap_or_default()), - None => procfs.children(pid), - }; - let found = children.and_then(|children| Ok((procfs.stat(pid)?, children))); - let expected = |stat: &Stat| match parent { - None => stat.start == identity && !stat.zombie, - Some(parent) => stat.ppid == parent, - }; - let (stat, children) = match found { - Ok((stat, children)) if expected(&stat) => (stat, children), - // An exited child, or one whose PID now names another process. - Ok(_) | Err(ReadError::Gone) if parent.is_some() => continue, - Ok(_) | Err(ReadError::Gone) => return Err(reason::HOST_EXITED), - Err(ReadError::Unreadable) => return Err(reason::PROCESS_UNREADABLE), - }; - cpu_ticks = cpu_ticks.saturating_add(stat.cpu_ticks); - if !stat.zombie { - rss_pages = rss_pages.saturating_add(stat.rss_pages); - read.process_count += 1; - } - pending.extend(children.into_iter().map(|child| (child, Some(pid)))); - } - read.cpu_nanos = ticks_to_nanos(cpu_ticks, ticks_per_second); - read.resident_bytes = rss_pages.saturating_mul(page_size); - Ok(read) + read_tree_with(&live, root, identity) } pub(super) fn ticks_to_nanos(ticks: u64, ticks_per_second: u64) -> u64 { @@ -443,167 +527,90 @@ mod platform { #[cfg(target_os = "macos")] mod platform { - //! `libproc` reads. One pass lists every process's parent and start time, - //! then the tree is read from the root down, each parent before its - //! children. + //! `libproc` reads. `proc_listchildpids` lists a process's children, + //! including exited ones that are not yet reaped, and `proc_pid_rusage` + //! reads any of them. - use std::collections::{BTreeMap, VecDeque}; use std::sync::OnceLock; - use super::{TreeRead, reason}; + use super::TreeRead; + use super::tree::{ + Children, PROCESS_UNREADABLE, Process, ProcessTable, ReadError, read_tree_with, + }; pub(super) const SUPPORTED: bool = true; + /// Child PIDs the first list call has room for. A full buffer doubles it. + const INITIAL_LISTED_CHILDREN: usize = 64; - /// Why one process could not be read. - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) enum ReadError { - /// The process exited, or is a zombie the call cannot describe. - Gone, - /// The process exists but this process may not read it. - Denied, - } - - /// A process's parent and start time, which together identify it. - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) struct Parent { - pub ppid: u32, - /// `pbi_start_tvsec` and `pbi_start_tvusec`, or `None` for another - /// user's process, whose parent only the short record shows. - pub started: Option<(u64, u64)>, - } - - /// `struct proc_bsdshortinfo` from ``, which the `libc` - /// crate version in use does not bind: PID, parent PID, then 56 bytes of - /// group, status, name, flags, and user and group IDs that the reader ignores. - #[repr(C)] - struct ProcBsdShortInfo { - pid: u32, - ppid: u32, - rest: [u32; 14], - } - - const PROC_PIDT_SHORTBSDINFO: libc::c_int = 13; - - /// The `proc_pid_rusage` fields a snapshot needs, CPU times in Mach units. - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) struct Usage { - pub cpu_units: u64, - pub resident_bytes: u64, - /// `ri_proc_start_abstime`. - pub start: u64, - pub exited: bool, - } - - /// The process reads a tree read makes. Tests script them. - pub(super) trait Libproc { - /// Every PID, or `None` when the process table cannot be listed. - fn pids(&self) -> Option>; - fn parent(&self, pid: u32) -> Result; - fn usage(&self, pid: u32) -> Result; + struct Live { + timebase: (u64, u64), } - struct Live; - - impl Libproc for Live { - fn pids(&self) -> Option> { - let mut pids = vec![0 as libc::pid_t; 1024]; - let count = loop { - let bytes = libc::c_int::try_from(std::mem::size_of_val(pids.as_slice())).ok()?; + impl ProcessTable for Live { + fn children(&self, pid: u32) -> Result { + let ppid = libc::pid_t::try_from(pid).map_err(|_| ReadError::Gone)?; + let mut pids = vec![0 as libc::pid_t; INITIAL_LISTED_CHILDREN]; + loop { + let bytes = libc::c_int::try_from(std::mem::size_of_val(pids.as_slice())) + .map_err(|_| ReadError::Unavailable(PROCESS_UNREADABLE))?; + // SAFETY: mach_absolute_time has no preconditions. + let listed_at = unsafe { mach_absolute_time() }; // SAFETY: the buffer is writable and sized in bytes as libproc requires. - let count = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) }; - let count = usize::try_from(count).ok().filter(|count| *count > 0)?; + let count = + unsafe { libc::proc_listchildpids(ppid, pids.as_mut_ptr().cast(), bytes) }; + let count = usize::try_from(count) + .map_err(|_| ReadError::Unavailable(PROCESS_UNREADABLE))?; + // A full buffer can mean the list was cut short. if count < pids.len() { - break count; + pids.truncate(count); + let pids = pids + .into_iter() + .filter_map(|pid| u32::try_from(pid).ok()) + .collect(); + return Ok(Children { pids, listed_at }); } pids.resize(pids.len() * 2, 0); - }; - Some( - pids[..count] - .iter() - .filter_map(|pid| u32::try_from(*pid).ok()) - .collect(), - ) - } - - /// `PROC_PIDTBSDINFO` fails for a zombie, so a zombie reads as gone. - /// It is denied for another user's process, whose parent the short - /// record still shows. - fn parent(&self, pid: u32) -> Result { - let pid = libc::pid_t::try_from(pid).map_err(|_| ReadError::Gone)?; - // SAFETY: proc_bsdinfo is plain data; zero is a valid initial value. - let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; - let size = libc::c_int::try_from(std::mem::size_of::()) - .map_err(|_| ReadError::Denied)?; - // SAFETY: `info` is writable and `size` bytes long, as PROC_PIDTBSDINFO needs. - let written = unsafe { - libc::proc_pidinfo(pid, libc::PROC_PIDTBSDINFO, 0, (&raw mut info).cast(), size) - }; - if written == size { - return Ok(Parent { - ppid: info.pbi_ppid, - started: Some((info.pbi_start_tvsec, info.pbi_start_tvusec)), - }); - } - match last_read_error() { - ReadError::Denied => short_parent(pid), - gone => Err(gone), } } - fn usage(&self, pid: u32) -> Result { - let pid = libc::c_int::try_from(pid).map_err(|_| ReadError::Gone)?; - // SAFETY: rusage_info_v2 is plain data; zero is a valid initial value. - let mut info: libc::rusage_info_v2 = unsafe { std::mem::zeroed() }; - // SAFETY: `info` is a writable rusage_info_v2, the layout RUSAGE_INFO_V2 names. - let status = unsafe { - libc::proc_pid_rusage( - pid, - libc::RUSAGE_INFO_V2, - (&raw mut info).cast::(), - ) - }; - if status != 0 { - return Err(last_read_error()); - } - Ok(Usage { - cpu_units: info - .ri_user_time - .saturating_add(info.ri_system_time) - .saturating_add(info.ri_child_user_time) - .saturating_add(info.ri_child_system_time), - resident_bytes: info.ri_resident_size, + fn read(&self, pid: u32) -> Result { + let info = usage(pid)?; + Ok(Process { + ppid: None, start: info.ri_proc_start_abstime, exited: info.ri_proc_exit_abstime != 0, + cpu_nanos: mach_to_nanos( + info.ri_user_time + .saturating_add(info.ri_system_time) + .saturating_add(info.ri_child_user_time) + .saturating_add(info.ri_child_system_time), + self.timebase, + ), + resident_bytes: info.ri_resident_size, }) } } - fn short_parent(pid: libc::pid_t) -> Result { - let mut info = ProcBsdShortInfo { - pid: 0, - ppid: 0, - rest: [0; 14], - }; - let size = libc::c_int::try_from(std::mem::size_of::()) - .map_err(|_| ReadError::Denied)?; - // SAFETY: `info` is writable and `size` bytes long, as PROC_PIDT_SHORTBSDINFO needs. - let written = unsafe { - libc::proc_pidinfo(pid, PROC_PIDT_SHORTBSDINFO, 0, (&raw mut info).cast(), size) + /// `proc_pid_rusage` of `pid`. CPU and start times are in Mach absolute + /// time units. + fn usage(pid: u32) -> Result { + let pid = libc::c_int::try_from(pid).map_err(|_| ReadError::Gone)?; + // SAFETY: rusage_info_v2 is plain data; zero is a valid initial value. + let mut info: libc::rusage_info_v2 = unsafe { std::mem::zeroed() }; + // SAFETY: `info` is a writable rusage_info_v2, the layout RUSAGE_INFO_V2 names. + let status = unsafe { + libc::proc_pid_rusage( + pid, + libc::RUSAGE_INFO_V2, + (&raw mut info).cast::(), + ) }; - if written == size { - return Ok(Parent { - ppid: info.ppid, - started: None, - }); + if status == 0 { + return Ok(info); } - Err(last_read_error()) - } - - /// Classify the error of the libproc call that just failed. - fn last_read_error() -> ReadError { match std::io::Error::last_os_error().raw_os_error() { - Some(libc::ESRCH) => ReadError::Gone, - _ => ReadError::Denied, + Some(libc::ESRCH) => Err(ReadError::Gone), + _ => Err(ReadError::Unavailable(PROCESS_UNREADABLE)), } } @@ -614,10 +621,12 @@ mod platform { denom: u32, } - // libc marks its binding deprecated in favour of the `mach2` crate; one - // system call does not justify a new dependency. + // libc marks these bindings deprecated in favour of the `mach2` crate; two + // system calls do not justify a new dependency. unsafe extern "C" { fn mach_timebase_info(info: *mut MachTimebaseInfo) -> libc::c_int; + /// The clock that `ri_proc_start_abstime` uses. + fn mach_absolute_time() -> u64; } /// Nanoseconds per Mach absolute time unit, as `(numer, denom)`, or `None` @@ -640,94 +649,12 @@ mod platform { } pub(super) fn start_identity(pid: u32) -> Option { - Live.usage(pid).ok().map(|usage| usage.start) + usage(pid).ok().map(|info| info.ri_proc_start_abstime) } pub(super) fn read_tree(root: u32, identity: u64) -> Result { - let timebase = timebase().ok_or(reason::PROCESS_UNREADABLE)?; - read_tree_with(&Live, root, identity, timebase) - } - - /// Child PIDs, each with its parent record, by parent PID. - type ChildrenByParent = BTreeMap>; - - /// Every live process's children, each with its parent record. One - /// `proc_listallpids` call and one `proc_pidinfo` per process: - /// `proc_listchildpids` scans the whole process table on each call, which - /// costs too much for a wide tree. A process that exited during the pass - /// is left out; one that cannot be read leaves the listing incomplete. - fn children_by_parent(procs: &impl Libproc) -> Result { - let pids = procs.pids().ok_or(reason::TABLE_UNREADABLE)?; - let mut children = ChildrenByParent::new(); - for pid in pids { - match procs.parent(pid) { - Ok(parent) => children.entry(parent.ppid).or_default().push((pid, parent)), - Err(ReadError::Gone) => {} - Err(ReadError::Denied) => return Err(reason::TABLE_UNREADABLE), - } - } - Ok(children) - } - - /// Read the tree under `root`. Fails when the root exited or its PID now - /// names a process with another start time than `identity`, or when a - /// live process in the tree cannot be read. - pub(super) fn read_tree_with( - procs: &impl Libproc, - root: u32, - identity: u64, - timebase: (u64, u64), - ) -> Result { - let children = children_by_parent(procs)?; - // Read after the listing: a root that is still the same live process - // was alive during the whole listing, so its listed children are its own. - let root_usage = match procs.usage(root) { - Ok(usage) if usage.start == identity && !usage.exited => usage, - Ok(_) | Err(ReadError::Gone) => return Err(reason::HOST_EXITED), - Err(ReadError::Denied) => return Err(reason::PROCESS_UNREADABLE), - }; - let mut read = TreeRead::default(); - let mut cpu_units: u64 = 0; - let mut pending = VecDeque::from([(root, root_usage)]); - while let Some((pid, usage)) = pending.pop_front() { - cpu_units = cpu_units.saturating_add(usage.cpu_units); - read.resident_bytes = read.resident_bytes.saturating_add(usage.resident_bytes); - read.process_count += 1; - for &(child, listed) in children.get(&pid).into_iter().flatten() { - if let Some(usage) = listed_child_usage(procs, child, listed)? { - pending.push_back((child, usage)); - } - } - } - read.cpu_nanos = mach_to_nanos(cpu_units, timebase); - Ok(read) - } - - /// The usage of `child`, or `None` when it exited or its PID no longer - /// names the listed process. A process that still has the listed parent - /// and start time after the usage read was the listed one throughout, so - /// the usage is its own. Fails when the child is live but cannot be read. - fn listed_child_usage( - procs: &impl Libproc, - child: u32, - listed: Parent, - ) -> Result, &'static str> { - if listed.started.is_none() { - return Err(reason::PROCESS_UNREADABLE); - } - let usage = match procs.usage(child) { - Ok(usage) if !usage.exited => usage, - Ok(_) | Err(ReadError::Gone) => return Ok(None), - Err(ReadError::Denied) => return Err(reason::PROCESS_UNREADABLE), - }; - match procs.parent(child) { - Ok(now) if now == listed => Ok(Some(usage)), - Ok(now) if now.ppid == listed.ppid && now.started.is_none() => { - Err(reason::PROCESS_UNREADABLE) - } - Ok(_) | Err(ReadError::Gone) => Ok(None), - Err(ReadError::Denied) => Err(reason::PROCESS_UNREADABLE), - } + let timebase = timebase().ok_or(PROCESS_UNREADABLE)?; + read_tree_with(&Live { timebase }, root, identity) } } @@ -748,6 +675,13 @@ mod platform { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + #[cfg(any(target_os = "linux", target_os = "macos"))] + use std::time::{Duration, Instant}; + + use super::tree::{ + Children, HOST_EXITED, PROCESS_UNREADABLE, Process, ProcessTable, ReadError, read_tree_with, + }; use super::*; #[test] @@ -822,472 +756,329 @@ mod tests { .expect("spawn child"); let host = LocalHostProcess::new(child.id(), SystemTime::now()); child.kill().expect("kill child"); - std::thread::sleep(std::time::Duration::from_millis(200)); - let error = snapshot_local_host("runtime-1", Some(&host)).expect_err("zombie host"); + let deadline = Instant::now() + Duration::from_secs(5); + let mut result = snapshot_local_host("runtime-1", Some(&host)); + while result.is_ok() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + result = snapshot_local_host("runtime-1", Some(&host)); + } child.wait().expect("reap child"); assert!(matches!( - error, - FabricError::ResourceObservationUnavailable { .. } + result, + Err(FabricError::ResourceObservationUnavailable { ref reason, .. }) + if reason == HOST_EXITED )); } - #[cfg(target_os = "linux")] + /// The host is `sleep`, which never waits for children. Before its `exec`, + /// the shell starts a child that burns CPU and exits, so the child stays + /// unreaped under the host. + #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] - fn linux_stat_parse_reads_cpu_with_waited_children_start_time_and_ticks() { - let stat = "42 (a (weird) name) S 7 42 42 0 -1 4194560 100 0 0 0 30 20 5 4 20 0 1 0 \ - 9001 1000000 250 18446744073709551615"; + fn snapshot_local_host_counts_cpu_of_exited_unreaped_child() { + let mut host_process = std::process::Command::new("/bin/sh") + .args([ + "-c", + "(i=0; while [ $i -lt 100000 ]; do i=$((i+1)); done) & exec sleep 30", + ]) + .spawn() + .expect("spawn host"); + let host = LocalHostProcess::new(host_process.id(), SystemTime::now()); + let deadline = Instant::now() + Duration::from_secs(30); + let snapshot_until = |done: fn(u64) -> bool| loop { + let snapshot = snapshot_local_host("runtime-1", Some(&host)).expect("snapshot"); + if done(snapshot.process_count) || Instant::now() > deadline { + return snapshot; + } + std::thread::sleep(Duration::from_millis(20)); + }; - let parsed = platform::parse_stat(stat).expect("stat parses"); + let running = snapshot_until(|count| count >= 2); + let exited = snapshot_until(|count| count == 1); + host_process.kill().expect("kill host"); + host_process.wait().expect("reap host"); - assert_eq!(parsed.ppid, 7); - assert_eq!(parsed.cpu_ticks, 30 + 20 + 5 + 4); - assert_eq!(parsed.start, 9001); - assert_eq!(parsed.rss_pages, 250); - assert!(!parsed.zombie); - assert_eq!(platform::ticks_to_nanos(150, 100), 1_500_000_000); + assert_eq!(running.process_count, 2, "child never seen: {running:?}"); + assert_eq!(exited.process_count, 1, "child still running: {exited:?}"); + assert!( + exited.cpu_time_seconds >= 0.1, + "exited child's CPU missing: {exited:?}" + ); } - /// Scripted `libproc` answers for the macOS tree walk. - #[cfg(target_os = "macos")] + /// When a scripted table read the listed children. + const LISTED_AT: u64 = 100; + /// The scripted host: PID 10, started at time 5. + const HOST: u32 = 10; + const HOST_START: u64 = 5; + + /// Scripted answers for the platform-neutral tree walk. A PID without + /// answers has exited. #[derive(Default)] - struct ScriptedLibproc { - pids: Option>, - /// Answers to `parent`, first the listing's, then the later checks'. - parents: std::collections::BTreeMap< - u32, - std::cell::RefCell>>, - >, - usages: std::collections::BTreeMap< - u32, - std::result::Result, - >, + struct ScriptedTable { + children: BTreeMap, ReadError>>, + reads: BTreeMap>, } - #[cfg(target_os = "macos")] - impl ScriptedLibproc { + impl ScriptedTable { fn process( mut self, pid: u32, - parents: Vec>, - usage: std::result::Result, + children: std::result::Result, ReadError>, + read: std::result::Result, ) -> Self { - self.pids.get_or_insert_with(Vec::new).push(pid); - self.parents.insert(pid, std::cell::RefCell::new(parents)); - self.usages.insert(pid, usage); + self.children.insert(pid, children); + self.reads.insert(pid, read); self } - } - #[cfg(target_os = "macos")] - impl platform::Libproc for ScriptedLibproc { - fn pids(&self) -> Option> { - self.pids.clone() - } - - fn parent(&self, pid: u32) -> std::result::Result { - let mut answers = self.parents.get(&pid).expect("scripted pid").borrow_mut(); - if answers.len() > 1 { - answers.remove(0) - } else { - answers[0] - } - } - - fn usage(&self, pid: u32) -> std::result::Result { - self.usages[&pid] + fn read_host_tree(&self) -> std::result::Result { + read_tree_with(self, HOST, HOST_START) } } - #[cfg(target_os = "macos")] - fn mac_parent(ppid: u32, started: u64) -> platform::Parent { - platform::Parent { - ppid, - started: Some((started, 0)), + impl ProcessTable for ScriptedTable { + fn children(&self, pid: u32) -> std::result::Result { + let pids = self + .children + .get(&pid) + .cloned() + .unwrap_or(Err(ReadError::Gone))?; + Ok(Children { + pids, + listed_at: LISTED_AT, + }) } - } - #[cfg(target_os = "macos")] - fn mac_usage(start: u64, cpu_units: u64) -> platform::Usage { - platform::Usage { - cpu_units, - resident_bytes: 1000, - start, - exited: false, + fn read(&self, pid: u32) -> std::result::Result { + self.reads + .get(&pid) + .copied() + .unwrap_or(Err(ReadError::Gone)) } } - /// The host (PID 10) has one listed child (PID 11). After the listing the - /// child exits and an unrelated, newer process takes PID 11. - #[cfg(target_os = "macos")] - #[test] - fn macos_tree_read_skips_child_pid_reused_after_listing() { - let procs = ScriptedLibproc::default() - .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) - .process( - 11, - vec![Ok(mac_parent(10, 200)), Ok(mac_parent(1, 300))], - Ok(mac_usage(9, 1_000_000)), - ); - - let read = platform::read_tree_with(&procs, 10, 5, (1, 1)).expect("tree read"); - - assert_eq!(read.process_count, 1); - assert_eq!(read.cpu_nanos, 7); + /// How a platform shows that a listed child is still the listed process. + #[derive(Debug, Clone, Copy)] + enum Lineage { + /// Linux: the parent PID in the same record. + ParentPid, + /// macOS: a start time before the list was read. + StartedBefore, } - #[cfg(target_os = "macos")] - #[test] - fn macos_tree_read_counts_child_that_is_still_the_listed_process() { - let procs = ScriptedLibproc::default() - .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) - .process(11, vec![Ok(mac_parent(10, 200))], Ok(mac_usage(9, 3))); - - let read = platform::read_tree_with(&procs, 10, 5, (1, 1)).expect("tree read"); - - assert_eq!(read.process_count, 2); - assert_eq!(read.cpu_nanos, 10); - assert_eq!(read.resident_bytes, 2000); - } - - #[cfg(target_os = "macos")] - fn mac_tree_with_child( - child_parents: Vec>, - child_usage: std::result::Result, - ) -> std::result::Result { - let procs = ScriptedLibproc::default() - .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) - .process(11, child_parents, child_usage); - platform::read_tree_with(&procs, 10, 5, (1, 1)) - } - - #[cfg(target_os = "macos")] - #[test] - fn macos_tree_read_with_unreadable_live_child_is_unavailable() { - let denied_usage = mac_tree_with_child( - vec![Ok(mac_parent(10, 200))], - Err(platform::ReadError::Denied), - ); - let other_user = mac_tree_with_child( - vec![Ok(platform::Parent { - ppid: 10, - started: None, - })], - Ok(mac_usage(9, 3)), - ); - let denied_recheck = mac_tree_with_child( - vec![Ok(mac_parent(10, 200)), Err(platform::ReadError::Denied)], - Ok(mac_usage(9, 3)), - ); - - assert_eq!(denied_usage.unwrap_err(), reason::PROCESS_UNREADABLE); - assert_eq!(other_user.unwrap_err(), reason::PROCESS_UNREADABLE); - assert_eq!(denied_recheck.unwrap_err(), reason::PROCESS_UNREADABLE); - } - - /// The child is listed, then exits before its usage read or before the - /// identity check that follows it. - #[cfg(target_os = "macos")] - #[test] - fn macos_tree_read_skips_child_reaped_between_reads() { - let before_usage = mac_tree_with_child( - vec![Ok(mac_parent(10, 200))], - Err(platform::ReadError::Gone), - ) - .expect("tree read"); - let before_check = mac_tree_with_child( - vec![Ok(mac_parent(10, 200)), Err(platform::ReadError::Gone)], - Ok(mac_usage(9, 3)), - ) - .expect("tree read"); - let exited = mac_tree_with_child( - vec![Ok(mac_parent(10, 200))], - Ok(platform::Usage { - exited: true, - ..mac_usage(9, 3) - }), - ) - .expect("tree read"); - - for read in [before_usage, before_check, exited] { - assert_eq!(read.process_count, 1); - assert_eq!(read.cpu_nanos, 7); - } - } - - #[cfg(target_os = "macos")] - #[test] - fn macos_tree_read_with_incomplete_process_table_is_unavailable() { - let unlisted = ScriptedLibproc::default(); - let unreadable_other = ScriptedLibproc::default() - .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) - .process( - 50, - vec![Err(platform::ReadError::Denied)], - Err(platform::ReadError::Denied), - ); - - assert_eq!( - platform::read_tree_with(&unlisted, 10, 5, (1, 1)).unwrap_err(), - reason::TABLE_UNREADABLE - ); - assert_eq!( - platform::read_tree_with(&unreadable_other, 10, 5, (1, 1)).unwrap_err(), - reason::TABLE_UNREADABLE - ); - } - - #[cfg(target_os = "macos")] - #[test] - fn macos_tree_read_of_exited_or_unreadable_host_is_unavailable() { - let host = |usage| { - let procs = ScriptedLibproc::default().process(10, vec![Ok(mac_parent(1, 100))], usage); - platform::read_tree_with(&procs, 10, 5, (1, 1)).unwrap_err() - }; - - assert_eq!(host(Err(platform::ReadError::Gone)), reason::HOST_EXITED); - assert_eq!(host(Ok(mac_usage(6, 7))), reason::HOST_EXITED); - assert_eq!( - host(Ok(platform::Usage { - exited: true, - ..mac_usage(5, 7) - })), - reason::HOST_EXITED - ); - assert_eq!( - host(Err(platform::ReadError::Denied)), - reason::PROCESS_UNREADABLE - ); - } - - /// Scripted `/proc` answers for the Linux tree walk. - #[cfg(target_os = "linux")] - #[derive(Default)] - struct ScriptedProcfs { - /// Children by parent from children files, or `None` for a kernel - /// without them. - children: Option< - std::collections::BTreeMap, platform::ReadError>>, - >, - pids: Vec, - unlisted: bool, - /// Answers to `stat`, first the listing's, then the walk's. - stats: std::collections::BTreeMap< - u32, - std::cell::RefCell>>, - >, - } - - #[cfg(target_os = "linux")] - impl ScriptedProcfs { - fn process( - mut self, - pid: u32, - stats: Vec>, - ) -> Self { - self.pids.push(pid); - self.stats.insert(pid, std::cell::RefCell::new(stats)); - self - } - } - - #[cfg(target_os = "linux")] - impl platform::Procfs for ScriptedProcfs { - fn has_children_files(&self) -> bool { - self.children.is_some() - } - - fn children(&self, pid: u32) -> std::result::Result, platform::ReadError> { - let children = self.children.as_ref().expect("children files"); - children.get(&pid).cloned().unwrap_or(Ok(Vec::new())) - } - - fn pids(&self) -> std::result::Result, platform::ReadError> { - if self.unlisted { - return Err(platform::ReadError::Unreadable); + impl Lineage { + /// A live process listed under `parent`, with 1000 resident bytes. + fn child(self, parent: u32, cpu_nanos: u64) -> Process { + let (ppid, start) = match self { + Self::ParentPid => (Some(parent), LISTED_AT + 1), + Self::StartedBefore => (None, LISTED_AT - 1), + }; + Process { + ppid, + start, + exited: false, + cpu_nanos, + resident_bytes: 1000, } - Ok(self.pids.clone()) } - fn stat(&self, pid: u32) -> std::result::Result { - let mut answers = self.stats.get(&pid).expect("scripted pid").borrow_mut(); - if answers.len() > 1 { - answers.remove(0) - } else { - answers[0] + /// A process that took the PID of a child of `parent` after the list + /// was read. + fn replacement(self, parent: u32) -> Process { + match self { + Self::ParentPid => Process { + ppid: Some(1), + ..self.child(parent, 900) + }, + Self::StartedBefore => Process { + start: LISTED_AT, + ..self.child(parent, 900) + }, } } - } - #[cfg(target_os = "linux")] - fn linux_stat(ppid: u32, start: u64, cpu_ticks: u64) -> platform::Stat { - platform::Stat { - zombie: false, - ppid, - cpu_ticks, - start, - rss_pages: 1, + fn host(self, cpu_nanos: u64) -> Process { + Process { + start: HOST_START, + ..self.child(1, cpu_nanos) + } } } - /// The host is PID 10 started at tick 5; its child is PID 11. - #[cfg(target_os = "linux")] - fn linux_tree_with_child( - child: Vec>, - ) -> std::result::Result { - let procfs = ScriptedProcfs::default() - .process(10, vec![Ok(linux_stat(1, 5, 7))]) - .process(11, child); - platform::read_tree_with(&procfs, 10, 5, 1, 1) - } + const LINEAGES: [Lineage; 2] = [Lineage::ParentPid, Lineage::StartedBefore]; - #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_with_unreadable_live_child_is_unavailable() { - let read = linux_tree_with_child(vec![ - Ok(linux_stat(10, 9, 3)), - Err(platform::ReadError::Unreadable), - ]); - - assert_eq!(read.unwrap_err(), reason::PROCESS_UNREADABLE); + fn tree_read_counts_live_descendants_and_cpu_of_exited_ones() { + for lineage in LINEAGES { + let table = ScriptedTable::default() + .process(HOST, Ok(vec![11]), Ok(lineage.host(7))) + .process(11, Ok(vec![12, 13]), Ok(lineage.child(HOST, 3))) + .process(12, Ok(vec![]), Ok(lineage.child(11, 2))) + .process( + 13, + Ok(vec![]), + Ok(Process { + exited: true, + ..lineage.child(11, 4) + }), + ); + + let read = table.read_host_tree().expect("tree read"); + + assert_eq!(read.process_count, 3, "{lineage:?}"); + assert_eq!(read.cpu_nanos, 7 + 3 + 2 + 4, "{lineage:?}"); + assert_eq!(read.resident_bytes, 3000, "{lineage:?}"); + } } - #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_skips_child_reaped_or_replaced_between_reads() { - let reaped = linux_tree_with_child(vec![ - Ok(linux_stat(10, 9, 3)), - Err(platform::ReadError::Gone), - ]) - .expect("tree read"); - let replaced = - linux_tree_with_child(vec![Ok(linux_stat(10, 9, 3)), Ok(linux_stat(1, 12, 900))]) - .expect("tree read"); - let counted = linux_tree_with_child(vec![Ok(linux_stat(10, 9, 3))]).expect("tree read"); - - for read in [reaped, replaced] { - assert_eq!(read.process_count, 1); - assert_eq!(read.cpu_nanos, 7_000_000_000); + fn tree_read_skips_child_reaped_or_replaced_after_listing() { + for lineage in LINEAGES { + let with_child = |children, read| { + ScriptedTable::default() + .process(HOST, Ok(vec![11]), Ok(lineage.host(7))) + .process(11, children, read) + .read_host_tree() + .expect("tree read") + }; + let reaped = with_child(Ok(vec![]), Err(ReadError::Gone)); + let reaped_before_its_list = + with_child(Err(ReadError::Gone), Ok(lineage.child(HOST, 3))); + let replaced = with_child(Ok(vec![]), Ok(lineage.replacement(HOST))); + + for read in [reaped, reaped_before_its_list, replaced] { + assert_eq!(read.process_count, 1, "{lineage:?}"); + assert_eq!(read.cpu_nanos, 7, "{lineage:?}"); + } } - assert_eq!(counted.process_count, 2); - assert_eq!(counted.cpu_nanos, 10_000_000_000); } - #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_with_incomplete_process_table_is_unavailable() { - let unlisted = ScriptedProcfs { - unlisted: true, - ..ScriptedProcfs::default() + fn tree_read_with_unreadable_live_child_is_unavailable() { + let with_child = |children, read| { + ScriptedTable::default() + .process(HOST, Ok(vec![11]), Ok(Lineage::ParentPid.host(7))) + .process(11, children, read) + .read_host_tree() }; - let unreadable_other = ScriptedProcfs::default() - .process(10, vec![Ok(linux_stat(1, 5, 7))]) - .process(50, vec![Err(platform::ReadError::Unreadable)]); - assert_eq!( - platform::read_tree_with(&unlisted, 10, 5, 1, 1).unwrap_err(), - reason::TABLE_UNREADABLE - ); - assert_eq!( - platform::read_tree_with(&unreadable_other, 10, 5, 1, 1).unwrap_err(), - reason::TABLE_UNREADABLE + let unreadable = with_child(Ok(vec![]), Err(ReadError::Unavailable(PROCESS_UNREADABLE))); + let unlisted = with_child( + Err(ReadError::Unavailable("children cannot be listed")), + Ok(Lineage::ParentPid.child(HOST, 3)), ); + + assert_eq!(unreadable.unwrap_err(), PROCESS_UNREADABLE); + assert_eq!(unlisted.unwrap_err(), "children cannot be listed"); } - #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_of_exited_or_unreadable_host_is_unavailable() { - let host = |walk| { - let procfs = ScriptedProcfs::default().process(10, vec![Ok(linux_stat(1, 5, 7)), walk]); - platform::read_tree_with(&procfs, 10, 5, 1, 1).unwrap_err() + fn tree_read_of_exited_replaced_or_unreadable_host_is_unavailable() { + let host = |children, read| { + ScriptedTable::default() + .process(HOST, children, read) + .read_host_tree() + .unwrap_err() }; + let live = Lineage::ParentPid.host(7); - assert_eq!(host(Err(platform::ReadError::Gone)), reason::HOST_EXITED); - assert_eq!(host(Ok(linux_stat(1, 6, 7))), reason::HOST_EXITED); + assert_eq!(host(Ok(vec![]), Err(ReadError::Gone)), HOST_EXITED); + assert_eq!(host(Err(ReadError::Gone), Ok(live)), HOST_EXITED); assert_eq!( - host(Ok(platform::Stat { - zombie: true, - ..linux_stat(1, 5, 7) - })), - reason::HOST_EXITED + host( + Ok(vec![]), + Ok(Process { + start: HOST_START + 1, + ..live + }) + ), + HOST_EXITED ); assert_eq!( - host(Err(platform::ReadError::Unreadable)), - reason::PROCESS_UNREADABLE + host( + Ok(vec![]), + Ok(Process { + exited: true, + ..live + }) + ), + HOST_EXITED + ); + assert_eq!( + host(Ok(vec![]), Err(ReadError::Unavailable(PROCESS_UNREADABLE))), + PROCESS_UNREADABLE ); - } - - /// A kernel with children files: the host is PID 10 started at tick 5 and - /// lists child PID 11. The process table is never listed. - #[cfg(target_os = "linux")] - fn linux_children_tree( - host_children: std::result::Result, platform::ReadError>, - child: std::result::Result, - ) -> std::result::Result { - let mut procfs = ScriptedProcfs { - children: Some(std::collections::BTreeMap::from([(10, host_children)])), - unlisted: true, - ..ScriptedProcfs::default() - } - .process(10, vec![Ok(linux_stat(1, 5, 7))]) - .process(11, vec![child]); - procfs.pids.clear(); - platform::read_tree_with(&procfs, 10, 5, 1, 1) } #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_with_children_files_reads_only_the_tree() { - let read = linux_children_tree(Ok(vec![11]), Ok(linux_stat(10, 9, 3))).expect("tree read"); + fn linux_stat_parse_reads_cpu_with_waited_children_start_time_and_ticks() { + let stat = "42 (a (weird) name) S 7 42 42 0 -1 4194560 100 0 0 0 30 20 5 4 20 0 1 0 \ + 9001 1000000 250 18446744073709551615"; + + let parsed = platform::parse_stat(stat).expect("stat parses"); - assert_eq!(read.process_count, 2); - assert_eq!(read.cpu_nanos, 10_000_000_000); - assert_eq!(read.resident_bytes, 2); + assert_eq!(parsed.ppid, 7); + assert_eq!(parsed.cpu_ticks, 30 + 20 + 5 + 4); + assert_eq!(parsed.start, 9001); + assert_eq!(parsed.rss_pages, 250); + assert!(!parsed.zombie); + assert_eq!(platform::ticks_to_nanos(150, 100), 1_500_000_000); } + /// The second read finds a child the first skipped; the third agrees. #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_with_children_files_skips_reaped_or_replaced_child() { - let reaped = - linux_children_tree(Ok(vec![11]), Err(platform::ReadError::Gone)).expect("tree read"); - let replaced = - linux_children_tree(Ok(vec![11]), Ok(linux_stat(1, 12, 900))).expect("tree read"); - - for read in [reaped, replaced] { - assert_eq!(read.process_count, 1); - assert_eq!(read.cpu_nanos, 7_000_000_000); - } + fn linux_children_read_waits_for_two_reads_that_agree() { + let mut reads = vec![vec![11, 13], vec![11, 12, 13], vec![11, 12, 13]].into_iter(); + let mut changing = (0..).map(|n| vec![n]); + + let stable = platform::read_until_stable(|| Ok(reads.next().expect("read"))); + let unstable = platform::read_until_stable(|| Ok(changing.next().expect("read"))); + + assert_eq!(stable, Ok(vec![11, 12, 13])); + assert_eq!( + unstable, + Err(ReadError::Unavailable(platform::TREE_CHANGING)) + ); } #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_with_children_files_and_unreadable_process_is_unavailable() { - let child = linux_children_tree(Ok(vec![11]), Err(platform::ReadError::Unreadable)); - let host_children = linux_children_tree( - Err(platform::ReadError::Unreadable), - Ok(linux_stat(10, 9, 3)), - ); - let host_gone = - linux_children_tree(Err(platform::ReadError::Gone), Ok(linux_stat(10, 9, 3))); + fn linux_process_scan_groups_children_and_needs_every_parent() { + let parents = BTreeMap::from([ + (10, Ok(1)), + (11, Ok(10)), + (12, Ok(10)), + (13, Err(ReadError::Gone)), + ]); + let parent_of = |pid| parents[&pid]; + + let scanned = platform::children_by_parent(vec![10, 11, 12, 13], parent_of).expect("scan"); + let incomplete = platform::children_by_parent(vec![10, 50], |pid| match pid { + 50 => Err(ReadError::Unavailable(PROCESS_UNREADABLE)), + _ => Ok(1), + }); - assert_eq!(child.unwrap_err(), reason::PROCESS_UNREADABLE); - assert_eq!(host_children.unwrap_err(), reason::PROCESS_UNREADABLE); - assert_eq!(host_gone.unwrap_err(), reason::HOST_EXITED); + assert_eq!(scanned, BTreeMap::from([(1, vec![10]), (10, vec![11, 12])])); + assert_eq!(incomplete.unwrap_err(), platform::TABLE_UNREADABLE); } - /// The second read finds a child the first skipped; the third agrees. + /// The fallback for a kernel without children files reads the tree too. #[cfg(target_os = "linux")] #[test] - fn linux_children_read_waits_for_two_reads_that_agree() { - let mut reads = vec![vec![11, 13], vec![11, 12, 13], vec![11, 12, 13]].into_iter(); - let mut changing = (0..).map(|n| vec![n]); + fn linux_tree_read_with_process_scan_reads_this_process_tree() { + let host = LocalHostProcess::new(std::process::id(), SystemTime::now()); + let children = platform::scan_children().expect("scan"); - let stable = platform::read_until_stable(|| Ok(reads.next().expect("read"))); - let unstable = platform::read_until_stable(|| Ok(changing.next().expect("read"))); + let read = platform::read_tree_from(children, host.pid, host.identity.expect("identity")) + .expect("tree read"); - assert_eq!(stable, Ok(vec![11, 12, 13])); - assert_eq!(unstable, Err(platform::ReadError::Unreadable)); + assert!(read.process_count >= 1); + assert!(read.resident_bytes > 0); } #[cfg(target_os = "macos")] diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index ae9926b55..cec4f2607 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -698,20 +698,20 @@ print(snapshot.process_count, snapshot.cpu_time_seconds) | `observed_at` | When NeMo Fabric read the values, as an RFC 3339 UTC timestamp. | | `resident_memory_bytes` | Sum of the resident set sizes of the live processes. Pages shared between processes count once per process. | | `process_count` | Live processes, including the adapter host. Exited processes that are not yet reaped are not counted. | -| `cpu_time_seconds` | CPU time used since the adapter host started, including exited descendants (on macOS, only once they are waited for). It can drop when a descendant leaves the tree. | +| `cpu_time_seconds` | CPU time used since the adapter host started, including exited descendants. It can drop when a descendant leaves the tree. | The scope is fixed: the local adapter host process that NeMo Fabric started for the runtime, plus its descendants. For a runtime started on a shared service, the service process is not included. A descendant that starts a new session or process group still counts while its parent chain leads to the host. A process whose parent exited is reparented outside the tree and no longer counts. The -snapshot reads `/proc` on Linux and `proc_pid_rusage` on macOS. On Linux it -finds the tree from each process's `/proc//task//children` list, so it -reads only the runtime's own processes. On macOS, and on a Linux kernel without -those lists, it reads the parent ID of every process on the machine to find the -tree, and macOS also reads each process's start time to detect a reused PID. -The snapshot exposes only the fields above, never command lines, environment -variables, credentials, or processes outside the runtime's tree. +snapshot finds the tree from each process's list of children: Linux reads +`/proc//task//children` and `/proc//stat`, and macOS reads +`proc_listchildpids` and `proc_pid_rusage`. Both read only the runtime's own +processes. On a Linux kernel without those children lists, the snapshot reads +the parent ID of every process on the machine to find the tree. The snapshot +exposes only the fields above, never command lines, environment variables, +credentials, or processes outside the runtime's tree. When NeMo Fabric cannot read the values, it raises an error instead of returning zeros or leaving a process out: @@ -728,8 +728,8 @@ once after the adapter returned the result. It is `None` when the read failed; a failed read never fails the invocation. To estimate one invocation's CPU time, take a snapshot before it and subtract its `cpu_time_seconds` from the result's snapshot. CPU time can drop when a descendant is reparented out of the -tree, and on macOS while an exited descendant is not yet waited for, so treat a -difference as an estimate and a negative difference as unknown. +tree, so treat a difference as an estimate and a negative difference as +unknown. Snapshots observe resources only. They do not enforce limits, stop runtimes, or queue requests. Use the operating system or execution environment, such as diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 072be7097..504d6b129 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -283,7 +283,7 @@ "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", "properties": { "cpu_time_seconds": { - "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants (on macOS, only once they are waited for).\nIt can drop when a descendant leaves the tree, and on macOS while an\nexited descendant is not yet waited for.", + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", "format": "double", "minimum": 0.0, "type": "number" diff --git a/schemas/sdk/runtime-resource-snapshot.schema.json b/schemas/sdk/runtime-resource-snapshot.schema.json index 8a5bf94ad..0b3ac2916 100644 --- a/schemas/sdk/runtime-resource-snapshot.schema.json +++ b/schemas/sdk/runtime-resource-snapshot.schema.json @@ -3,7 +3,7 @@ "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", "properties": { "cpu_time_seconds": { - "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants (on macOS, only once they are waited for).\nIt can drop when a descendant leaves the tree, and on macOS while an\nexited descendant is not yet waited for.", + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", "format": "double", "minimum": 0.0, "type": "number" diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index c1ddf0323..c6a3cb682 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1849,8 +1849,8 @@ class RuntimeResourceSnapshot(FabricMapping): Pages shared between processes count once per process. process_count: Live processes in the tree, including the adapter host. cpu_time_seconds: CPU time used since the adapter host started, - including exited descendants (on macOS, only once they are - waited for). It can drop when a descendant leaves the tree. + including exited descendants. It can drop when a descendant + leaves the tree. """ runtime_id: str From ced92e81553734430c18bade5c59352aa13f408f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:08:54 -0700 Subject: [PATCH 011/102] docs: explain unavailable and unsupported resource snapshots On macOS, a descendant that runs as another user, including the setuid programs ps, top, and sudo, cannot be read, so the snapshot is unavailable while it runs. The docs now name those programs. Unsupported is also returned for the http and native_plugin adapter kinds, which the error docs left out. RunResult.resource_snapshot is None when the final read fails, and the docs now say that the snapshot call returns the error behind it. The SDK API inventory lists Runtime.resource_snapshot(). Signed-off-by: mnajafian-nv --- crates/fabric-core/src/error.rs | 3 ++- crates/fabric-core/src/runtime.rs | 7 ++++--- crates/fabric-python/src/lib.rs | 2 +- docs/sdk/python.mdx | 15 +++++++++------ schemas/sdk/run-result.schema.json | 2 +- .../nemo-fabric-runtime/src/nemo_fabric/errors.py | 6 +++++- .../src/nemo_fabric/runtime.py | 5 +++-- .../nemo-fabric-runtime/src/nemo_fabric/types.py | 3 ++- .../references/results-and-errors.md | 4 ++-- .../references/sdk-api-inventory.md | 1 + 10 files changed, 30 insertions(+), 18 deletions(-) diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index fb4075af0..2665c8a61 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -287,7 +287,8 @@ pub enum FabricError { /// Why the runtime cannot be observed. reason: String, }, - /// The runtime's environment provider or operating system has no resource reader. + /// The runtime's adapter kind, environment provider, or operating system has + /// no resource reader. #[error("resource observation is unsupported for runtime `{runtime_id}`: {reason}")] ResourceObservationUnsupported { /// Runtime handle id. diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 589cdbd78..2ea389429 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -142,7 +142,8 @@ pub struct RunResult { #[serde(default, skip_serializing_if = "Option::is_none")] pub usage: Option, /// Resource snapshot of the runtime, read once after the adapter returned this - /// result. Absent when the runtime's resources could not be observed. + /// result. Absent when the runtime's resources could not be observed; + /// `runtime_resource_snapshot` returns the error that explains it. #[serde(default, skip_serializing_if = "Option::is_none")] pub resource_snapshot: Option, /// Artifacts produced or collected by this run. @@ -1179,8 +1180,8 @@ pub fn invoke_runtime( /// [`FabricError::ResourceObservationUnavailable`] when the runtime stopped, is /// stopping, its host exited, or a live process in its tree or the process /// table cannot be read, and -/// [`FabricError::ResourceObservationUnsupported`] when the environment provider -/// or operating system has no resource reader. +/// [`FabricError::ResourceObservationUnsupported`] when the adapter kind, +/// environment provider, or operating system has no resource reader. pub fn runtime_resource_snapshot( plan: &RunPlan, runtime: &RuntimeHandle, diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index c068a184a..0d65c3d0b 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -34,7 +34,7 @@ pyo3::create_exception!( _native, ResourceObservationUnsupportedError, PyRuntimeError, - "Raised when a runtime's environment provider or operating system has no resource reader." + "Raised when a runtime's adapter kind, environment provider, or operating system has no resource reader." ); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index cec4f2607..21a34673f 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -718,14 +718,17 @@ returning zeros or leaving a process out: - `ResourceObservationUnavailableError` when the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the - process table cannot be read, for example a macOS descendant that runs as - another user. -- `ResourceObservationUnsupportedError` when the runtime's environment provider - or operating system has no resource reader. Windows raises this error. + process table cannot be read. For example, on macOS, the snapshot is + unavailable while a descendant runs as another user, including setuid + programs such as `ps`, `top`, and `sudo`. +- `ResourceObservationUnsupportedError` when the runtime's adapter kind, + environment provider, or operating system has no resource reader. The `http` + and `native_plugin` adapter kinds and Windows raise this error. Each `RunResult` from a local runtime also carries `resource_snapshot`, read once after the adapter returned the result. It is `None` when the read failed; -a failed read never fails the invocation. To estimate one invocation's CPU time, +a failed read never fails the invocation, and `Runtime.resource_snapshot()` +raises the error that explains it. To estimate one invocation's CPU time, take a snapshot before it and subtract its `cpu_time_seconds` from the result's snapshot. CPU time can drop when a descendant is reparented out of the tree, so treat a difference as an estimate and a negative difference as @@ -1121,7 +1124,7 @@ All public SDK errors inherit from `FabricError`. | `FabricStateError` | Invalid runtime state transition, such as invoking after stop or starting overlapping invocations. | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | | `ResourceObservationUnavailableError` | `Runtime.resource_snapshot()` cannot read the runtime because it stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read. Subclass of `FabricRuntimeError`. | -| `ResourceObservationUnsupportedError` | The runtime's environment provider or operating system has no resource reader. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnsupportedError` | The runtime's adapter kind, environment provider, or operating system has no resource reader. Subclass of `FabricRuntimeError`. | Consumers own job-level retries and rollout-level failure policy. Core host-operation deadlines (`host_timeout`) become Python `TimeoutError`, then diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 504d6b129..78d09c973 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -415,7 +415,7 @@ "type": "null" } ], - "description": "Resource snapshot of the runtime, read once after the adapter returned this\nresult. Absent when the runtime's resources could not be observed." + "description": "Resource snapshot of the runtime, read once after the adapter returned this\nresult. Absent when the runtime's resources could not be observed;\n`runtime_resource_snapshot` returns the error that explains it." }, "runtime_id": { "description": "Runtime handle id.", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index 6f6b8e0c9..85671e577 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -90,4 +90,8 @@ class ResourceObservationUnavailableError(FabricRuntimeError): class ResourceObservationUnsupportedError(FabricRuntimeError): - """The runtime's environment provider or operating system has no resource reader.""" + """Runtime resources have no reader. + + The runtime's adapter kind, environment provider, or operating system has + no resource reader. + """ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index a421eb3ff..723084da0 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -751,8 +751,9 @@ async def resource_snapshot(self) -> RuntimeResourceSnapshot: ResourceObservationUnavailableError: If the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. - ResourceObservationUnsupportedError: If the runtime's environment - provider or operating system has no resource reader. + ResourceObservationUnsupportedError: If the runtime's adapter + kind, environment provider, or operating system has no + resource reader. FabricNativeUnavailableError: If the native extension is missing. FabricRuntimeError: If the native snapshot call fails otherwise. """ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index c6a3cb682..f98c4e4f8 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1921,7 +1921,8 @@ class RunResult(FabricMapping): usage: Normalized invocation usage, or ``None`` when unavailable. resource_snapshot: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's - resources could not be observed. + resources could not be observed. ``Runtime.resource_snapshot()`` + raises the error that explains a ``None``. artifacts: Normalized artifact manifest. telemetry: Ordered telemetry references. events: Ordered lifecycle and invocation events. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 2500fa488..93321c87d 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -17,7 +17,7 @@ Every invocation that reaches the adapter boundary returns a normalized | `error` | Structured failure metadata when available. Branch on `status`, not on this field. | | `output` | Harness output normalized to the configured output schema. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | -| `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed. | +| `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed; `runtime.resource_snapshot()` raises the error that explains it. | | `artifacts` | Output files, logs, patches, and other materialized references. | | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | @@ -84,7 +84,7 @@ Refer to the [errors reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/ | `FabricStateError` | Invalid runtime state transition (invoking after stop, overlapping invocations). | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | | `ResourceObservationUnavailableError` | A resource snapshot cannot be read because the runtime stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read. Subclass of `FabricRuntimeError`. | -| `ResourceObservationUnsupportedError` | The runtime's environment provider or operating system has no resource reader. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnsupportedError` | The runtime's adapter kind, environment provider, or operating system has no resource reader. Subclass of `FabricRuntimeError`. | ## Cleanup And Resilience diff --git a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md index 3e2f09c99..b184af10d 100644 --- a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md +++ b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md @@ -47,6 +47,7 @@ The following table lists the `Runtime` members for driving a stateful runtime. | `invoke(*, input=... \| request=...)` | Yes | One turn on an active runtime. One active invocation at a time; overlap raises `FabricStateError`. | | `invoke_openai_stream(*, input=... \| request=...)` | No | Start exactly one descriptor-gated native invocation and return an async `OpenAIInvokeStream` of `chat.completion.chunk` mappings. Await `stream.result()` for the separate terminal `RunResult`. | | `invoke_stream(*, input=... \| request=...)` | No | Start one NeMo Relay turn and return an async `InvokeStream` of raw ATOF records. Await `stream.result()` for the terminal `RunResult`. | +| `resource_snapshot()` | Yes | Timestamped memory, process count, and CPU time of the local adapter host and its descendants. Does not wait for an active invocation. | | `stop()` | Yes | Stop the runtime. Called automatically by `async with`. | | `status` | No | `RuntimeStatus`: `ACTIVE`, `STOPPED`, or `FAILED`. | | `supports_openai_streaming` | No | `True` when the selected descriptor declares `capabilities.streaming` for native OpenAI Chat Completions chunks. | From 8e52556e491f97940061855be8eb2323e1aec47c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:08:55 -0700 Subject: [PATCH 012/102] test: name the snapshot mapping test for both checks it makes The test checks that a snapshot mapping round-trips and that invalid fields raise FabricConfigError, so its name now says both. Signed-off-by: mnajafian-nv --- tests/python/test_runtime_resource_snapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/test_runtime_resource_snapshot.py b/tests/python/test_runtime_resource_snapshot.py index db63e546a..d14f4354a 100644 --- a/tests/python/test_runtime_resource_snapshot.py +++ b/tests/python/test_runtime_resource_snapshot.py @@ -91,7 +91,7 @@ def _runtime_with_native(native_module: Any) -> Runtime: ) -def test_runtime_resource_snapshot_with_invalid_field_raises_config_error(): +def test_runtime_resource_snapshot_round_trips_and_rejects_invalid_fields(): snapshot = RuntimeResourceSnapshot.from_mapping(_SNAPSHOT) assert snapshot.cpu_time_seconds == 1.25 assert snapshot.to_mapping() == _SNAPSHOT From 2e9cfed8502b2e5fbf9234ba1eccd237f2a13836 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:09:33 -0700 Subject: [PATCH 013/102] docs: regenerate API references for resource snapshot docs Picks up the macOS reader, Unsupported adapter kind, and RunResult snapshot wording from the Rust and Python docstrings. Signed-off-by: mnajafian-nv --- docs/reference/api/python-library-reference/index.md | 2 +- .../api/python-library-reference/nemo_fabric.errors.md | 4 +++- .../api/python-library-reference/nemo_fabric.runtime.md | 2 +- .../api/python-library-reference/nemo_fabric.types.md | 4 ++-- .../nemo-fabric-core/error/enum-fabricerror.mdx | 2 +- .../nemo-fabric-core/resource-snapshot/index.mdx | 6 +++--- .../resource-snapshot/struct-runtimeresourcesnapshot.mdx | 2 +- .../runtime/fn-runtime-resource-snapshot.mdx | 2 +- .../nemo-fabric-core/runtime/struct-runresult.mdx | 2 +- 9 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index 24fb1d403..3ab015d41 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -86,7 +86,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`errors.FabricRuntimeError`](./nemo_fabric.errors.md#class-fabricruntimeerror): Failure while starting, invoking, stopping, or otherwise driving a runtime. - [`errors.FabricStateError`](./nemo_fabric.errors.md#class-fabricstateerror): Operation rejected because a local runtime is in the wrong state. - [`errors.ResourceObservationUnavailableError`](./nemo_fabric.errors.md#class-resourceobservationunavailableerror): Runtime resources cannot be read now. -- [`errors.ResourceObservationUnsupportedError`](./nemo_fabric.errors.md#class-resourceobservationunsupportederror): The runtime's environment provider or operating system has no resource reader. +- [`errors.ResourceObservationUnsupportedError`](./nemo_fabric.errors.md#class-resourceobservationunsupportederror): Runtime resources have no reader. ## Functions diff --git a/docs/reference/api/python-library-reference/nemo_fabric.errors.md b/docs/reference/api/python-library-reference/nemo_fabric.errors.md index e3acafc12..8acf76f75 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.errors.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.errors.md @@ -321,7 +321,9 @@ Initialize a structured NeMo Fabric exception. ## class `ResourceObservationUnsupportedError` -The runtime's environment provider or operating system has no resource reader. +Runtime resources have no reader. + +The runtime's adapter kind, environment provider, or operating system has no resource reader. ### Inheritance diff --git a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md index 98ed49750..73fc8cd5b 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md @@ -198,7 +198,7 @@ The snapshot covers the local adapter host process and its live descendants. It **Raises:** - `ResourceObservationUnavailableError`: If the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. - - `ResourceObservationUnsupportedError`: If the runtime's environment provider or operating system has no resource reader. + - `ResourceObservationUnsupportedError`: If the runtime's adapter kind, environment provider, or operating system has no resource reader. - `FabricNativeUnavailableError`: If the native extension is missing. - `FabricRuntimeError`: If the native snapshot call fails otherwise. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 623c34593..f009db3cf 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1259,7 +1259,7 @@ Memory, process count, and CPU time cover the adapter host process and its live - `observed_at`: When Fabric read these values, as RFC 3339 UTC. - `resident_memory_bytes`: Sum of the live processes' resident set sizes. Pages shared between processes count once per process. - `process_count`: Live processes in the tree, including the adapter host. - - `cpu_time_seconds`: CPU time used since the adapter host started, including exited descendants (on macOS, only once they are waited for). It can drop when a descendant leaves the tree. + - `cpu_time_seconds`: CPU time used since the adapter host started, including exited descendants. It can drop when a descendant leaves the tree. ### Fields @@ -1348,7 +1348,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - `error`: Structured failure, or ``None`` on success. - `usage`: Normalized invocation usage, or ``None`` when unavailable. - - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. + - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. ``Runtime.resource_snapshot()`` raises the error that explains a ``None``. - `artifacts`: Normalized artifact manifest. - `telemetry`: Ordered telemetry references. - `events`: Ordered lifecycle and invocation events. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index d4082937d..805b8e7f7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -517,7 +517,7 @@ Why the runtime cannot be observed.
-The runtime's environment provider or operating system has no resource reader. +The runtime's adapter kind, environment provider, or operating system has no resource reader. #### Fields diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index eefcd4bb4..2ba0dad07 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -12,11 +12,11 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Resource snapshots of a runtime's local adapter host and its descendants. -A snapshot reads the process tree rooted at the adapter host process that NeMo Fabric started for a runtime. Each read visits a parent before its children. CPU time adds every process's own CPU time and the CPU time of the children it has already waited for, so work done by finished descendants still counts. A child reaped between the read of its parent and its own read is missed by that one snapshot and is never counted twice. CPU time can drop between snapshots when a descendant is reparented out of the tree, and on macOS while an exited descendant is not yet waited for, because macOS cannot list an unreaped process. +A snapshot reads the process tree rooted at the adapter host process that NeMo Fabric started for a runtime, each parent before its children. CPU time adds every process's own CPU time and the CPU time of the children it has already waited for. An exited child that is not yet waited for counts through its own CPU time, so work done by finished descendants still counts. A child reaped between the read of its parent and its own read is missed by that one snapshot and is never counted twice. CPU time can drop between snapshots when a descendant is reparented out of the tree. -A live process in the tree that cannot be read, or a process table that cannot be listed in full, makes the snapshot unavailable instead of leaving the process out. +Linux reads `/proc//task//children` and `/proc//stat`. On a kernel without those children files, it reads every process's parent ID once per snapshot to find the tree. macOS reads `proc_listchildpids` and `proc_pid_rusage`. A snapshot exposes only its own fields: never command lines, environment variables, open files, or processes outside the tree. -Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. To find the tree, the macOS reader, and the Linux reader on a kernel without `/proc//task//children`, also read other processes' parent IDs. A snapshot exposes only its own fields: never command lines, environment variables, open files, or processes outside the tree. +A live process in the tree that cannot be read, or a process table that the Linux fallback cannot read in full, makes the snapshot unavailable instead of leaving the process out. ## Structs diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx index ba0517771..aff244954 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx @@ -40,7 +40,7 @@ Live processes in the tree, including the adapter host. Exited processes that ar ### `cpu_time_seconds: f64` -CPU time consumed by the tree since the adapter host started, in seconds, including exited descendants (on macOS, only once they are waited for). It can drop when a descendant leaves the tree, and on macOS while an exited descendant is not yet waited for. +CPU time consumed by the tree since the adapter host started, in seconds, including exited descendants. It can drop when a descendant leaves the tree. ## Trait Implementations diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx index 745457bd3..1bf68862e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx @@ -13,4 +13,4 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Read a timestamped resource snapshot of a started runtime. -The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, or a live process in its tree or the process table cannot be read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the environment provider or operating system has no resource reader. +The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, or a live process in its tree or the process table cannot be read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the adapter kind, environment provider, or operating system has no resource reader. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 3e05964a5..e8aba8ae4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -61,7 +61,7 @@ Normalized invocation usage when reported by the adapter target. ### `resource_snapshot: Option` -Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed. +Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed; `runtime_resource_snapshot` returns the error that explains it. ### `artifacts: ArtifactManifest` From e46f641b9eff15fd5b931186e5a5a3360c2d5da8 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:58:42 -0700 Subject: [PATCH 014/102] docs: stop saying a later snapshot explains a missing result snapshot RunResult.resource_snapshot docs said that runtime_resource_snapshot returns the error behind a None. It does not: the final read's error is not kept, run_plan stops the runtime before a caller can ask, and a transient cause can be gone by the next read. The field now says only that it is absent when the runtime's resources could not be observed. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 3 +-- docs/sdk/python.mdx | 3 +-- schemas/sdk/run-result.schema.json | 2 +- sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py | 3 +-- skills/nemo-fabric-integrate/references/results-and-errors.md | 2 +- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 2ea389429..ec9daf329 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -142,8 +142,7 @@ pub struct RunResult { #[serde(default, skip_serializing_if = "Option::is_none")] pub usage: Option, /// Resource snapshot of the runtime, read once after the adapter returned this - /// result. Absent when the runtime's resources could not be observed; - /// `runtime_resource_snapshot` returns the error that explains it. + /// result. Absent when the runtime's resources could not be observed. #[serde(default, skip_serializing_if = "Option::is_none")] pub resource_snapshot: Option, /// Artifacts produced or collected by this run. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 21a34673f..829ab85e5 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -727,8 +727,7 @@ returning zeros or leaving a process out: Each `RunResult` from a local runtime also carries `resource_snapshot`, read once after the adapter returned the result. It is `None` when the read failed; -a failed read never fails the invocation, and `Runtime.resource_snapshot()` -raises the error that explains it. To estimate one invocation's CPU time, +a failed read never fails the invocation. To estimate one invocation's CPU time, take a snapshot before it and subtract its `cpu_time_seconds` from the result's snapshot. CPU time can drop when a descendant is reparented out of the tree, so treat a difference as an estimate and a negative difference as diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 78d09c973..504d6b129 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -415,7 +415,7 @@ "type": "null" } ], - "description": "Resource snapshot of the runtime, read once after the adapter returned this\nresult. Absent when the runtime's resources could not be observed;\n`runtime_resource_snapshot` returns the error that explains it." + "description": "Resource snapshot of the runtime, read once after the adapter returned this\nresult. Absent when the runtime's resources could not be observed." }, "runtime_id": { "description": "Runtime handle id.", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index f98c4e4f8..c6a3cb682 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1921,8 +1921,7 @@ class RunResult(FabricMapping): usage: Normalized invocation usage, or ``None`` when unavailable. resource_snapshot: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's - resources could not be observed. ``Runtime.resource_snapshot()`` - raises the error that explains a ``None``. + resources could not be observed. artifacts: Normalized artifact manifest. telemetry: Ordered telemetry references. events: Ordered lifecycle and invocation events. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 93321c87d..def96b269 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -17,7 +17,7 @@ Every invocation that reaches the adapter boundary returns a normalized | `error` | Structured failure metadata when available. Branch on `status`, not on this field. | | `output` | Harness output normalized to the configured output schema. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | -| `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed; `runtime.resource_snapshot()` raises the error that explains it. | +| `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed. | | `artifacts` | Output files, logs, patches, and other materialized references. | | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | From f4cba230e693d497ff38b82f2e655f0c392d29c9 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:58:50 -0700 Subject: [PATCH 015/102] docs: list a changing process tree and hidepid among snapshot limits The Linux reader returns ResourceObservationUnavailable when 4 reads of a process's children never give two equal results in a row, because the runtime's process tree kept changing during the read. The public error docs listed only a stopped runtime, an exited host, and unreadable processes. They now list the changing tree too. On Linux with /proc mounted hidepid=invisible, another user's descendant reads as gone, so the snapshot leaves it out instead of reporting it unavailable. The module doc and the Python guide now say so. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/error.rs | 5 +++-- crates/fabric-core/src/resource_snapshot.rs | 4 +++- crates/fabric-core/src/runtime.rs | 4 ++-- crates/fabric-python/src/lib.rs | 2 +- docs/sdk/python.mdx | 13 ++++++++----- .../nemo-fabric-runtime/src/nemo_fabric/errors.py | 5 +++-- .../nemo-fabric-runtime/src/nemo_fabric/runtime.py | 5 +++-- .../references/results-and-errors.md | 2 +- 8 files changed, 24 insertions(+), 16 deletions(-) diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index 2665c8a61..c8bda1957 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -278,8 +278,9 @@ pub enum FabricError { adapter_kind: AdapterKind, }, /// A runtime's resources cannot be read now: the runtime stopped, is - /// stopping, its adapter host process exited, or a live process in its - /// tree or the process table cannot be read. + /// stopping, its adapter host process exited, a live process in its tree + /// or the process table cannot be read, or the process tree kept changing + /// during the read. #[error("resource observation is unavailable for runtime `{runtime_id}`: {reason}")] ResourceObservationUnavailable { /// Runtime handle id. diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 3c4582cb6..09175db45 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -20,7 +20,9 @@ //! //! A live process in the tree that cannot be read, or a process table that //! the Linux fallback cannot read in full, makes the snapshot unavailable -//! instead of leaving the process out. +//! instead of leaving the process out. On Linux with `/proc` mounted +//! `hidepid=invisible`, another user's descendant is invisible and is not +//! counted. use std::time::{SystemTime, UNIX_EPOCH}; diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index ec9daf329..b4039f16b 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1177,8 +1177,8 @@ pub fn invoke_runtime( /// The snapshot covers the runtime's local adapter host process and its live /// descendants. It does not wait for a running invocation. It returns /// [`FabricError::ResourceObservationUnavailable`] when the runtime stopped, is -/// stopping, its host exited, or a live process in its tree or the process -/// table cannot be read, and +/// stopping, its host exited, a live process in its tree or the process table +/// cannot be read, or the process tree kept changing during the read, and /// [`FabricError::ResourceObservationUnsupported`] when the adapter kind, /// environment provider, or operating system has no resource reader. pub fn runtime_resource_snapshot( diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 0d65c3d0b..780ae2f08 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -28,7 +28,7 @@ pyo3::create_exception!( _native, ResourceObservationUnavailableError, PyRuntimeError, - "Raised when a runtime stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read." + "Raised when a runtime stopped, is stopping, its adapter host exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read." ); pyo3::create_exception!( _native, diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 829ab85e5..13b5b4d38 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -717,14 +717,17 @@ When NeMo Fabric cannot read the values, it raises an error instead of returning zeros or leaving a process out: - `ResourceObservationUnavailableError` when the runtime stopped, is stopping, - its adapter host process exited, or a live process in its tree or the - process table cannot be read. For example, on macOS, the snapshot is - unavailable while a descendant runs as another user, including setuid - programs such as `ps`, `top`, and `sudo`. + its adapter host process exited, a live process in its tree or the process + table cannot be read, or the process tree kept changing during the read. For + example, on macOS, the snapshot is unavailable while a descendant runs as + another user, including setuid programs such as `ps`, `top`, and `sudo`. - `ResourceObservationUnsupportedError` when the runtime's adapter kind, environment provider, or operating system has no resource reader. The `http` and `native_plugin` adapter kinds and Windows raise this error. +On Linux with `/proc` mounted `hidepid=invisible`, a descendant owned by +another user is invisible and is not counted. + Each `RunResult` from a local runtime also carries `resource_snapshot`, read once after the adapter returned the result. It is `None` when the read failed; a failed read never fails the invocation. To estimate one invocation's CPU time, @@ -1122,7 +1125,7 @@ All public SDK errors inherit from `FabricError`. | `FabricRuntimeError` | Runtime startup, invocation, or shutdown failed before a normalized result could be returned. | | `FabricStateError` | Invalid runtime state transition, such as invoking after stop or starting overlapping invocations. | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | -| `ResourceObservationUnavailableError` | `Runtime.resource_snapshot()` cannot read the runtime because it stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnavailableError` | `Runtime.resource_snapshot()` cannot read the runtime because it stopped, is stopping, its adapter host exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read. Subclass of `FabricRuntimeError`. | | `ResourceObservationUnsupportedError` | The runtime's adapter kind, environment provider, or operating system has no resource reader. Subclass of `FabricRuntimeError`. | Consumers own job-level retries and rollout-level failure policy. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index 85671e577..fae1fdfb0 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -84,8 +84,9 @@ class FabricNativeUnavailableError(FabricRuntimeError): class ResourceObservationUnavailableError(FabricRuntimeError): """Runtime resources cannot be read now. - The runtime stopped, is stopping, its adapter host process exited, or a - live process in its tree or the process table cannot be read. + The runtime stopped, is stopping, its adapter host process exited, a live + process in its tree or the process table cannot be read, or the process + tree kept changing during the read. """ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 723084da0..dc01e75c7 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -749,8 +749,9 @@ async def resource_snapshot(self) -> RuntimeResourceSnapshot: Raises: ResourceObservationUnavailableError: If the runtime stopped, is - stopping, its adapter host process exited, or a live process - in its tree or the process table cannot be read. + stopping, its adapter host process exited, a live process in + its tree or the process table cannot be read, or the process + tree kept changing during the read. ResourceObservationUnsupportedError: If the runtime's adapter kind, environment provider, or operating system has no resource reader. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index def96b269..0c790fc78 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -83,7 +83,7 @@ Refer to the [errors reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/ | `FabricRuntimeError` | Startup, invocation, or shutdown failed before a normalized result. | | `FabricStateError` | Invalid runtime state transition (invoking after stop, overlapping invocations). | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | -| `ResourceObservationUnavailableError` | A resource snapshot cannot be read because the runtime stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnavailableError` | A resource snapshot cannot be read because the runtime stopped, is stopping, its adapter host exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read. Subclass of `FabricRuntimeError`. | | `ResourceObservationUnsupportedError` | The runtime's adapter kind, environment provider, or operating system has no resource reader. Subclass of `FabricRuntimeError`. | ## Cleanup And Resilience From 0371c19a5ec5c0a09ca028a4362cd994e8155117 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:58:50 -0700 Subject: [PATCH 016/102] docs: regenerate API references for resource snapshot error docs Picks up the corrected RunResult.resource_snapshot field doc, the changing-tree unavailable reason, and the hidepid note from the Rust and Python docstrings. Signed-off-by: mnajafian-nv --- .../api/python-library-reference/nemo_fabric.errors.md | 2 +- .../api/python-library-reference/nemo_fabric.runtime.md | 2 +- .../reference/api/python-library-reference/nemo_fabric.types.md | 2 +- .../nemo-fabric-core/error/enum-fabricerror.mdx | 2 +- .../nemo-fabric-core/resource-snapshot/index.mdx | 2 +- .../nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx | 2 +- .../nemo-fabric-core/runtime/struct-runresult.mdx | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/reference/api/python-library-reference/nemo_fabric.errors.md b/docs/reference/api/python-library-reference/nemo_fabric.errors.md index 8acf76f75..413945d97 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.errors.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.errors.md @@ -281,7 +281,7 @@ Initialize a structured NeMo Fabric exception. Runtime resources cannot be read now. -The runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. +The runtime stopped, is stopping, its adapter host process exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read. ### Inheritance diff --git a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md index 73fc8cd5b..cb58196dd 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md @@ -197,7 +197,7 @@ The snapshot covers the local adapter host process and its live descendants. It **Raises:** - - `ResourceObservationUnavailableError`: If the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. + - `ResourceObservationUnavailableError`: If the runtime stopped, is stopping, its adapter host process exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read. - `ResourceObservationUnsupportedError`: If the runtime's adapter kind, environment provider, or operating system has no resource reader. - `FabricNativeUnavailableError`: If the native extension is missing. - `FabricRuntimeError`: If the native snapshot call fails otherwise. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index f009db3cf..59c7e7c06 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1348,7 +1348,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - `error`: Structured failure, or ``None`` on success. - `usage`: Normalized invocation usage, or ``None`` when unavailable. - - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. ``Runtime.resource_snapshot()`` raises the error that explains a ``None``. + - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. - `artifacts`: Normalized artifact manifest. - `telemetry`: Ordered telemetry references. - `events`: Ordered lifecycle and invocation events. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 805b8e7f7..ae0e4009e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -501,7 +501,7 @@ Adapter kind.
-A runtime's resources cannot be read now: the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. +A runtime's resources cannot be read now: the runtime stopped, is stopping, its adapter host process exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read. #### Fields diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index 2ba0dad07..e55480dcc 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -16,7 +16,7 @@ A snapshot reads the process tree rooted at the adapter host process that NeMo F Linux reads `/proc//task//children` and `/proc//stat`. On a kernel without those children files, it reads every process's parent ID once per snapshot to find the tree. macOS reads `proc_listchildpids` and `proc_pid_rusage`. A snapshot exposes only its own fields: never command lines, environment variables, open files, or processes outside the tree. -A live process in the tree that cannot be read, or a process table that the Linux fallback cannot read in full, makes the snapshot unavailable instead of leaving the process out. +A live process in the tree that cannot be read, or a process table that the Linux fallback cannot read in full, makes the snapshot unavailable instead of leaving the process out. On Linux with `/proc` mounted `hidepid=invisible`, another user's descendant is invisible and is not counted. ## Structs diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx index 1bf68862e..4275ce2f6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx @@ -13,4 +13,4 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Read a timestamped resource snapshot of a started runtime. -The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, or a live process in its tree or the process table cannot be read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the adapter kind, environment provider, or operating system has no resource reader. +The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the adapter kind, environment provider, or operating system has no resource reader. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index e8aba8ae4..3e05964a5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -61,7 +61,7 @@ Normalized invocation usage when reported by the adapter target. ### `resource_snapshot: Option` -Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed; `runtime_resource_snapshot` returns the error that explains it. +Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed. ### `artifacts: ArtifactManifest` From e7665eb0247762799394bc9ae19c438ab2857778 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 04:05:20 -0700 Subject: [PATCH 017/102] refactor: move the RFC 3339 formatter into a shared time module The run record needs the same millisecond RFC 3339 timestamps as resource snapshots. Moving the formatter into crate::time gives core one copy of the civil date algorithm, so a later feature imports it instead of copying it. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/lib.rs | 1 + crates/fabric-core/src/resource_snapshot.rs | 66 +-------------- crates/fabric-core/src/time.rs | 90 +++++++++++++++++++++ 3 files changed, 95 insertions(+), 62 deletions(-) create mode 100644 crates/fabric-core/src/time.rs diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index b20d70a09..72da8560d 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -12,6 +12,7 @@ pub mod error; pub mod resource_snapshot; pub mod runtime; pub mod schema; +mod time; pub use adapter_contract::{ADAPTER_CONTRACT_VERSION, AdapterExtensionPoint}; pub use agent_execution::{ diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 09175db45..a3a3be622 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -24,14 +24,14 @@ //! `hidepid=invisible`, another user's descendant is invisible and is not //! counted. -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::SystemTime; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::error::{FabricError, Result}; +use crate::time::rfc3339_millis; -const MILLIS_PER_DAY: i64 = 86_400_000; const NANOS_PER_SECOND: f64 = 1_000_000_000.0; /// Timestamped resource usage of one runtime's local adapter host process tree. @@ -79,7 +79,7 @@ impl LocalHostProcess { Self { pid, identity: platform::start_identity(pid), - started_at: rfc3339_millis(unix_millis(started_at)), + started_at: rfc3339_millis(started_at), } } } @@ -113,7 +113,7 @@ pub(crate) fn snapshot_local_host( Ok(RuntimeResourceSnapshot { runtime_id: runtime_id.to_string(), started_at: host.started_at.clone(), - observed_at: rfc3339_millis(unix_millis(SystemTime::now())), + observed_at: rfc3339_millis(SystemTime::now()), resident_memory_bytes: read.resident_bytes, process_count: read.process_count, cpu_time_seconds: read.cpu_nanos as f64 / NANOS_PER_SECOND, @@ -255,49 +255,6 @@ mod tree { } } -fn unix_millis(time: SystemTime) -> i64 { - match time.duration_since(UNIX_EPOCH) { - Ok(after) => i64::try_from(after.as_millis()).unwrap_or(i64::MAX), - Err(before) => -i64::try_from(before.duration().as_millis()).unwrap_or(i64::MAX), - } -} - -/// Format Unix milliseconds as RFC 3339 UTC with three fraction digits. -/// -/// Core has no date crate, so this converts days to a civil date with Howard -/// Hinnant's `civil_from_days` algorithm. -fn rfc3339_millis(unix_millis: i64) -> String { - let days = unix_millis.div_euclid(MILLIS_PER_DAY); - let millis_of_day = unix_millis.rem_euclid(MILLIS_PER_DAY); - let (year, month, day) = civil_from_days(days); - let seconds_of_day = millis_of_day / 1000; - format!( - "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{:03}Z", - seconds_of_day / 3600, - seconds_of_day / 60 % 60, - seconds_of_day % 60, - millis_of_day % 1000, - ) -} - -fn civil_from_days(days: i64) -> (i64, i64, i64) { - let z = days + 719_468; - let era = z.div_euclid(146_097); - let day_of_era = z.rem_euclid(146_097); - let year_of_era = - (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; - let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); - let month_index = (5 * day_of_year + 2) / 153; - let day = day_of_year - (153 * month_index + 2) / 5 + 1; - let month = if month_index < 10 { - month_index + 3 - } else { - month_index - 9 - }; - let year = year_of_era + era * 400 + i64::from(month <= 2); - (year, month, day) -} - #[cfg(target_os = "linux")] mod platform { //! `/proc` reads. Each process's children come from @@ -686,21 +643,6 @@ mod tests { }; use super::*; - #[test] - fn rfc3339_millis_formats_fixed_vectors() { - assert_eq!(rfc3339_millis(0), "1970-01-01T00:00:00.000Z"); - assert_eq!( - rfc3339_millis(1_792_000_870_205), - "2026-10-14T18:01:10.205Z" - ); - assert_eq!(rfc3339_millis(951_782_400_000), "2000-02-29T00:00:00.000Z"); - assert_eq!( - rfc3339_millis(4_107_542_399_999), - "2100-02-28T23:59:59.999Z" - ); - assert_eq!(rfc3339_millis(-1), "1969-12-31T23:59:59.999Z"); - } - #[test] fn snapshot_local_host_without_active_host_is_unavailable_or_unsupported() { let error = snapshot_local_host("runtime-1", None).expect_err("no host"); diff --git a/crates/fabric-core/src/time.rs b/crates/fabric-core/src/time.rs new file mode 100644 index 000000000..06ace9edf --- /dev/null +++ b/crates/fabric-core/src/time.rs @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! RFC 3339 UTC timestamps with millisecond precision. +//! +//! Core has no date crate, so this converts Unix milliseconds to a civil date +//! with Howard Hinnant's `civil_from_days` algorithm. + +use std::time::{SystemTime, UNIX_EPOCH}; + +const MILLIS_PER_DAY: i64 = 86_400_000; + +/// Format `time` as RFC 3339 UTC with three fraction digits, such as +/// `2026-10-14T18:01:10.205Z`. +pub(crate) fn rfc3339_millis(time: SystemTime) -> String { + format_unix_millis(unix_millis(time)) +} + +/// Milliseconds since the Unix epoch; negative before it. +fn unix_millis(time: SystemTime) -> i64 { + match time.duration_since(UNIX_EPOCH) { + Ok(after) => i64::try_from(after.as_millis()).unwrap_or(i64::MAX), + Err(before) => -i64::try_from(before.duration().as_millis()).unwrap_or(i64::MAX), + } +} + +fn format_unix_millis(unix_millis: i64) -> String { + let days = unix_millis.div_euclid(MILLIS_PER_DAY); + let millis_of_day = unix_millis.rem_euclid(MILLIS_PER_DAY); + let (year, month, day) = civil_from_days(days); + let seconds_of_day = millis_of_day / 1000; + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{:03}Z", + seconds_of_day / 3600, + seconds_of_day / 60 % 60, + seconds_of_day % 60, + millis_of_day % 1000, + ) +} + +fn civil_from_days(days: i64) -> (i64, i64, i64) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + (year, month, day) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn rfc3339_millis_formats_fixed_vectors() { + assert_eq!(format_unix_millis(0), "1970-01-01T00:00:00.000Z"); + assert_eq!( + format_unix_millis(1_792_000_870_205), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!( + format_unix_millis(951_782_400_000), + "2000-02-29T00:00:00.000Z" + ); + assert_eq!( + format_unix_millis(4_107_542_399_999), + "2100-02-28T23:59:59.999Z" + ); + assert_eq!(format_unix_millis(-1), "1969-12-31T23:59:59.999Z"); + assert_eq!( + rfc3339_millis(UNIX_EPOCH + Duration::from_millis(1_792_000_870_205)), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!( + rfc3339_millis(UNIX_EPOCH - Duration::from_millis(1)), + "1969-12-31T23:59:59.999Z" + ); + } +} From 5e4c29de1584e5c7d61724bd36adc380c7636c9c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 23:35:13 -0700 Subject: [PATCH 018/102] feat: write an opt-in run record for each invocation Core already writes adapter-invocation.json before each request, so a killed run shows which invocation started. Nothing on disk says how long it ran, how it ended, or why it stopped, and RunResult exists only in memory. Add an opt-in, append-only run-record.jsonl per invocation under /.fabric///. runtime.record.enabled turns it on; an artifact root alone does not. Planning rejects an enabled record without an artifact root, with a non-local environment provider, or with an adapter that is not a process or python adapter. Core writes a start line right before it sends the request to the host and an end line after the invocation returns or raises, then syncs the file once after the host lock is released. Both lines are built from typed, allowlisted fields: other context keys appear by name only, override_keys lists override names without values, and the end line keeps the error code, stage and retryable flag but never the message or adapter metadata. A failed write adds a record_write_failed event and never fails the run. The record artifact gets a unique name, as adapter and Relay artifacts do. RecordConfig derives Default (recording off), so a caller can turn recording on without spelling out every record field. The record module is crate-private; the line format is published as the v1alpha1 run-record-line schema. The RFC 3339 formatter lives in a crate-private time module so other core features can share it. Signed-off-by: mnajafian-nv --- crates/fabric-cli/src/presets.rs | 1 + crates/fabric-cli/templates/rust/main.rs.tmpl | 1 + crates/fabric-core/src/config.rs | 161 ++++++ crates/fabric-core/src/lib.rs | 11 +- crates/fabric-core/src/record/invocation.rs | 477 ++++++++++++++++++ crates/fabric-core/src/record/lines.rs | 264 ++++++++++ crates/fabric-core/src/record/mod.rs | 19 + crates/fabric-core/src/record/writer.rs | 104 ++++ crates/fabric-core/src/runtime.rs | 301 ++++++++++- crates/fabric-core/src/schema.rs | 9 +- crates/fabric-core/src/time.rs | 90 ++++ .../api/python-library-reference/index.md | 1 + .../nemo_fabric.models.md | 73 +++ .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../config/enum-capabilitykind.mdx | 2 +- .../config/enum-capabilitytarget.mdx | 2 +- .../config/enum-relayatifstorageconfig.mdx | 2 +- .../config/enum-relayatofmode.mdx | 2 +- .../config/enum-relayatofsinkconfig.mdx | 2 +- .../enum-relayatofstreamfieldnamepolicy.mdx | 2 +- .../config/enum-relayatofstreamtransport.mdx | 2 +- .../enum-relayopentelemetryendpointtype.mdx | 2 +- .../enum-relayopentelemetrymarkprojection.mdx | 2 +- .../config/enum-relayotlptransport.mdx | 2 +- .../config/enum-relayunsupportedbehavior.mdx | 2 +- .../config/enum-resolutionstrategy.mdx | 2 +- .../config/enum-telemetryprovider.mdx | 2 +- .../config/fn-load-adapter-descriptor.mdx | 2 +- .../fn-load-adapter-target-descriptor.mdx | 2 +- .../fn-resolve-run-plan-from-config.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 3 +- .../config/struct-recordconfig.mdx | 108 ++++ .../config/struct-relayatifconfig.mdx | 2 +- .../config/struct-relayatofconfig.mdx | 2 +- .../config/struct-relaycomponentconfig.mdx | 2 +- .../config/struct-relayconfig.mdx | 2 +- .../config/struct-relayconfigpolicy.mdx | 2 +- .../struct-relayobservabilityconfig.mdx | 2 +- .../struct-relayopentelemetryconfig.mdx | 2 +- ...truct-relayopentelemetryendpointconfig.mdx | 2 +- .../config/struct-resolvecontext.mdx | 2 +- .../struct-resolvedadapterdescriptor.mdx | 2 +- ...struct-resolvedadaptertargetdescriptor.mdx | 2 +- .../config/struct-runplan.mdx | 2 +- .../config/struct-runtimecapabilities.mdx | 2 +- .../config/struct-runtimeconfig.mdx | 8 +- .../config/struct-skillconfig.mdx | 2 +- .../config/struct-telemetryconfig.mdx | 2 +- .../config/struct-telemetryplan.mdx | 2 +- .../config/struct-telemetryproviderconfig.mdx | 2 +- .../config/struct-tooldefinitionconfig.mdx | 2 +- .../config/struct-toolsconfig.mdx | 2 +- .../config/struct-toolsplan.mdx | 2 +- .../config/struct-workflowconfig.mdx | 2 +- .../struct-workflowentrypointconfig.mdx | 2 +- .../config/struct-workflowtargetspec.mdx | 2 +- .../doctor/enum-doctorstatus.mdx | 2 +- .../doctor/fn-doctor-plan.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../doctor/struct-doctorcheck.mdx | 2 +- .../doctor/struct-doctorreport.mdx | 2 +- .../error/enum-fabricerror.mdx | 2 +- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/error/type-result.mdx | 2 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/index.mdx | 1 + .../nemo-fabric-core/runtime/index.mdx | 2 +- .../schema/enum-schemaname.mdx | 9 +- .../nemo-fabric-core/schema/index.mdx | 2 +- docs/sdk/python.mdx | 74 ++- schemas/SCHEMA.md | 3 + schemas/sdk/agent.schema.json | 25 + schemas/sdk/run-plan.schema.json | 25 + schemas/sdk/run-record-line.schema.json | 429 ++++++++++++++++ .../src/nemo_fabric/__init__.py | 2 + .../src/nemo_fabric/models.py | 16 + .../src/nemo_fabric/types.py | 19 +- .../references/results-and-errors.md | 15 + tests/python/test_sdk_consumer.py | 15 +- tests/python/test_sdk_contract.py | 23 + 82 files changed, 2321 insertions(+), 72 deletions(-) create mode 100644 crates/fabric-core/src/record/invocation.rs create mode 100644 crates/fabric-core/src/record/lines.rs create mode 100644 crates/fabric-core/src/record/mod.rs create mode 100644 crates/fabric-core/src/record/writer.rs create mode 100644 crates/fabric-core/src/time.rs create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx create mode 100644 schemas/sdk/run-record-line.schema.json diff --git a/crates/fabric-cli/src/presets.rs b/crates/fabric-cli/src/presets.rs index 803c6667e..5ebaa3202 100644 --- a/crates/fabric-cli/src/presets.rs +++ b/crates/fabric-cli/src/presets.rs @@ -287,6 +287,7 @@ fn config( artifacts: None, timeout_seconds: None, max_turns: None, + record: None, extensions: BTreeMap::new(), }, environment: Some(EnvironmentConfig { diff --git a/crates/fabric-cli/templates/rust/main.rs.tmpl b/crates/fabric-cli/templates/rust/main.rs.tmpl index e7f0ccfd3..be1c9771f 100644 --- a/crates/fabric-cli/templates/rust/main.rs.tmpl +++ b/crates/fabric-cli/templates/rust/main.rs.tmpl @@ -38,6 +38,7 @@ fn build_config() -> FabricConfig { artifacts: None, timeout_seconds: {{TIMEOUT_SECONDS}}, max_turns: {{MAX_TURNS}}, + record: None, extensions: BTreeMap::new(), }, environment: Some(EnvironmentConfig { diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index bef84b454..1cb2d7dcf 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -1057,11 +1057,34 @@ pub struct RuntimeConfig { #[serde(default, skip_serializing_if = "Option::is_none")] #[schemars(range(min = 1, max = u32::MAX))] pub max_turns: Option, + /// Opt-in per-invocation run record. Core-only; never sent to adapters. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub record: Option, /// Additive normalized runtime fields. #[serde(default, flatten)] pub extensions: BTreeMap, } +impl RuntimeConfig { + /// Whether `runtime.record.enabled` is set. + pub(crate) fn record_enabled(&self) -> bool { + self.record.as_ref().is_some_and(|record| record.enabled) + } +} + +/// Per-invocation run record settings. +/// +/// When enabled, core appends a start line and an end line for each local +/// invocation to `/.fabric///run-record.jsonl`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RecordConfig { + /// Whether to write a run record for each invocation. Requires an + /// artifact root, the `local` environment provider, and a `process` or + /// `python` adapter. + pub enabled: bool, +} + fn default_input_schema() -> String { "text".to_string() } @@ -1905,6 +1928,7 @@ pub(crate) fn validate_config(config: &FabricConfig) -> Result<()> { "must be a finite number greater than zero", ); } + validate_record_config(config)?; if let Some(system) = config .instructions .as_ref() @@ -2337,6 +2361,52 @@ fn validate_names(field: &str, names: &[String]) -> Result<()> { Ok(()) } +/// A record needs a host-side artifact root and a local adapter host, so an +/// enabled record that could never be written fails validation instead of +/// silently writing nothing. +fn validate_record_config(config: &FabricConfig) -> Result<()> { + if !config.runtime.record_enabled() { + return Ok(()); + } + let environment = config.environment.as_ref(); + if environment.is_some_and(|environment| environment.provider != "local") { + return invalid_config( + "runtime.record.enabled", + "supported only with the local environment provider", + ); + } + let has_artifact_root = config.runtime.artifacts.is_some() + || environment.is_some_and(|environment| environment.artifacts.is_some()); + if !has_artifact_root { + return invalid_config( + "runtime.record.enabled", + "requires runtime.artifacts or an environment artifact root", + ); + } + Ok(()) +} + +/// Core writes the record from its local adapter host path, so an adapter +/// that runs elsewhere would get an enabled record that is never written. +fn validate_record_adapter( + config: &FabricConfig, + descriptor: Option<&AdapterDescriptor>, +) -> Result<()> { + let local_host = descriptor.is_none_or(|descriptor| { + matches!( + descriptor.adapter_kind, + AdapterKind::Process | AdapterKind::Python + ) + }); + if config.runtime.record_enabled() && !local_host { + return invalid_config( + "runtime.record.enabled", + "supported only with a process or python adapter", + ); + } + Ok(()) +} + fn invalid_config(field: impl Into, reason: impl Into) -> Result { Err(FabricError::InvalidConfig { field: field.into(), @@ -2537,6 +2607,7 @@ fn resolve_selected_descriptors( let resolution = resolve_resolution(&config, descriptor)?; let environment_plan = resolve_environment_plan(&config, &base_dir); validate_control_location(descriptor, environment_plan.as_ref())?; + validate_record_adapter(&config, descriptor)?; let capability_plan = resolve_capability_plan(&config, &base_dir, adapter_descriptor.as_ref()); if enforce_compatibility { validate_capability_plan_compatibility(&capability_plan, descriptor)?; @@ -7342,4 +7413,94 @@ mod tests { && paths.len() == 2 )); } + + fn record_error(config: &FabricConfig) -> Option<(String, String)> { + match validate_config(config) { + Err(FabricError::InvalidConfig { field, reason }) => Some((field, reason)), + Err(error) => panic!("unexpected error: {error}"), + Ok(()) => None, + } + } + + #[test] + fn record_enabled_without_an_artifact_root_fails_validation() { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.record = Some(RecordConfig { enabled: true }); + + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "requires runtime.artifacts or an environment artifact root".to_string() + )) + ); + config.environment.as_mut().expect("environment").artifacts = Some(PathBuf::from("runs")); + assert_eq!(record_error(&config), None); + config.environment.as_mut().expect("environment").artifacts = None; + config.runtime.artifacts = Some(PathBuf::from("runs")); + assert_eq!(record_error(&config), None); + } + + #[test] + fn record_enabled_with_a_non_local_provider_fails_validation() { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + config.environment.as_mut().expect("environment").provider = "docker".to_string(); + + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "supported only with the local environment provider".to_string() + )) + ); + } + + #[test] + fn record_enabled_with_an_adapter_outside_the_local_host_fails_validation() { + let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); + let mut descriptor = load_adapter_descriptor(&path).expect("Claude descriptor"); + let mut config = typed_config("nvidia.fabric.claude"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); + + for kind in [AdapterKind::Http, AdapterKind::NativePlugin] { + descriptor.adapter_kind = kind; + assert!(matches!( + validate_record_adapter(&config, Some(&descriptor)), + Err(FabricError::InvalidConfig { field, reason }) + if field == "runtime.record.enabled" + && reason == "supported only with a process or python adapter" + )); + } + config.runtime.record = None; + assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); + } + + #[test] + fn record_disabled_or_absent_needs_no_artifact_root() { + let mut config = typed_config("nvidia.fabric.hermes"); + assert_eq!(record_error(&config), None); + config.runtime.record = Some(RecordConfig { enabled: false }); + assert_eq!(record_error(&config), None); + } + + #[test] + fn record_block_requires_enabled_and_rejects_unknown_fields() { + for runtime in [ + serde_json::json!({"record": {}}), + serde_json::json!({"record": {"enabled": true, "durable": true}}), + serde_json::json!({"record": true}), + ] { + let mut value = + serde_json::to_value(typed_config("nvidia.fabric.hermes")).expect("config JSON"); + value["runtime"] = runtime.clone(); + assert!( + serde_json::from_value::(value).is_err(), + "{runtime}" + ); + } + } } diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index 2dc251c9f..2e7f01458 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -9,8 +9,10 @@ pub mod agent_execution; pub mod config; pub mod doctor; pub mod error; +mod record; pub mod runtime; pub mod schema; +mod time; pub use adapter_contract::{ADAPTER_CONTRACT_VERSION, AdapterExtensionPoint}; pub use agent_execution::{ @@ -28,10 +30,11 @@ pub use config::{ EnvironmentOwnership, EnvironmentPlan, FabricConfig, HarnessConfig, InstructionConfig, InstructionMode, InstructionsConfig, McpAuthenticationConfig, McpConfig, McpExposure, McpServerPlan, McpTransport, MetadataConfig, ModelConfig, OAuthTokenEndpointAuthMethod, - ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, ResolvedAdapterTargetDescriptor, - RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, TelemetryConfig, TelemetryPlan, - TelemetryProvider, TelemetryProviderConfig, ToolDefinitionConfig, ToolsConfig, WorkflowConfig, - WorkflowEntrypointConfig, WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, + RecordConfig, ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, + ResolvedAdapterTargetDescriptor, RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, + TelemetryConfig, TelemetryPlan, TelemetryProvider, TelemetryProviderConfig, + ToolDefinitionConfig, ToolsConfig, WorkflowConfig, WorkflowEntrypointConfig, + WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, load_adapter_target_descriptor, resolve_diagnostic_plan_from_config, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config, resolve_run_plan_from_config_with_adapter_directories, diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs new file mode 100644 index 000000000..7d8a2595a --- /dev/null +++ b/crates/fabric-core/src/record/invocation.rs @@ -0,0 +1,477 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One invocation's record: the start line before the request, the end line +//! after the invocation. +//! +//! Recording never changes the run. A failed create, write or sync stops the +//! record for that invocation and adds a `record_write_failed` event to the +//! result; nothing here returns an error to the caller. + +use std::collections::BTreeMap; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime}; + +use serde_json::Value; + +use super::lines::{ + ContextValue, EndArtifact, EndError, EndLine, EndUsage, LineBody, ModelName, NotRecorded, + RecordLine, RecordSchema, RecordVersion, StartLine, cut_text, recorded_code, +}; +use super::writer::{RECORD_FILE_NAME, RecordWriter}; +use crate::config::RunPlan; +use crate::error::{FabricError, Result}; +use crate::runtime::{ + ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, INVALID_AGENT_RUN_RESULT, + InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, RuntimeHandle, + event_with_metadata, unique_artifact_name, +}; +use crate::time::rfc3339_millis; + +/// Name of the result artifact that points at the record. +const RECORD_ARTIFACT_NAME: &str = "record"; +/// Kind of the result artifact that points at the record. +const RECORD_ARTIFACT_KIND: &str = "run_record"; +/// Event added to the result when the record could not be written. +const RECORD_WRITE_FAILED: &str = "record_write_failed"; +/// `RunRequest.context` keys whose string or integer values the start line +/// keeps: task identity and the Harbor trial link. +const RECORDED_CONTEXT_KEYS: [&str; 7] = [ + "task_id", + "task_revision", + "attempt", + "step", + "harbor_session_id", + "harbor_context_id", + "harbor_trial_name", +]; +/// End-line error code for an artifact write that failed after the adapter returned. +const ARTIFACT_WRITE_FAILED: &str = "artifact_write_failed"; +/// End-line error code for any other Fabric error after the request was sent. +const FABRIC_ERROR: &str = "fabric_error"; + +/// What the start line needs from core's invocation path. +pub(crate) struct RecordStart<'a> { + pub plan: &'a RunPlan, + pub runtime: &'a RuntimeHandle, + pub invocation: &'a InvocationHandle, + pub request: &'a RunRequest, + /// The invocation folder, `/.fabric//`. + pub fabric_home: &'a Path, + /// The host timeout that applies to this invocation. + pub timeout: Duration, +} + +/// The record of one running invocation. +#[derive(Debug)] +pub(crate) struct InvocationRecorder { + runtime_id: String, + invocation_id: String, + request_id: String, + path: PathBuf, + started: Instant, + writer: Option, + wrote_start: bool, + failure: Option, +} + +impl InvocationRecorder { + /// Create the record and write its start line. + pub(crate) fn start(start: RecordStart<'_>) -> Self { + let path = start.fabric_home.join(RECORD_FILE_NAME); + let mut recorder = Self { + runtime_id: start.invocation.runtime_id.clone(), + invocation_id: start.invocation.invocation_id.clone(), + request_id: start.invocation.request_id.clone(), + started: Instant::now(), + writer: None, + wrote_start: false, + failure: None, + path, + }; + match RecordWriter::create(&recorder.path) { + Ok(writer) => { + recorder.writer = Some(writer); + recorder.append(LineBody::Start(start_line(&start))); + recorder.wrote_start = recorder.failure.is_none(); + } + Err(error) => recorder.failure = Some(error), + } + recorder + } + + /// Write and sync the end line for `outcome`, then return it. A result + /// gains the `record` artifact when the start line was written, and a + /// `record_write_failed` event when any write or the sync failed. + pub(crate) fn finish(mut self, outcome: Result) -> Result { + let elapsed_ms = u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX); + let end = match &outcome { + Ok(result) => result_end_line(result, elapsed_ms), + Err(error) => error_end_line(error, elapsed_ms), + }; + self.append(LineBody::End(end)); + if let Some(writer) = &self.writer + && let Err(error) = writer.sync() + { + self.failure = Some(error); + } + let mut result = outcome?; + if self.wrote_start { + let name = unique_artifact_name(&result.artifacts, RECORD_ARTIFACT_NAME); + result.artifacts.artifacts.push(ArtifactRef { + name, + kind: RECORD_ARTIFACT_KIND.to_string(), + path: self.path.clone(), + media_type: Some("application/jsonl".to_string()), + metadata: BTreeMap::new(), + }); + } + if let Some(error) = &self.failure { + result.events.push(event_with_metadata( + RECORD_WRITE_FAILED, + format!("run record write failed; the record stops here: {error}"), + BTreeMap::from([ + ( + "runtime_id".to_string(), + Value::from(self.runtime_id.as_str()), + ), + ( + "invocation_id".to_string(), + Value::from(self.invocation_id.as_str()), + ), + ]), + )); + } + Ok(result) + } + + /// Append one line. After the first failure the record stops. + fn append(&mut self, body: LineBody) { + let Some(writer) = self.writer.as_mut() else { + return; + }; + let line = RecordLine { + schema: RecordSchema::RunRecord, + version: RecordVersion::V1Alpha1, + runtime_id: self.runtime_id.clone(), + invocation_id: self.invocation_id.clone(), + request_id: self.request_id.clone(), + time: rfc3339_millis(SystemTime::now()), + body, + }; + if let Err(error) = writer.append(&line) { + self.failure = Some(error); + self.writer = None; + } + } +} + +fn start_line(start: &RecordStart<'_>) -> StartLine { + StartLine { + agent_name: cut_text(&start.runtime.agent_name), + harness: cut_text(&start.runtime.harness), + adapter_id: start.runtime.adapter_id.as_deref().map(cut_text), + models: start + .plan + .config + .models + .iter() + .map(|(role, model)| { + ( + cut_text(role), + ModelName { + provider: cut_text(&model.provider), + model: cut_text(&model.model), + }, + ) + }) + .collect(), + fabric_version: crate::version().to_string(), + timeout_ms: u64::try_from(start.timeout.as_millis()).unwrap_or(u64::MAX), + context: start + .request + .context + .iter() + .map(|(key, value)| (cut_text(key), context_value(key, value))) + .collect(), + override_keys: override_keys(start.request.overrides.as_ref()), + } +} + +/// The sorted top-level key names of `overrides`; empty unless it is an object. +fn override_keys(overrides: Option<&Value>) -> Vec { + let Some(Value::Object(overrides)) = overrides else { + return Vec::new(); + }; + let mut keys: Vec = overrides.keys().map(|key| cut_text(key)).collect(); + keys.sort(); + keys +} + +/// An allowlisted key keeps a string (cut) or integer value; any other key, +/// or an allowlisted key with another value type, is recorded by name only. +fn context_value(key: &str, value: &Value) -> ContextValue { + if !RECORDED_CONTEXT_KEYS.contains(&key) { + return ContextValue::NotRecorded(NotRecorded::VALUE); + } + match value { + Value::String(text) => ContextValue::Text(cut_text(text)), + Value::Number(number) => number + .as_i64() + .map_or(ContextValue::NotRecorded(NotRecorded::VALUE), |integer| { + ContextValue::Integer(integer) + }), + _ => ContextValue::NotRecorded(NotRecorded::VALUE), + } +} + +fn result_end_line(result: &RunResult, elapsed_ms: u64) -> EndLine { + EndLine { + status: result.status, + elapsed_ms, + error: result.error.as_ref().map(result_error), + usage: result.usage.as_ref().map(end_usage), + artifacts: end_artifacts(&result.artifacts), + } +} + +fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { + let (code, stage) = match error { + FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), + // After the record starts, only the check of the adapter's result + // extensions raises this, the same fault as a rejected result. + FabricError::AdapterCompatibility { .. } => (INVALID_AGENT_RUN_RESULT, ErrorStage::Invoke), + FabricError::Write { .. } => (ARTIFACT_WRITE_FAILED, ErrorStage::Artifact), + _ => (FABRIC_ERROR, ErrorStage::Invoke), + }; + EndLine { + status: RunStatus::Failed, + elapsed_ms, + error: Some(EndError { + code: recorded_code(code), + stage, + retryable: false, + }), + usage: None, + artifacts: Vec::new(), + } +} + +fn result_error(error: &ErrorInfo) -> EndError { + EndError { + code: recorded_code(&error.code), + stage: error.stage, + retryable: error.retryable, + } +} + +fn end_usage(usage: &RunUsage) -> EndUsage { + EndUsage { + input_tokens: usage.input_tokens, + cached_input_tokens: usage.cached_input_tokens, + input_tokens_include_cache: usage.input_tokens_include_cache, + output_tokens: usage.output_tokens, + total_tokens: usage.total_tokens, + cost_usd: usage.cost_usd, + } +} + +fn end_artifacts(manifest: &ArtifactManifest) -> Vec { + manifest + .artifacts + .iter() + .map(|artifact| EndArtifact { + name: cut_text(&artifact.name), + kind: cut_text(&artifact.kind), + media_type: artifact.media_type.as_deref().map(cut_text), + path: manifest + .root + .as_deref() + .and_then(|root| artifact.path.strip_prefix(root).ok()) + .map(|relative| relative.to_string_lossy().into_owned()), + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; + + fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { + let config: FabricConfig = serde_json::from_value(serde_json::json!({ + "schema_version": "fabric.agent/v1alpha1", + "metadata": {"name": "demo"}, + "harness": {"adapter_id": "nvidia.fabric.hermes", "resolution": "preinstalled"}, + "runtime": {}, + "models": {"default": {"provider": "nvidia", "model": "nemotron"}}, + })) + .expect("config"); + let plan = resolve_run_plan_from_config(config, ResolveContext::new("/tmp/fabric-base")) + .expect("plan"); + let runtime: RuntimeHandle = serde_json::from_value(serde_json::json!({ + "runtime_id": "rt", "runtime_binding": "binding", "agent_name": "demo", + "harness": "nvidia.fabric.hermes", "adapter_kind": "python", + "adapter_id": "nvidia.fabric.hermes", + "environment": { + "environment_id": "env", "provider": "local", + "control_location": "external_control", "ownership": "caller_owned", + }, + })) + .expect("runtime handle"); + let invocation = InvocationHandle { + invocation_id: "inv".to_string(), + request_id: "req".to_string(), + runtime_id: "rt".to_string(), + }; + (plan, runtime, invocation) + } + + #[test] + fn start_line_keeps_override_and_unknown_context_key_names_only() { + let (plan, runtime, invocation) = record_test_fixture(); + let mut request = RunRequest::text("input"); + request.context = BTreeMap::from([ + ("task_id".to_string(), Value::from("t".repeat(300))), + ("task_revision".to_string(), Value::from(1.5)), + ( + "headers".to_string(), + serde_json::json!({"authorization": "Bearer abc"}), + ), + ]); + request.overrides = Some(serde_json::json!({"zeta": 1, "acme": {"api_key": "sk-123"}})); + let start = RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: Path::new("/unused"), + timeout: Duration::from_secs(5), + }; + + let line = serde_json::to_value(RecordLine { + schema: RecordSchema::RunRecord, + version: RecordVersion::V1Alpha1, + runtime_id: "rt".to_string(), + invocation_id: "inv".to_string(), + request_id: "req".to_string(), + time: "2026-10-14T18:01:10.205Z".to_string(), + body: LineBody::Start(start_line(&start)), + }) + .expect("start line"); + + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunRecordLine) + .expect("record line schema"); + let validator = jsonschema::validator_for(&schema).expect("valid schema"); + assert!(validator.is_valid(&line), "{line}"); + assert_eq!(line["override_keys"], serde_json::json!(["acme", "zeta"])); + assert_eq!( + line["context"]["headers"], + serde_json::json!({"recorded": false}) + ); + assert_eq!( + line["context"]["task_revision"], + serde_json::json!({"recorded": false}) + ); + assert_eq!( + line["context"]["task_id"] + .as_str() + .expect("task id") + .chars() + .count(), + 256 + ); + assert_eq!(line["timeout_ms"], 5_000); + assert_eq!( + line["models"], + serde_json::json!({"default": {"provider": "nvidia", "model": "nemotron"}}) + ); + let text = line.to_string(); + assert!( + !text.contains("sk-123") && !text.contains("Bearer"), + "{text}" + ); + } + + fn temp_folder(name: &str) -> PathBuf { + let folder = + std::env::temp_dir().join(format!("fabric-recorder-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&folder); + std::fs::create_dir_all(&folder).expect("folder"); + folder + } + + fn succeeded_result(invocation: &InvocationHandle) -> RunResult { + serde_json::from_value(serde_json::json!({ + "agent_name": "a", "harness": "h", "adapter_kind": "python", + "runtime_id": invocation.runtime_id, "invocation_id": invocation.invocation_id, + "request_id": invocation.request_id, "status": "succeeded", "output": "done", + })) + .expect("result") + } + + #[test] + fn finish_names_the_record_artifact_apart_from_an_adapter_artifact_named_record() { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder("unique-name"); + let request = RunRequest::text("input"); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + let mut result = succeeded_result(&invocation); + result.artifacts.artifacts.push(ArtifactRef { + name: RECORD_ARTIFACT_NAME.to_string(), + kind: "report".to_string(), + path: folder.join("report.json"), + media_type: None, + metadata: BTreeMap::new(), + }); + + let finished = recorder.finish(Ok(result)).expect("result"); + + let names: Vec<&str> = finished + .artifacts + .artifacts + .iter() + .map(|artifact| artifact.name.as_str()) + .collect(); + assert_eq!(names, ["record", "record_2"]); + assert_eq!(finished.artifacts.artifacts[1].kind, RECORD_ARTIFACT_KIND); + let _ = std::fs::remove_dir_all(folder); + } + + #[test] + fn finish_after_a_failed_create_adds_the_event_and_no_record_artifact() { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder("create-failure"); + std::fs::write(folder.join(RECORD_FILE_NAME), "planted").expect("planted file"); + let request = RunRequest::text("input"); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + let result = succeeded_result(&invocation); + + let finished = recorder.finish(Ok(result.clone())).expect("result"); + + assert_eq!(finished.artifacts, result.artifacts); + assert_eq!(finished.events.len(), 1); + assert_eq!(finished.events[0].kind, RECORD_WRITE_FAILED); + assert_eq!(finished.status, RunStatus::Succeeded); + assert_eq!( + std::fs::read_to_string(folder.join(RECORD_FILE_NAME)).expect("read"), + "planted" + ); + let _ = std::fs::remove_dir_all(folder); + } +} diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs new file mode 100644 index 000000000..ebe9df282 --- /dev/null +++ b/crates/fabric-core/src/record/lines.rs @@ -0,0 +1,264 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Line types of `run-record.jsonl`, schema `nemo.fabric.run_record` version +//! `v1alpha1`. +//! +//! Every field is typed. A line holds identifiers, names, numbers, closed codes +//! and the values of a few allowlisted request context keys. It never holds a +//! value outside that allowlist: no prompt, output, error message or metadata. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::Serialize; + +use crate::runtime::{ErrorStage, RunStatus}; + +/// Longest string a line keeps from config or the request, in characters. +const MAX_VALUE_CHARS: usize = 256; +/// Longest adapter error code a line keeps. +const MAX_CODE_CHARS: usize = 128; +/// Error code written in place of a code outside `^[a-z0-9_.:-]{1,128}$`. +const UNRECOGNIZED_CODE: &str = "unrecognized"; + +/// One line of a run record. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +#[schemars(title = "RunRecordLine")] +pub(crate) struct RecordLine { + /// Always `nemo.fabric.run_record`. + pub schema: RecordSchema, + /// Line format version. `v1alpha1` may change until a record reader ships. + pub version: RecordVersion, + /// Runtime id. + pub runtime_id: String, + /// Invocation id. + pub invocation_id: String, + /// Request id. + pub request_id: String, + /// UTC wall-clock time the line was written, RFC 3339 with milliseconds. + #[schemars(extend("format" = "date-time"))] + pub time: String, + /// Line kind and its fields. + #[serde(flatten)] + pub body: LineBody, +} + +/// The record's schema name. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) enum RecordSchema { + /// `nemo.fabric.run_record`. + #[serde(rename = "nemo.fabric.run_record")] + RunRecord, +} + +/// The record's line format version. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) enum RecordVersion { + /// `v1alpha1`. + #[serde(rename = "v1alpha1")] + V1Alpha1, +} + +/// A line's kind, with the fields that kind carries. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub(crate) enum LineBody { + /// Written once, before core sends the request to the adapter host. + Start(StartLine), + /// Written once, after the invocation returned a result or failed. + End(EndLine), +} + +/// Fields of the start line. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub(crate) struct StartLine { + /// Config name (`metadata.name`). + pub agent_name: String, + /// Harness identifier. + pub harness: String, + /// Adapter implementation id. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub adapter_id: Option, + /// Configured models by role: provider and model name only. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub models: BTreeMap, + /// NeMo Fabric version that wrote the record. + pub fabric_version: String, + /// Host invocation timeout that applied, in milliseconds. + pub timeout_ms: u64, + /// `RunRequest.context`. Allowlisted identity and Harbor link keys keep a + /// string or integer value; every other key is listed by name only. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub context: BTreeMap, + /// Top-level key names of `RunRequest.overrides`, sorted. Values are never + /// recorded. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub override_keys: Vec, +} + +/// One configured model. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct ModelName { + /// Model provider name. + pub provider: String, + /// Provider model identifier. + pub model: String, +} + +/// A request context value as the start line keeps it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(untagged)] +pub(crate) enum ContextValue { + /// An allowlisted key's string value, cut to 256 characters. + Text(String), + /// An allowlisted key's integer value. + Integer(i64), + /// Any other key or value: the key name only. + NotRecorded(NotRecorded), +} + +/// Marks a key whose value the record leaves out. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct NotRecorded { + /// Always `false`. + #[schemars(extend("const" = false))] + pub recorded: bool, +} + +impl NotRecorded { + pub(crate) const VALUE: Self = Self { recorded: false }; +} + +/// Fields of the end line. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub(crate) struct EndLine { + /// Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`. + pub status: RunStatus, + /// Monotonic time from the start line to the end line, in milliseconds. + pub elapsed_ms: u64, + /// Error code, stage and retryable flag, without the message. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Numeric usage counters, without adapter metadata. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, + /// Result artifacts, with paths relative to the artifact root. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub artifacts: Vec, +} + +/// The end line's error. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct EndError { + /// Error code matching `^[a-z0-9_.:-]{1,128}$`, else `unrecognized`. + #[schemars(regex(pattern = r"^[a-z0-9_.:-]{1,128}$"))] + pub code: String, + /// Lifecycle stage where the failure surfaced. + pub stage: ErrorStage, + /// Whether a consumer-level retry is considered safe. + pub retryable: bool, +} + +/// The end line's usage: the numeric `RunUsage` fields. +#[derive(Debug, Clone, Default, PartialEq, Serialize, JsonSchema)] +pub(crate) struct EndUsage { + /// Input tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens: Option, + /// Cached input tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cached_input_tokens: Option, + /// Whether `input_tokens` includes `cached_input_tokens`; absent means unknown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens_include_cache: Option, + /// Output tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_tokens: Option, + /// Total tokens reported by the provider. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_tokens: Option, + /// Reported cost in US dollars. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 0.0))] + pub cost_usd: Option, +} + +/// One result artifact on the end line. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct EndArtifact { + /// Logical artifact name. + pub name: String, + /// Artifact kind. + pub kind: String, + /// Media type. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub media_type: Option, + /// Path relative to the artifact root; absent for a path outside it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, +} + +/// Cut `text` to [`MAX_VALUE_CHARS`] characters: 255 plus "…". +pub(crate) fn cut_text(text: &str) -> String { + if text.chars().count() <= MAX_VALUE_CHARS { + return text.to_string(); + } + let mut cut: String = text.chars().take(MAX_VALUE_CHARS - 1).collect(); + cut.push('…'); + cut +} + +/// An error code as the end line keeps it: unchanged when it matches +/// `^[a-z0-9_.:-]{1,128}$`, else [`UNRECOGNIZED_CODE`], because a code outside +/// that pattern may be a message. +pub(crate) fn recorded_code(code: &str) -> String { + let valid = (1..=MAX_CODE_CHARS).contains(&code.len()) + && code.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"_.:-".contains(&byte) + }); + if valid { + code.to_string() + } else { + UNRECOGNIZED_CODE.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cut_text_over_the_limit_keeps_255_chars_and_an_ellipsis() { + let cut = cut_text(&"é".repeat(1000)); + + assert_eq!(cut.chars().count(), MAX_VALUE_CHARS); + assert!(cut.ends_with('…')); + assert_eq!(cut_text("short"), "short"); + assert_eq!(cut_text(&"a".repeat(256)), "a".repeat(256)); + } + + #[test] + fn recorded_code_keeps_only_codes_in_the_pattern() { + for code in [ + "timeout", + "host_timeout", + "provider.rate_limit", + "http:429", + "a-b", + ] { + assert_eq!(recorded_code(code), code); + } + for code in [ + "", + "Timeout", + "rate limit", + "error: key sk-123", + "über", + &"a".repeat(129), + ] { + assert_eq!(recorded_code(code), UNRECOGNIZED_CODE, "{code}"); + } + assert_eq!(recorded_code(&"a".repeat(128)), "a".repeat(128)); + } +} diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs new file mode 100644 index 000000000..32f867bb6 --- /dev/null +++ b/crates/fabric-core/src/record/mod.rs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The opt-in per-invocation run record, `run-record.jsonl`. +//! +//! When `runtime.record.enabled` is set, core appends one start line before it +//! sends the request to the adapter host and one end line after the invocation +//! returns or fails. Recording never changes the run: a failed write stops the +//! record and becomes a `record_write_failed` event on the result. +//! +//! Everything here is crate-private. The line format is public only through +//! the `run-record-line` JSON Schema, version `v1alpha1`. + +mod invocation; +mod lines; +mod writer; + +pub(crate) use invocation::{InvocationRecorder, RecordStart}; +pub(crate) use lines::RecordLine; diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs new file mode 100644 index 000000000..1a849ac60 --- /dev/null +++ b/crates/fabric-core/src/record/writer.rs @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The run record file: created once, owner-only, appended one line per write. +//! +//! Each line is one `write_all` on an unbuffered file, so a SIGKILL of Fabric +//! or the adapter host keeps every line whose write returned. Only the end +//! line is synced, once, so the record survives a crash of the machine only +//! up to its last synced line. + +use std::fs::{File, OpenOptions}; +use std::io::{self, Write}; +use std::path::Path; + +use super::lines::RecordLine; + +/// File name of the record inside the invocation folder. +pub(crate) const RECORD_FILE_NAME: &str = "run-record.jsonl"; + +/// Appends lines to one invocation's record. +#[derive(Debug)] +pub(crate) struct RecordWriter { + file: File, +} + +impl RecordWriter { + /// Create the record at `path`. `create_new` refuses an existing file or + /// symbolic link at `path`, and on Unix the file is mode 0600. + pub(crate) fn create(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.append(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + Ok(Self { + file: options.open(path)?, + }) + } + + /// Append `line` and its newline with one write. + pub(crate) fn append(&mut self, line: &RecordLine) -> io::Result<()> { + let mut bytes = serde_json::to_vec(line).map_err(io::Error::other)?; + bytes.push(b'\n'); + self.file.write_all(&bytes) + } + + /// Flush the file's data and metadata to stable storage. + pub(crate) fn sync(&self) -> io::Result<()> { + self.file.sync_all() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_folder(name: &str) -> std::path::PathBuf { + let folder = std::env::temp_dir().join(format!( + "fabric-record-writer-{name}-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&folder); + std::fs::create_dir_all(&folder).expect("create temp folder"); + folder + } + + #[test] + fn record_writer_refuses_an_existing_file_or_a_symlink() { + let folder = temp_folder("excl"); + let path = folder.join(RECORD_FILE_NAME); + std::fs::write(&path, "old").expect("seed file"); + + let error = RecordWriter::create(&path).expect_err("existing file"); + + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!(std::fs::read_to_string(&path).expect("read"), "old"); + #[cfg(unix)] + { + let linked = folder.join("linked"); + std::fs::create_dir_all(&linked).expect("linked folder"); + let target = folder.join("target"); + std::os::unix::fs::symlink(&target, linked.join(RECORD_FILE_NAME)).expect("symlink"); + assert!(RecordWriter::create(&linked.join(RECORD_FILE_NAME)).is_err()); + assert!(!target.exists()); + } + let _ = std::fs::remove_dir_all(folder); + } + + #[cfg(unix)] + #[test] + fn record_writer_new_file_is_owner_only() { + use std::os::unix::fs::PermissionsExt; + let folder = temp_folder("mode"); + let path = folder.join(RECORD_FILE_NAME); + + RecordWriter::create(&path).expect("create"); + let mode = std::fs::metadata(&path).expect("stat").permissions().mode(); + + assert_eq!(mode & 0o777, 0o600); + let _ = std::fs::remove_dir_all(folder); + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index ab16add07..fce59395e 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -31,9 +31,12 @@ use crate::config::{ validate_tool_definitions, validate_workflow, }; use crate::error::{FabricError, Result}; +use crate::record::{InvocationRecorder, RecordStart}; static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; +/// Lifecycle error code for an adapter result that core rejects. +pub(crate) const INVALID_AGENT_RUN_RESULT: &str = "invalid_agent_run_result"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; const LOCAL_HOST_START_TIMEOUT: Duration = Duration::from_secs(90); // Protocol liveness backstop. Adapters remain responsible for normal request @@ -1726,11 +1729,38 @@ impl LocalHostInvocation { } fn run_local_host_invocation_with_timeout( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, + invocation_kind: LocalHostInvocation, + invoke_timeout: Duration, +) -> Result { + let mut recorder = None; + let outcome = local_host_invocation_result( + plan, + runtime, + request, + invocation_kind, + invoke_timeout, + &mut recorder, + ); + match recorder { + // The end line and its sync run here, after the host lock is released. + Some(recorder) => recorder.finish(outcome), + None => outcome, + } +} + +/// Run one invocation on the local host. When the plan turns the run record +/// on, `recorder` is set right before core sends the request to the host, so +/// every invocation that core tried to send has a record. +fn local_host_invocation_result( plan: &RunPlan, runtime: &RuntimeHandle, mut request: RunRequest, invocation_kind: LocalHostInvocation, invoke_timeout: Duration, + recorder: &mut Option, ) -> Result { let operation = invocation_kind.operation(); if request.request_id.is_empty() { @@ -1806,6 +1836,16 @@ fn run_local_host_invocation_with_timeout( } }; let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; + if plan.config.runtime.record_enabled() && artifacts.root.is_some() { + *recorder = Some(InvocationRecorder::start(RecordStart { + plan, + runtime, + invocation: &invocation, + request: &request, + fabric_home: &fabric_home, + timeout: invoke_timeout, + })); + } match exchange_lifecycle_message( &mut host_guard, &runtime.runtime_id, @@ -1851,7 +1891,7 @@ fn run_local_host_invocation_with_timeout( lifecycle_error( operation, &runtime.runtime_id, - "invalid_agent_run_result", + INVALID_AGENT_RUN_RESULT, format!("adapter returned an invalid AgentRunResult: {error}"), &stderr, ) @@ -1860,7 +1900,7 @@ fn run_local_host_invocation_with_timeout( lifecycle_error( operation, &runtime.runtime_id, - "invalid_agent_run_result", + INVALID_AGENT_RUN_RESULT, format!("adapter returned an invalid AgentRunResult: {error}"), &stderr, ) @@ -1870,7 +1910,7 @@ fn run_local_host_invocation_with_timeout( return Err(lifecycle_error( operation, &runtime.runtime_id, - "invalid_agent_run_result", + INVALID_AGENT_RUN_RESULT, "adapter returned artifacts without a configured runtime artifact root", &stderr, )); @@ -3297,7 +3337,7 @@ fn relay_artifact_media_type(kind: &str) -> Option<&'static str> { } } -fn unique_artifact_name(manifest: &ArtifactManifest, base: &str) -> String { +pub(crate) fn unique_artifact_name(manifest: &ArtifactManifest, base: &str) -> String { if !manifest .artifacts .iter() @@ -3658,7 +3698,7 @@ fn telemetry_ref( }) } -fn event_with_metadata( +pub(crate) fn event_with_metadata( kind: impl Into, message: impl Into, metadata: BTreeMap, @@ -3947,12 +3987,16 @@ for line in sys.stdin: "context_env": invocation["runtime_context"]["environment"].get("env", {}), "deadline_millis": invocation["runtime_context"].get("deadline_millis"), } - if MODE in {"adapter_reported_failure", "blank_error_code"}: + if MODE in {"adapter_reported_failure", "blank_error_code", "free_text_error_code"}: + codes = { + "blank_error_code": " \t", + "free_text_error_code": "Rate limit for key sk-live-123 exceeded", + } result = { "status": "failed", "output": output, "error": { - "code": " \t" if MODE == "blank_error_code" else "fake_adapter_failure", + "code": codes.get(MODE, "fake_adapter_failure"), "message": "adapter rejected the invocation", "retryable": True, "extensions": {"source": "fake-host"}, @@ -5458,4 +5502,247 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + + fn record_plan(mode: &str) -> (PathBuf, RunPlan) { + let (root, mut plan) = local_host_plan(mode); + plan.config.runtime.record = Some(crate::config::RecordConfig { enabled: true }); + (root, plan) + } + + fn record_files(root: &Path) -> Vec { + let mut records = Vec::new(); + let mut folders = vec![root.join("artifacts/.fabric")]; + while let Some(folder) = folders.pop() { + let Ok(entries) = fs::read_dir(&folder) else { + continue; + }; + for path in entries.flatten().map(|entry| entry.path()) { + if path.is_dir() { + folders.push(path); + } else if path.ends_with("run-record.jsonl") { + records.push(path); + } + } + } + records + } + + fn record_line_validator() -> jsonschema::Validator { + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunRecordLine) + .expect("record line schema"); + jsonschema::validator_for(&schema).expect("valid schema") + } + + /// The only record under `root`, with every line checked against the + /// published line schema. + fn only_record(root: &Path) -> (String, Vec) { + let records = record_files(root); + assert_eq!(records.len(), 1, "{records:?}"); + let text = fs::read_to_string(&records[0]).expect("read record"); + let validator = record_line_validator(); + let lines: Vec = text + .lines() + .map(|line| serde_json::from_str(line).expect("record line is JSON")) + .collect(); + for line in &lines { + assert!(validator.is_valid(line), "{line}"); + } + (text, lines) + } + + /// The one line of `kind`. Lookup by kind keeps these tests independent + /// of other line kinds written between the start and end lines. + fn line_of_kind<'a>(lines: &'a [Value], kind: &str) -> &'a Value { + let mut matching = lines.iter().filter(|line| line["kind"] == kind); + let line = matching.next().unwrap_or_else(|| panic!("no {kind} line")); + assert!(matching.next().is_none(), "more than one {kind} line"); + line + } + + #[test] + fn record_off_with_an_artifact_root_writes_no_record() { + let (root, plan) = local_host_plan("typed_result"); + assert!(plan.config.runtime.artifacts.is_some()); + + let result = run_plan(&plan, RunRequest::text("no record")).expect("run"); + + assert!(record_files(&root).is_empty()); + assert!( + result + .artifacts + .artifacts + .iter() + .all(|artifact| artifact.kind != "run_record") + ); + assert!( + result + .events + .iter() + .all(|event| event.kind != "record_write_failed") + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_on_writes_a_start_line_and_an_end_line() { + let (root, plan) = record_plan("typed_result"); + let mut request = RunRequest::text("record"); + request.context = BTreeMap::from([ + ("task_id".to_string(), serde_json::json!("repo-io-fix")), + ("attempt".to_string(), serde_json::json!(2)), + ( + "harbor_trial_name".to_string(), + serde_json::json!("trial-1"), + ), + ("step".to_string(), serde_json::json!({"nested": "value"})), + ("api_key".to_string(), serde_json::json!("sk-live-SECRET")), + ]); + + let result = run_plan(&plan, request).expect("run"); + let (text, lines) = only_record(&root); + + let (start, end) = (line_of_kind(&lines, "start"), line_of_kind(&lines, "end")); + assert_eq!(lines.first(), Some(start), "{text}"); + assert_eq!(lines.last(), Some(end), "{text}"); + for line in &lines { + assert_eq!(line["schema"], "nemo.fabric.run_record"); + assert_eq!(line["version"], "v1alpha1"); + assert_eq!(line["runtime_id"], result.runtime_id.as_str()); + assert_eq!(line["invocation_id"], result.invocation_id.as_str()); + assert_eq!(line["request_id"], result.request_id.as_str()); + } + assert_eq!(start["agent_name"], "local-host-test-agent"); + assert_eq!(start["adapter_id"], "acme.fabric.local-host"); + assert_eq!(start["timeout_ms"], 3_600_000); + assert_eq!( + start["context"], + serde_json::json!({ + "api_key": {"recorded": false}, + "attempt": 2, + "harbor_trial_name": "trial-1", + "step": {"recorded": false}, + "task_id": "repo-io-fix", + }) + ); + assert!(!text.contains("SECRET"), "{text}"); + assert_eq!(end["status"], "succeeded"); + assert!(end["elapsed_ms"].is_u64()); + assert_eq!( + end["usage"], + serde_json::json!({ + "input_tokens": 3, + "cached_input_tokens": 1, + "input_tokens_include_cache": true, + "output_tokens": 5, + "total_tokens": 8, + "cost_usd": 0.25, + }) + ); + let report = end["artifacts"] + .as_array() + .expect("artifacts") + .iter() + .find(|artifact| artifact["name"] == "report") + .expect("report artifact"); + assert_eq!( + report, + &serde_json::json!({ + "name": "report", + "kind": "report", + "media_type": "application/json", + "path": "reports/result.json", + }) + ); + for marker in ["fake-host", "\"fake\"", "trace-1"] { + assert!(!text.contains(marker), "{marker} in {text}"); + } + let record = result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.name == "record") + .expect("record artifact"); + assert_eq!(record.kind, "run_record"); + assert_eq!(record.media_type.as_deref(), Some("application/jsonl")); + assert_eq!(record.path, record_files(&root)[0]); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_end_line_keeps_the_error_code_and_drops_the_message() { + let (root, plan) = record_plan("adapter_reported_failure"); + + let result = run_plan(&plan, RunRequest::text("fail")).expect("failed result"); + let (text, lines) = only_record(&root); + + assert_eq!(result.status, RunStatus::Failed); + assert_eq!( + line_of_kind(&lines, "end")["error"], + serde_json::json!({"code": "fake_adapter_failure", "stage": "invoke", "retryable": true}) + ); + assert!(!text.contains("adapter rejected"), "{text}"); + let mut wrong_kind = line_of_kind(&lines, "end").clone(); + wrong_kind["kind"] = serde_json::json!("start"); + assert!(!record_line_validator().is_valid(&wrong_kind)); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_end_line_replaces_a_free_text_error_code() { + let (root, plan) = record_plan("free_text_error_code"); + + let result = run_plan(&plan, RunRequest::text("fail")).expect("failed result"); + let (text, lines) = only_record(&root); + + assert!(result.error.expect("error").code.contains("sk-live-123")); + assert_eq!(line_of_kind(&lines, "end")["error"]["code"], "unrecognized"); + assert!(!text.contains("sk-live-123"), "{text}"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_lifecycle_error_writes_a_failed_end_line() { + let (root, plan) = record_plan("invoke_timeout"); + let runtime = start_runtime(&plan).expect("start local host"); + + let error = run_local_host_adapter_with_timeout( + &plan, + &runtime, + RunRequest::text("hang"), + Duration::from_millis(100), + ) + .expect_err("timeout raises"); + let (_, lines) = only_record(&root); + + assert!(error.to_string().contains("host_timeout"), "{error}"); + let end = line_of_kind(&lines, "end"); + assert_eq!(line_of_kind(&lines, "start")["timeout_ms"], 100); + assert_eq!(end["status"], "failed"); + assert_eq!( + end["error"], + serde_json::json!({"code": "host_timeout", "stage": "invoke", "retryable": false}) + ); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_warm_runtime_writes_one_record_per_invocation() { + let (root, plan) = record_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + + let first = invoke_runtime(&plan, &runtime, RunRequest::text("one")).expect("first"); + let second = invoke_runtime(&plan, &runtime, RunRequest::text("two")).expect("second"); + stop_runtime(&plan, &runtime).expect("stop"); + + let records = record_files(&root); + assert_eq!(records.len(), 2); + for result in [&first, &second] { + assert!(records.iter().any(|path| path.ends_with(format!( + ".fabric/{}/{}/run-record.jsonl", + result.runtime_id, result.invocation_id + )))); + } + let _ = fs::remove_dir_all(root); + } } diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index ee15fda12..aefc27b27 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -13,6 +13,7 @@ use crate::config::{ AdapterDescriptor, AdapterTargetDescriptor, AgentConfig, FabricConfig, RunPlan, }; use crate::error::{FabricError, Result}; +use crate::record::RecordLine; use crate::runtime::{ AdapterInvocation, ArtifactManifest, EnvironmentHandle, ErrorInfo, FabricEvent, InvocationHandle, OpenAiStreamInvocation, OpenAiStreamRecord, RunRequest, RunResult, @@ -65,11 +66,13 @@ pub enum SchemaName { ErrorInfo, /// NeMo Fabric lifecycle event schema. FabricEvent, + /// One line of an opt-in per-invocation run record, version `v1alpha1`. + RunRecordLine, } impl SchemaName { /// All public schemas in stable output order. - pub const ALL: [Self; 21] = [ + pub const ALL: [Self; 22] = [ Self::Agent, Self::AgentConfig, Self::AgentRunRequest, @@ -91,6 +94,7 @@ impl SchemaName { Self::ArtifactManifest, Self::ErrorInfo, Self::FabricEvent, + Self::RunRecordLine, ]; /// Stable file stem for this schema. @@ -117,6 +121,7 @@ impl SchemaName { Self::ArtifactManifest => "artifact-manifest", Self::ErrorInfo => "error-info", Self::FabricEvent => "fabric-event", + Self::RunRecordLine => "run-record-line", } } @@ -170,6 +175,7 @@ impl SchemaName { "artifact-manifest" | "artifact_manifest" => Ok(Self::ArtifactManifest), "error-info" | "error_info" => Ok(Self::ErrorInfo), "fabric-event" | "fabric_event" => Ok(Self::FabricEvent), + "run-record-line" | "run_record_line" => Ok(Self::RunRecordLine), _ => Err(FabricError::UnknownSchema { schema: value.to_string(), available: Self::ALL @@ -205,6 +211,7 @@ pub fn generate_schema(schema: SchemaName) -> Result { SchemaName::ArtifactManifest => to_value(schema_for!(ArtifactManifest)), SchemaName::ErrorInfo => to_value(schema_for!(ErrorInfo)), SchemaName::FabricEvent => to_value(schema_for!(FabricEvent)), + SchemaName::RunRecordLine => to_value(schema_for!(RecordLine)), } } diff --git a/crates/fabric-core/src/time.rs b/crates/fabric-core/src/time.rs new file mode 100644 index 000000000..06ace9edf --- /dev/null +++ b/crates/fabric-core/src/time.rs @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! RFC 3339 UTC timestamps with millisecond precision. +//! +//! Core has no date crate, so this converts Unix milliseconds to a civil date +//! with Howard Hinnant's `civil_from_days` algorithm. + +use std::time::{SystemTime, UNIX_EPOCH}; + +const MILLIS_PER_DAY: i64 = 86_400_000; + +/// Format `time` as RFC 3339 UTC with three fraction digits, such as +/// `2026-10-14T18:01:10.205Z`. +pub(crate) fn rfc3339_millis(time: SystemTime) -> String { + format_unix_millis(unix_millis(time)) +} + +/// Milliseconds since the Unix epoch; negative before it. +fn unix_millis(time: SystemTime) -> i64 { + match time.duration_since(UNIX_EPOCH) { + Ok(after) => i64::try_from(after.as_millis()).unwrap_or(i64::MAX), + Err(before) => -i64::try_from(before.duration().as_millis()).unwrap_or(i64::MAX), + } +} + +fn format_unix_millis(unix_millis: i64) -> String { + let days = unix_millis.div_euclid(MILLIS_PER_DAY); + let millis_of_day = unix_millis.rem_euclid(MILLIS_PER_DAY); + let (year, month, day) = civil_from_days(days); + let seconds_of_day = millis_of_day / 1000; + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{:03}Z", + seconds_of_day / 3600, + seconds_of_day / 60 % 60, + seconds_of_day % 60, + millis_of_day % 1000, + ) +} + +fn civil_from_days(days: i64) -> (i64, i64, i64) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + (year, month, day) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn rfc3339_millis_formats_fixed_vectors() { + assert_eq!(format_unix_millis(0), "1970-01-01T00:00:00.000Z"); + assert_eq!( + format_unix_millis(1_792_000_870_205), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!( + format_unix_millis(951_782_400_000), + "2000-02-29T00:00:00.000Z" + ); + assert_eq!( + format_unix_millis(4_107_542_399_999), + "2100-02-28T23:59:59.999Z" + ); + assert_eq!(format_unix_millis(-1), "1969-12-31T23:59:59.999Z"); + assert_eq!( + rfc3339_millis(UNIX_EPOCH + Duration::from_millis(1_792_000_870_205)), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!( + rfc3339_millis(UNIX_EPOCH - Duration::from_millis(1)), + "1969-12-31T23:59:59.999Z" + ); + } +} diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index c1bdd6e0c..06d04bb04 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -42,6 +42,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`models.McpServerConfig`](./nemo_fabric.models.md#class-mcpserverconfig): MCP server configuration. - [`models.MetadataConfig`](./nemo_fabric.models.md#class-metadataconfig): Human-readable agent identity. - [`models.ModelConfig`](./nemo_fabric.models.md#class-modelconfig): Configuration for one model role. +- [`models.RecordConfig`](./nemo_fabric.models.md#class-recordconfig): Opt-in per-invocation run record. - [`models.RelayAtifConfig`](./nemo_fabric.models.md#class-relayatifconfig): NeMo Relay ATIF export configuration. - [`models.RelayAtofConfig`](./nemo_fabric.models.md#class-relayatofconfig): NeMo Relay ATOF export configuration. - [`models.RelayAtofFileSinkConfig`](./nemo_fabric.models.md#class-relayatoffilesinkconfig): NeMo Relay ATOF file sink configuration. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index db3a42208..703cff7b4 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -472,6 +472,78 @@ Returns the set of fields that have been explicitly set on this model instance. +**Returns:** + A set of strings representing the fields that have been set, i.e. that were not filled from defaults. + + + +--- + + +### classmethod `from_mapping` + +```python +def from_mapping(value: Mapping[str, Any]) -> Self +``` + +Validate a mapping using this Pydantic model. + +--- + + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached JSON-compatible mapping for Rust/core calls. + + +--- + + +## class `RecordConfig` + +Opt-in per-invocation run record. + +When ``enabled`` is true, NeMo Fabric writes ``/.fabric///run-record.jsonl`` for each invocation. It requires an artifact root, the ``local`` environment provider, and a ``process`` or ``python`` adapter. + + + +### Fields + +The model defines the following fields: + +| Field | Type | Required | Default | Constraints | Description | +| --- | --- | --- | --- | --- | --- | +| `enabled` | `StrictBool` | Yes | — | `Strict(strict=True)` | — | + +--- + +### property extra_fields + +Return fields preserved by the extension point for this model. + +--- + +### property model_extra + +Get extra fields set during validation. + + + +**Returns:** + A dictionary of extra fields, or `None` if `config.extra` is not set to `"allow"`. + +--- + +### property model_fields_set + +Returns the set of fields that have been explicitly set on this model instance. + + + **Returns:** A set of strings representing the fields that have been set, i.e. that were not filled from defaults. @@ -520,6 +592,7 @@ The model defines the following fields: | `artifacts` | `str \| Path \| None` | No | `None` | — | — | | `timeout_seconds` | `float \| None` | No | `None` | `Gt(gt=0), _PydanticGeneralMetadata(allow_inf_nan=False)` | — | | `max_turns` | `int \| None` | No | `None` | `Gt(gt=0), Le(le=4294967295)` | — | +| `record` | `RecordConfig \| None` | No | `None` | — | — | --- diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 52a3b6f57..feabb8df6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 125 +position: 126 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 26a286a0c..4c4e21b44 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 126 +position: 127 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 2c89a0e39..de4682271 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 127 +position: 128 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx index 40e387ace..65c0f3eaf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Kind" sidebar-title: "CapabilityKind" description: "Capability kind." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx index e121e6b73..010851195 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Target" sidebar-title: "CapabilityTarget" description: "Component responsible for executing a configured capability." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx index 74dc754f8..b1e6d19ac 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atif Storage Config" sidebar-title: "RelayAtifStorageConfig" description: "Relay ATIF remote storage configuration." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx index bbf8ff9e5..2e4357780 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Mode" sidebar-title: "RelayAtofMode" description: "Relay ATOF file mode." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx index d53f00f8d..d699b5db0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Sink Config" sidebar-title: "RelayAtofSinkConfig" description: "Relay ATOF sink configuration." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx index 892413a9e..9ee176668 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Field Name Policy" sidebar-title: "RelayAtofStreamFieldNamePolicy" description: "Relay ATOF stream field-name policy." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx index ea48753dd..a90cec5f3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Transport" sidebar-title: "RelayAtofStreamTransport" description: "Relay ATOF stream transport." -position: 80 +position: 81 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx index beb3da6eb..ba9321f6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Endpoint Type" sidebar-title: "RelayOpenTelemetryEndpointType" description: "Relay OpenTelemetry span projection." -position: 81 +position: 82 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx index da13057aa..9b78e9a7e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Mark Projection" sidebar-title: "RelayOpenTelemetryMarkProjection" description: "Relay OpenTelemetry mark projection policy." -position: 82 +position: 83 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx index 8db6e0872..a73f2ad1b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Otlp Transport" sidebar-title: "RelayOtlpTransport" description: "Relay OTLP transport." -position: 83 +position: 84 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx index 95ec26616..506a2c506 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Unsupported Behavior" sidebar-title: "RelayUnsupportedBehavior" description: "Relay unsupported/unknown config handling." -position: 84 +position: 85 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx index f130d1764..7e6f001b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx @@ -2,7 +2,7 @@ title: "Enum Resolution Strategy" sidebar-title: "ResolutionStrategy" description: "Adapter install or availability strategy." -position: 54 +position: 55 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx index 8ce5c9908..d1bfc189f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx @@ -2,7 +2,7 @@ title: "Enum Telemetry Provider" sidebar-title: "TelemetryProvider" description: "Telemetry runtime provider." -position: 64 +position: 65 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx index fc493bae7..de34aedc1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx @@ -2,7 +2,7 @@ title: "Function load_adapter_descriptor" sidebar-title: "load_adapter_descriptor" description: "Load an adapter descriptor from JSON package metadata." -position: 71 +position: 72 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx index 142fdba30..bd5cd97d5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx @@ -2,7 +2,7 @@ title: "Function load_adapter_target_descriptor" sidebar-title: "load_adapter_target_descriptor" description: "Load an Adapter Target Descriptor from declarative JSON metadata." -position: 72 +position: 73 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx index 6bc8a973b..a6d607703 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx @@ -2,7 +2,7 @@ title: "Function resolve_run_plan_from_config" sidebar-title: "resolve_run_plan_from_config" description: "Resolve a typed NVIDIA NeMo Fabric config into a runnable plan." -position: 73 +position: 74 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index fa7255361..7f45f6f6e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 128 +position: 129 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -53,6 +53,7 @@ NeMo Fabric config models and loading helpers. - [McpServerPlan](struct-mcpserverplan.mdx): Resolved MCP server exposure. - [MetadataConfig](struct-metadataconfig.mdx): Human-readable metadata. - [ModelConfig](struct-modelconfig.mdx): Model configuration. +- [RecordConfig](struct-recordconfig.mdx): Per-invocation run record settings. - [RelayAtifConfig](struct-relayatifconfig.mdx): Relay ATIF export configuration. - [RelayAtofConfig](struct-relayatofconfig.mdx): Relay ATOF export configuration. - [RelayComponentConfig](struct-relaycomponentconfig.mdx): Generic NeMo Relay plugin component configuration. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx new file mode 100644 index 000000000..d33e42889 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx @@ -0,0 +1,108 @@ +--- +title: "Struct Record Config" +sidebar-title: "RecordConfig" +description: "Per-invocation run record settings." +position: 54 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
bool,\n}"}} />
+ +Per-invocation run record settings. + +When enabled, core appends a start line and an end line for each local invocation to `/.fabric///run-record.jsonl`. + +## Fields + +### `enabled: bool` + +Whether to write a run record for each invocation. Requires an artifact root, the `local` environment provider, and a `process` or `python` adapter. + +## Trait Implementations + +### `impl Clone for RecordConfig` + +
Clone for RecordConfig"}} />
+ +#### `clone` + +
clone(&self) -> RecordConfig"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RecordConfig` + +
Debug for RecordConfig"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for RecordConfig` + +
Default for RecordConfig"}} />
+ +#### `default` + +
default() -> RecordConfig"}} />
+ +### `impl<'de> Deserialize<'de> for RecordConfig` + +
Deserialize<'de> for RecordConfig"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RecordConfig` + +
RecordConfig"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RecordConfig` + +
PartialEq for RecordConfig"}} />
+ +#### `eq` + +
eq(&self, other: &RecordConfig) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RecordConfig` + +
Serialize for RecordConfig"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for RecordConfig` + +
Eq for RecordConfig"}} />
+ +### `impl StructuralPartialEq for RecordConfig` + +
StructuralPartialEq for RecordConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx index 4a2989ac6..030a4323a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Atif Config" sidebar-title: "RelayAtifConfig" description: "Relay ATIF export configuration." -position: 38 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx index 4ff5d7de9..d595a140a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Atof Config" sidebar-title: "RelayAtofConfig" description: "Relay ATOF export configuration." -position: 39 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx index 56259d7a5..4c490558c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Component Config" sidebar-title: "RelayComponentConfig" description: "Generic NeMo Relay plugin component configuration." -position: 40 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx index 570ef8d8e..ffc52a18c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Config" sidebar-title: "RelayConfig" description: "NeMo Relay integration configuration." -position: 41 +position: 42 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx index 712c72795..8d9e4a386 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Config Policy" sidebar-title: "RelayConfigPolicy" description: "Relay validation policy." -position: 42 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx index a01db8716..4db7528da 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Observability Config" sidebar-title: "RelayObservabilityConfig" description: "NeMo Relay observability component configuration." -position: 43 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx index 4bade1462..922156d9f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Open Telemetry Config" sidebar-title: "RelayOpenTelemetryConfig" description: "Relay OpenTelemetry export section." -position: 44 +position: 45 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx index da8138f4e..6bb243f6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Open Telemetry Endpoint Config" sidebar-title: "RelayOpenTelemetryEndpointConfig" description: "One typed Relay OpenTelemetry OTLP destination." -position: 45 +position: 46 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx index 8e70ade88..bf1537e85 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Resolve Context" sidebar-title: "ResolveContext" description: "Source context used when resolving an in-memory NeMo Fabric config." -position: 55 +position: 56 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx index 7c1e65663..34ad4c681 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx @@ -2,7 +2,7 @@ title: "Struct Resolved Adapter Descriptor" sidebar-title: "ResolvedAdapterDescriptor" description: "Adapter descriptor selected for a run plan." -position: 56 +position: 57 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx index 8adac3efc..b3be2d837 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx @@ -2,7 +2,7 @@ title: "Struct Resolved Adapter Target Descriptor" sidebar-title: "ResolvedAdapterTargetDescriptor" description: "Adapter target descriptor selected for a run plan." -position: 57 +position: 58 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx index e3c43e4e1..b8f75e62d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx @@ -2,7 +2,7 @@ title: "Struct RunPlan" sidebar-title: "RunPlan" description: "Resolved NeMo Fabric run plan." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx index 0d972cb87..5c2282536 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Capabilities" sidebar-title: "RuntimeCapabilities" description: "Lifecycle behavior implemented by a resolved runtime path." -position: 59 +position: 60 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx index 2c89a11b2..d9b6e6015 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Runtime Config" sidebar-title: "RuntimeConfig" description: "Invocation runtime contract." -position: 60 +position: 61 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub record: Option<RecordConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Invocation runtime contract. @@ -35,6 +35,10 @@ Maximum duration of one invocation in seconds. Maximum number of harness turns within one invocation. +### `record: Option` + +Opt-in per-invocation run record. Core-only; never sent to adapters. + ### `extensions: BTreeMap` Additive normalized runtime fields. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx index 47644f228..1f4e2fe76 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Skill Config" sidebar-title: "SkillConfig" description: "Skill capability configuration." -position: 61 +position: 62 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx index 2e1b4af6b..61e3aa7d0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Config" sidebar-title: "TelemetryConfig" description: "Telemetry configuration." -position: 62 +position: 63 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx index f12176331..326bcdbb5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Plan" sidebar-title: "TelemetryPlan" description: "Resolved telemetry plan." -position: 63 +position: 64 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx index f683b0886..bef871340 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Provider Config" sidebar-title: "TelemetryProviderConfig" description: "Provider-specific telemetry configuration." -position: 65 +position: 66 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx index b5e0b460f..771156fff 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Tool Definition Config" sidebar-title: "ToolDefinitionConfig" description: "One named normalized tool or tool-group definition." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx index 64b2d6eec..f9f3714b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Tools Config" sidebar-title: "ToolsConfig" description: "Harness-neutral tool capability configuration." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx index d2b51fe8b..ffe368f07 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx @@ -2,7 +2,7 @@ title: "Struct Tools Plan" sidebar-title: "ToolsPlan" description: "Normalized tool policy for a run." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx index 21beec0df..ab0796f56 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Workflow Config" sidebar-title: "WorkflowConfig" description: "Registered workflow target and immutable construction settings." -position: 68 +position: 69 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx index a002d8fc3..03fecb115 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Workflow Entrypoint Config" sidebar-title: "WorkflowEntrypointConfig" description: "Adapter-owned workflow entry point supplied by an Adapter Target Descriptor." -position: 69 +position: 70 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx index e66041abb..a0e9695a1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx @@ -2,7 +2,7 @@ title: "Struct Workflow Target Spec" sidebar-title: "WorkflowTargetSpec" description: "Workflow-specific Adapter Target Descriptor fields." -position: 70 +position: 71 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx index d8c0693d4..bbae2fe19 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx @@ -2,7 +2,7 @@ title: "Enum Doctor Status" sidebar-title: "DoctorStatus" description: "Diagnostic status." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx index bb8aa23e8..b97461157 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx @@ -2,7 +2,7 @@ title: "Function doctor_plan" sidebar-title: "doctor_plan" description: "Inspect a resolved run plan without mutating the environment." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index ea299ed94..4f6f26b00 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 129 +position: 130 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx index c64d5792e..9ae58d2b6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx @@ -2,7 +2,7 @@ title: "Struct Doctor Check" sidebar-title: "DoctorCheck" description: "Diagnostic check result." -position: 74 +position: 75 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx index c7d65e76f..c015cc391 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx @@ -2,7 +2,7 @@ title: "Struct Doctor Report" sidebar-title: "DoctorReport" description: "Diagnostic report for a resolved run plan." -position: 75 +position: 76 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 95cf2f59a..70e1c1cc4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -2,7 +2,7 @@ title: "Enum Fabric Error" sidebar-title: "FabricError" description: "Errors raised by NeMo Fabric config loading and validation." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index cf7208b32..e389cbb01 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 130 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx index 0a91b0129..6f11a50d8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx @@ -2,7 +2,7 @@ title: "Type Alias Result" sidebar-title: "Result" description: "Core NeMo Fabric result type." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 8af97f312..f9c667fa9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 133 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 6670ab4d8..94890397c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -66,6 +66,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use config::MetadataConfig;` - `pub use config::ModelConfig;` - `pub use config::OAuthTokenEndpointAuthMethod;` +- `pub use config::RecordConfig;` - `pub use config::ResolutionStrategy;` - `pub use config::ResolveContext;` - `pub use config::ResolvedAdapterDescriptor;` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index e4ce802a1..3b8f07380 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 131 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index 9ae563e23..041dbb1fa 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -32,6 +32,7 @@ pub enum SchemaName { ArtifactManifest, ErrorInfo, FabricEvent, + RunRecordLine, } ``` @@ -165,6 +166,12 @@ Structured error metadata schema. NeMo Fabric lifecycle event schema. +### `RunRecordLine` + +
+ +One line of an opt-in per-invocation run record, version `v1alpha1`. + ## Implementations ### `impl SchemaName` @@ -173,7 +180,7 @@ NeMo Fabric lifecycle event schema. #### `ALL` -
21]"}} />
+
22]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 0569b34b2..722c3a849 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 132 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 227e0bd44..555e6ba5c 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -237,7 +237,7 @@ Relay subfields: | `models..top_p`, `.max_tokens` | No | No | Yes | Yes | Yes; passed through LiteLLM | | `instructions.system` | `replace`, `append` | `replace`; maps to Codex base instructions | `replace` | `replace` | `replace` | | `runtime.input_schema`, `.output_schema` | Core | Core | Core | Core | Core | -| `runtime.artifacts`, `.timeout_seconds` | Core | Core | Core | Core | Core | +| `runtime.artifacts`, `.timeout_seconds`, `.record` | Core | Core | Core | Core | Core | | `runtime.max_turns` | Yes | No | Yes; maps to LangGraph supersteps | Yes; maps to Hermes iterations | Yes | | `environment.provider`, `.control_location`, `.ownership` | Core | Core | Core | Core | Core | | `environment.workspace`, `.artifacts`, `.env`, `.env_names` | Core | Core | Core | Core | Core | @@ -941,6 +941,78 @@ If NeMo Fabric cannot resolve config, start a runtime, or obtain a normalized re the SDK raises a `FabricError` subclass instead of returning a partial `RunResult`. +## Run Records + +A run record is an append-only JSON Lines file that NeMo Fabric writes for each +invocation, so a run that is killed still leaves its identity, start time, and +timeout behind. Recording is off by default. Setting `runtime.artifacts` alone +does not turn it on. Turn it on with `runtime.record`: + +```python +from nemo_fabric import RecordConfig, RuntimeConfig + +runtime = RuntimeConfig(artifacts="runs", record=RecordConfig(enabled=True)) +``` + +`enabled` is required inside `record`. The record needs an artifact root +(`runtime.artifacts` or `environment.artifacts`), the `local` environment +provider, and a `process` or `python` adapter. Planning fails with +`runtime.record.enabled` when any of these is missing. + +Each invocation that NeMo Fabric tries to send to the adapter host writes +`/.fabric///run-record.jsonl`, and the +result lists it as the `record` artifact with kind `run_record`: + +```python +record = next(a for a in result.artifacts.artifacts if a.kind == "run_record") +``` + +The file holds a start line and an end line. Every line carries `schema` +(`nemo.fabric.run_record`), `version` (`v1alpha1`), `kind`, `runtime_id`, +`invocation_id`, `request_id`, and `time`. + +- The `start` line is written before the request is sent. It holds the agent + name, harness, adapter ID, each configured model's provider and model name, + the NeMo Fabric version, and the timeout in milliseconds. From + `RunRequest.context`, it keeps the string or integer values of `task_id`, + `task_revision`, `attempt`, `step`, `harbor_session_id`, `harbor_context_id`, + and `harbor_trial_name`, cut to 256 characters. Every other context key + appears by name only as `{"recorded": false}`. `override_keys` lists the + top-level `RunRequest.overrides` key names, sorted, without their values. +- The `end` line is written after the invocation returns or fails. It holds + `status`, `elapsed_ms`, the error's `code`, `stage`, and `retryable` flag, + the numeric `RunUsage` counters, and each result artifact's name, kind, + media type, and path relative to the artifact root. An error code outside + `^[a-z0-9_.:-]{1,128}$` is written as `unrecognized`. When NeMo Fabric + raises instead of returning a result, the end line reads `failed`. An + adapter host error keeps its code, such as `host_timeout`, `host_crashed`, + `host_io`, or `invalid_agent_run_result`. A result extension that the + adapter's descriptor does not declare also reads `invalid_agent_run_result`. + A failed artifact write reads `artifact_write_failed`, and any other error + reads `fabric_error`. + +The end line describes the invocation. A one-shot `Fabric.run` stops the +runtime after the invocation; when that stop fails, the result reads `failed` +with stage `stop`, and the record still holds the invocation's own end line. + +`run-record.jsonl` never holds prompts, model output, error messages, tool +payloads, adapter metadata, or context values outside the keys above. The same +folder also holds `adapter-invocation.json` and `stdout.txt`, which NeMo Fabric +writes for every invocation whether or not recording is on; those files are +not covered by this rule. + +Each line is written with one append, so a killed process keeps every line +whose write returned. NeMo Fabric syncs the file once, after the end line. A +record without an end line belongs to an invocation that is still running or +that never finished; treat its outcome as unknown, not as failed. If a write +fails, the record stops, the result gains a `record_write_failed` event, and +the invocation continues unchanged. A record whose end line failed to write +therefore has no end line, and that event says why. The file is created with +mode `0600` on Linux and macOS, and never through a symbolic link. + +The line format is published as `schemas/sdk/run-record-line.schema.json` at +version `v1alpha1`, which may change until a reader for run records ships. + ## Events Each `RunResult` includes the normalized events collected for that invocation. diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 43867290c..af78b2daf 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -127,6 +127,9 @@ descriptor schema. - `sdk/artifact-manifest`: normalized artifact references. - `sdk/error-info`: structured runtime or adapter error metadata. - `sdk/fabric-event`: NeMo Fabric lifecycle or progress event. +- `sdk/run-record-line`: one line of the opt-in per-invocation run record, + `run-record.jsonl`. The schema is `v1alpha1` and may change until a reader + for run records ships. ### Deferred Core Objects diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index 720929df3..7d6c5e994 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -558,6 +558,20 @@ } ] }, + "RecordConfig": { + "additionalProperties": false, + "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "properties": { + "enabled": { + "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -1329,6 +1343,17 @@ "description": "Output schema label.", "type": "string" }, + "record": { + "anyOf": [ + { + "$ref": "#/$defs/RecordConfig" + }, + { + "type": "null" + } + ], + "description": "Opt-in per-invocation run record. Core-only; never sent to adapters." + }, "timeout_seconds": { "description": "Maximum duration of one invocation in seconds.", "exclusiveMinimum": 0.0, diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index b33341bf2..5d3211b4e 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -1926,6 +1926,20 @@ } ] }, + "RecordConfig": { + "additionalProperties": false, + "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "properties": { + "enabled": { + "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -2772,6 +2786,17 @@ "description": "Output schema label.", "type": "string" }, + "record": { + "anyOf": [ + { + "$ref": "#/$defs/RecordConfig" + }, + { + "type": "null" + } + ], + "description": "Opt-in per-invocation run record. Core-only; never sent to adapters." + }, "timeout_seconds": { "description": "Maximum duration of one invocation in seconds.", "exclusiveMinimum": 0.0, diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json new file mode 100644 index 000000000..7b0f8642e --- /dev/null +++ b/schemas/sdk/run-record-line.schema.json @@ -0,0 +1,429 @@ +{ + "$defs": { + "ContextValue": { + "anyOf": [ + { + "description": "An allowlisted key's string value, cut to 256 characters.", + "type": "string" + }, + { + "description": "An allowlisted key's integer value.", + "format": "int64", + "type": "integer" + }, + { + "$ref": "#/$defs/NotRecorded", + "description": "Any other key or value: the key name only." + } + ], + "description": "A request context value as the start line keeps it." + }, + "EndArtifact": { + "description": "One result artifact on the end line.", + "properties": { + "kind": { + "description": "Artifact kind.", + "type": "string" + }, + "media_type": { + "description": "Media type.", + "type": [ + "string", + "null" + ] + }, + "name": { + "description": "Logical artifact name.", + "type": "string" + }, + "path": { + "description": "Path relative to the artifact root; absent for a path outside it.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "name", + "kind" + ], + "type": "object" + }, + "EndError": { + "description": "The end line's error.", + "properties": { + "code": { + "description": "Error code matching `^[a-z0-9_.:-]{1,128}$`, else `unrecognized`.", + "pattern": "^[a-z0-9_.:-]{1,128}$", + "type": "string" + }, + "retryable": { + "description": "Whether a consumer-level retry is considered safe.", + "type": "boolean" + }, + "stage": { + "$ref": "#/$defs/ErrorStage", + "description": "Lifecycle stage where the failure surfaced." + } + }, + "required": [ + "code", + "stage", + "retryable" + ], + "type": "object" + }, + "EndLine": { + "description": "Fields of the end line.", + "properties": { + "artifacts": { + "description": "Result artifacts, with paths relative to the artifact root.", + "items": { + "$ref": "#/$defs/EndArtifact" + }, + "type": "array" + }, + "elapsed_ms": { + "description": "Monotonic time from the start line to the end line, in milliseconds.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/EndError" + }, + { + "type": "null" + } + ], + "description": "Error code, stage and retryable flag, without the message." + }, + "status": { + "$ref": "#/$defs/RunStatus", + "description": "Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`." + }, + "usage": { + "anyOf": [ + { + "$ref": "#/$defs/EndUsage" + }, + { + "type": "null" + } + ], + "description": "Numeric usage counters, without adapter metadata." + } + }, + "required": [ + "status", + "elapsed_ms" + ], + "type": "object" + }, + "EndUsage": { + "description": "The end line's usage: the numeric `RunUsage` fields.", + "properties": { + "cached_input_tokens": { + "description": "Cached input tokens.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Reported cost in US dollars.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Input tokens.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether `input_tokens` includes `cached_input_tokens`; absent means unknown.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Output tokens.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Total tokens reported by the provider.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "ErrorStage": { + "description": "NeMo Fabric lifecycle stage associated with an error.", + "oneOf": [ + { + "const": "config", + "description": "Configuration resolution failed.", + "type": "string" + }, + { + "const": "plan", + "description": "Effective config planning failed.", + "type": "string" + }, + { + "const": "prepare", + "description": "Environment preparation failed.", + "type": "string" + }, + { + "const": "start", + "description": "Runtime start/connect failed.", + "type": "string" + }, + { + "const": "invoke", + "description": "Runtime invocation failed.", + "type": "string" + }, + { + "const": "stop", + "description": "Runtime stop/detach failed.", + "type": "string" + }, + { + "const": "release", + "description": "Environment release failed.", + "type": "string" + }, + { + "const": "artifact", + "description": "Artifact collection or writing failed.", + "type": "string" + } + ] + }, + "ModelName": { + "description": "One configured model.", + "properties": { + "model": { + "description": "Provider model identifier.", + "type": "string" + }, + "provider": { + "description": "Model provider name.", + "type": "string" + } + }, + "required": [ + "provider", + "model" + ], + "type": "object" + }, + "NotRecorded": { + "description": "Marks a key whose value the record leaves out.", + "properties": { + "recorded": { + "const": false, + "description": "Always `false`.", + "type": "boolean" + } + }, + "required": [ + "recorded" + ], + "type": "object" + }, + "RecordSchema": { + "description": "The record's schema name.", + "oneOf": [ + { + "const": "nemo.fabric.run_record", + "description": "`nemo.fabric.run_record`.", + "type": "string" + } + ] + }, + "RecordVersion": { + "description": "The record's line format version.", + "oneOf": [ + { + "const": "v1alpha1", + "description": "`v1alpha1`.", + "type": "string" + } + ] + }, + "RunStatus": { + "description": "Runtime completion status.", + "oneOf": [ + { + "const": "succeeded", + "description": "The harness completed successfully.", + "type": "string" + }, + { + "const": "failed", + "description": "The harness completed with a non-zero status.", + "type": "string" + }, + { + "const": "cancelled", + "description": "The invocation or runtime was cancelled.", + "type": "string" + } + ] + }, + "StartLine": { + "description": "Fields of the start line.", + "properties": { + "adapter_id": { + "description": "Adapter implementation id.", + "type": [ + "string", + "null" + ] + }, + "agent_name": { + "description": "Config name (`metadata.name`).", + "type": "string" + }, + "context": { + "additionalProperties": { + "$ref": "#/$defs/ContextValue" + }, + "description": "`RunRequest.context`. Allowlisted identity and Harbor link keys keep a\nstring or integer value; every other key is listed by name only.", + "type": "object" + }, + "fabric_version": { + "description": "NeMo Fabric version that wrote the record.", + "type": "string" + }, + "harness": { + "description": "Harness identifier.", + "type": "string" + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelName" + }, + "description": "Configured models by role: provider and model name only.", + "type": "object" + }, + "override_keys": { + "description": "Top-level key names of `RunRequest.overrides`, sorted. Values are never\nrecorded.", + "items": { + "type": "string" + }, + "type": "array" + }, + "timeout_ms": { + "description": "Host invocation timeout that applied, in milliseconds.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "agent_name", + "harness", + "fabric_version", + "timeout_ms" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "One line of a run record.", + "oneOf": [ + { + "$ref": "#/$defs/StartLine", + "description": "Written once, before core sends the request to the adapter host.", + "properties": { + "kind": { + "const": "start", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "$ref": "#/$defs/EndLine", + "description": "Written once, after the invocation returned a result or failed.", + "properties": { + "kind": { + "const": "end", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ], + "properties": { + "invocation_id": { + "description": "Invocation id.", + "type": "string" + }, + "request_id": { + "description": "Request id.", + "type": "string" + }, + "runtime_id": { + "description": "Runtime id.", + "type": "string" + }, + "schema": { + "$ref": "#/$defs/RecordSchema", + "description": "Always `nemo.fabric.run_record`." + }, + "time": { + "description": "UTC wall-clock time the line was written, RFC 3339 with milliseconds.", + "format": "date-time", + "type": "string" + }, + "version": { + "$ref": "#/$defs/RecordVersion", + "description": "Line format version. `v1alpha1` may change until a record reader ships." + } + }, + "required": [ + "schema", + "version", + "runtime_id", + "invocation_id", + "request_id", + "time" + ], + "title": "RunRecordLine", + "type": "object" +} \ No newline at end of file diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index aaa15bfe0..a585aac9d 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -24,6 +24,7 @@ from nemo_fabric.models import McpServerConfig from nemo_fabric.models import MetadataConfig from nemo_fabric.models import ModelConfig +from nemo_fabric.models import RecordConfig from nemo_fabric.models import RelayAtifConfig from nemo_fabric.models import RelayAtofConfig from nemo_fabric.models import RelayAtofFileSinkConfig @@ -94,6 +95,7 @@ "MetadataConfig", "ModelConfig", "OpenAIInvokeStream", + "RecordConfig", "RelayAtifConfig", "RelayAtofConfig", "RelayAtofFileSinkConfig", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index d72220f03..66ca0ba7d 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -25,6 +25,7 @@ from pydantic import ConfigDict from pydantic import Field from pydantic import SerializerFunctionWrapHandler +from pydantic import StrictBool from pydantic import field_validator from pydantic import model_serializer from pydantic import model_validator @@ -187,6 +188,20 @@ class InstructionsConfig(FabricBaseModel): system: InstructionConfig | None = None +class RecordConfig(FabricBaseModel): + """Opt-in per-invocation run record. + + When ``enabled`` is true, NeMo Fabric writes + ``/.fabric///run-record.jsonl`` + for each invocation. It requires an artifact root, the ``local`` + environment provider, and a ``process`` or ``python`` adapter. + """ + + model_config = ConfigDict(extra="forbid") + + enabled: StrictBool + + class RuntimeConfig(FabricBaseModel): """Invocation runtime contract.""" @@ -199,6 +214,7 @@ class RuntimeConfig(FabricBaseModel): allow_inf_nan=False, ) max_turns: int | None = Field(default=None, gt=0, le=(1 << 32) - 1) + record: RecordConfig | None = None class EnvironmentConfig(FabricBaseModel): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 04504e4dc..0456b68ff 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -372,11 +372,19 @@ class _RuntimeConfig(_ConfigMapping): artifacts: Optional artifact-root path. timeout_seconds: Optional invocation deadline in seconds. max_turns: Optional harness turn limit. + record: Optional run record settings, ``{"enabled": bool}``. extra_fields: Preserved extension fields not recognized by this SDK. """ _fields = frozenset( - {"input_schema", "output_schema", "artifacts", "timeout_seconds", "max_turns"} + { + "input_schema", + "output_schema", + "artifacts", + "timeout_seconds", + "max_turns", + "record", + } ) def __init__( @@ -387,6 +395,7 @@ def __init__( artifacts: str | Path | None = None, timeout_seconds: float | None = None, max_turns: int | None = None, + record: Mapping[str, Any] | None = None, extra_fields: Mapping[str, Any] | None = None, ) -> None: values: dict[str, Any] = {} @@ -419,6 +428,13 @@ def __init__( "runtime max_turns must be between 1 and 4294967295" ) values["max_turns"] = max_turns + if record is not None: + record = _mapping(record, "runtime record") + if set(record) != {"enabled"} or not isinstance(record["enabled"], bool): + raise FabricConfigError( + "runtime record must be an object with only a boolean enabled field" + ) + values["record"] = {"enabled": record["enabled"]} super().__init__(values, extra_fields=extra_fields) @classmethod @@ -432,6 +448,7 @@ def from_mapping(cls, value: Mapping[str, Any]) -> "_RuntimeConfig": artifacts=data.get("artifacts"), timeout_seconds=data.get("timeout_seconds"), max_turns=data.get("max_turns"), + record=data.get("record"), extra_fields={ key: item for key, item in data.items() if key not in cls._fields }, diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index f2660d5e3..ebfeb1896 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -46,6 +46,21 @@ if result.usage is not None: Treat missing counters and cost as unknown, not zero. `cached_input_tokens` reports cached prompt tokens; `input_tokens_include_cache` declares whether `input_tokens` includes them (`True`), excludes them (`False`), or has unknown semantics (absent). Do not add cache to an inclusive input count. Adapters can put cache-write, reasoning, peak request, and per-model detail in the descriptor-declared `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` object. NeMo Fabric preserves that object at `result.usage["metadata"]["nemo_fabric.detailed_token_usage.v1"]`. Treat detailed input and output independently: any detailed accounting field for a dimension makes that dimension authoritative, with no fallback to its top-level counters. The views can diverge. Calculate an inclusive total only when the selected detailed counters provide enough information; otherwise leave the total unknown. Keep estimated costs separate in provider metadata; `cost_usd` is reported cost, not an estimate. +## Run Records + +Set `RuntimeConfig(artifacts=..., record=RecordConfig(enabled=True))` to have +NeMo Fabric write `run-record.jsonl` for each invocation under +`/.fabric///`. Recording is off by +default and needs an artifact root, the `local` environment provider, and a +`process` or `python` adapter. The +result lists the file as the artifact with kind `run_record`. The file has a +`start` line, written before the request, and an `end` line with status, +elapsed time, error code, numeric usage, and artifact names. It holds no +prompts, output, or error messages. A record without an end line has an +unknown outcome; do not count it as failed. The format is the `v1alpha1` +[run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) +and may change until a reader ships. + ## Correlation IDs `runtime_id` identifies the runtime lifecycle, `invocation_id` identifies one diff --git a/tests/python/test_sdk_consumer.py b/tests/python/test_sdk_consumer.py index bac8db3e3..a9e5e5727 100644 --- a/tests/python/test_sdk_consumer.py +++ b/tests/python/test_sdk_consumer.py @@ -10,7 +10,7 @@ from unittest.mock import MagicMock import pytest -from nemo_fabric import Fabric, FabricConfigError, RunPlan +from nemo_fabric import Fabric, FabricConfigError, RecordConfig, RunPlan from _utils.configs import hermes_shim_config @@ -148,3 +148,16 @@ async def test_installed_descriptors_reject_invalid_config_before_native_start( base_dir=hermes_shim_agent_dir, ) mock_start.assert_not_called() + + +def test_record_without_an_artifact_root_fails_native_planning(hermes_shim_agent_dir): + config = hermes_shim_config() + config.runtime.artifacts = None + config.environment.artifacts = None + config.runtime.record = RecordConfig(enabled=True) + + with pytest.raises(FabricConfigError, match="runtime.record.enabled"): + Fabric().plan(config, base_dir=hermes_shim_agent_dir) + + config.runtime.artifacts = "./artifacts" + assert Fabric().plan(config, base_dir=hermes_shim_agent_dir).config.runtime.record == {"enabled": True} diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index 5361e4697..e527e819a 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -36,6 +36,7 @@ from nemo_fabric import McpConfig from nemo_fabric import McpServerConfig from nemo_fabric import MetadataConfig +from nemo_fabric import RecordConfig from nemo_fabric import RelayAtifConfig from nemo_fabric import RelayAtofConfig from nemo_fabric import RelayAtofFileSinkConfig @@ -698,6 +699,28 @@ def test_instruction_content_must_be_non_empty(content: str): _FabricConfigSnapshot.from_mapping(raw) +def test_runtime_record_round_trips_and_rejects_other_shapes(): + config = FabricConfig( + metadata=MetadataConfig(name="demo"), + harness=HarnessConfig(adapter_id="test.fabric.shim"), + runtime=RuntimeConfig(artifacts="runs", record=RecordConfig(enabled=True)), + ) + + assert config.to_mapping()["runtime"]["record"] == {"enabled": True} + assert "record" not in RuntimeConfig().to_mapping() + for invalid in ({}, {"enabled": "yes"}, {"enabled": True, "durable": True}): + with pytest.raises(ValidationError): + RecordConfig.model_validate(invalid) + + raw = _plan()["config"] + raw["runtime"] = {"record": {"enabled": True}} + assert _FabricConfigSnapshot.from_mapping(raw).runtime.record == {"enabled": True} + for invalid in ({}, {"enabled": 1}, {"enabled": True, "durable": True}): + raw["runtime"]["record"] = invalid + with pytest.raises(FabricConfigError, match="boolean enabled field"): + _FabricConfigSnapshot.from_mapping(raw) + + def test_max_turns_matches_rust_u32_range(): maximum = (1 << 32) - 1 From 3d0b6edab8886c743ac5a953e51cbdd495c1d68a Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:31:15 -0700 Subject: [PATCH 019/102] fix: check the parent's generation before counting a Linux child The Linux tree walk accepted a child when its stat named the queued parent PID. If the parent and child both exited and other processes reused both PIDs with the same parent-child relation, the walk counted the unrelated process. The walk now reads the parent again after the child and counts the child only while the parent still has the start time it was counted with: a parent that is the same process after the child's read held its PID the whole time, so the child's parent PID names it. A parent that exited no longer shows that, so the child is skipped like a reparented one, and a parent that cannot be read makes the snapshot unavailable. Both Linux listings use the shared walk, so the check covers the children files and the fallback scan. The scripted regressions fail without it with 3 processes and 900 ns of CPU time from the unrelated process. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 185 +++++++++++++++++--- 1 file changed, 157 insertions(+), 28 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index a3a3be622..d362a979d 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -197,15 +197,40 @@ mod tree { fn read(&self, pid: u32) -> Result; } - /// Whether `process`, listed as a child of `parent` at `listed_at` and - /// read afterwards, is still the listed process. A parent PID read with - /// the record shows it directly. Without one, a process that started - /// before the list was read already held its PID then, so it is the - /// listed process. - fn is_listed_child(process: &Process, parent: u32, listed_at: u64) -> bool { + /// The process that listed a child, as the tree read saw it. + #[derive(Debug, Clone, Copy)] + struct Parent { + pid: u32, + /// The parent's start time when the tree read counted it. + start: u64, + /// When the parent's children were listed. + listed_at: u64, + } + + /// Whether `process`, listed as a child of `parent` and read afterwards, + /// is still a child of that same parent process. + /// + /// With a parent PID in the record, the parent is read again after the + /// child. A parent that still has the start time it was counted with + /// held its PID throughout, so the child's parent PID names it and not a + /// later process that reused its PID. A parent that exited no longer + /// shows that, so its child is not counted, like a reparented child. + /// + /// Without a parent PID, a process that started before the list was read + /// already held its PID then, so it is the listed process. + fn is_listed_child( + table: &impl ProcessTable, + process: &Process, + parent: Parent, + ) -> Result { match process.ppid { - Some(ppid) => ppid == parent, - None => process.start < listed_at, + Some(ppid) if ppid != parent.pid => Ok(false), + Some(_) => match table.read(parent.pid) { + Ok(now) => Ok(now.start == parent.start), + Err(ReadError::Gone) => Ok(false), + Err(ReadError::Unavailable(reason)) => Err(reason), + }, + None => Ok(process.start < parent.listed_at), } } @@ -218,38 +243,41 @@ mod tree { identity: u64, ) -> Result { let mut read = TreeRead::default(); - // Each entry is a PID and, for a child, its parent and the time the - // parent's children were listed. + // Each entry is a PID and, for a child, the parent that listed it. let mut pending = VecDeque::from([(root, None)]); - while let Some((pid, listed)) = pending.pop_front() { + while let Some((pid, parent)) = pending.pop_front() { // List the children before reading the process: a read that still // shows the expected process proves the listed children were its own. let found = table .children(pid) .and_then(|children| Ok((table.read(pid)?, children))); - let expected = |process: &Process| match listed { - None => process.start == identity && !process.exited, - Some((parent, listed_at)) => is_listed_child(process, parent, listed_at), - }; - let (process, children) = match found { - Ok((process, children)) if expected(&process) => (process, children), - // A reaped child, or one whose PID now names another process. - Ok(_) | Err(ReadError::Gone) if listed.is_some() => continue, - Ok(_) | Err(ReadError::Gone) => return Err(HOST_EXITED), - Err(ReadError::Unavailable(reason)) => return Err(reason), + let (process, children) = match (found, parent) { + (Ok((process, children)), None) if process.start == identity && !process.exited => { + (process, children) + } + (Ok(_) | Err(ReadError::Gone), None) => return Err(HOST_EXITED), + (Ok((process, children)), Some(parent)) => { + if !is_listed_child(table, &process, parent)? { + // Its PID now names another process, or it left the tree. + continue; + } + (process, children) + } + // A reaped child. + (Err(ReadError::Gone), Some(_)) => continue, + (Err(ReadError::Unavailable(reason)), _) => return Err(reason), }; read.cpu_nanos = read.cpu_nanos.saturating_add(process.cpu_nanos); if !process.exited { read.resident_bytes = read.resident_bytes.saturating_add(process.resident_bytes); read.process_count += 1; } - let listed_at = children.listed_at; - pending.extend( - children - .pids - .into_iter() - .map(|child| (child, Some((pid, listed_at)))), - ); + let parent = Parent { + pid, + start: process.start, + listed_at: children.listed_at, + }; + pending.extend(children.pids.into_iter().map(|child| (child, Some(parent)))); } Ok(read) } @@ -900,6 +928,107 @@ mod tests { } } + /// How the Linux reader lists children. + #[derive(Debug, Clone, Copy)] + enum Listing { + /// `/proc//task//children`, read when the walk reaches the process. + ChildrenFiles, + /// One pass over every process's parent, made before the walk. + Scan, + } + + const LISTINGS: [Listing; 2] = [Listing::ChildrenFiles, Listing::Scan]; + + /// A table in which other processes take some PIDs once the walk has + /// read `reused_after`: `before` answers until then and `after` from then + /// on. A scan lists children from `before`, because it ran before the walk. + struct PidReuseTable { + before: ScriptedTable, + after: ScriptedTable, + reused_after: u32, + reused: std::cell::Cell, + listing: Listing, + } + + impl PidReuseTable { + fn current(&self) -> &ScriptedTable { + if self.reused.get() { + &self.after + } else { + &self.before + } + } + } + + impl ProcessTable for PidReuseTable { + fn children(&self, pid: u32) -> std::result::Result { + match self.listing { + Listing::ChildrenFiles => self.current().children(pid), + Listing::Scan => self.before.children(pid), + } + } + + fn read(&self, pid: u32) -> std::result::Result { + let read = self.current().read(pid); + if pid == self.reused_after { + self.reused.set(true); + } + read + } + } + + /// The host's child 11 and grandchild 12 exit right after the walk reads + /// 11. `after_11` is what PID 11 then reads as; PID 12 is an unrelated + /// process whose parent PID is 11 and which used 900 ns of CPU time. + fn read_tree_after_reuse( + listing: Listing, + after_11: std::result::Result, + ) -> std::result::Result { + let lineage = Lineage::ParentPid; + let before = ScriptedTable::default() + .process(HOST, Ok(vec![11]), Ok(lineage.host(7))) + .process(11, Ok(vec![12]), Ok(lineage.child(HOST, 3))) + .process(12, Ok(vec![]), Ok(lineage.child(11, 2))); + let after = ScriptedTable::default() + .process(HOST, Ok(vec![]), Ok(lineage.host(7))) + .process(11, Ok(vec![12]), after_11) + .process(12, Ok(vec![]), Ok(lineage.child(11, 900))); + let table = PidReuseTable { + before, + after, + reused_after: 11, + reused: std::cell::Cell::new(false), + listing, + }; + read_tree_with(&table, HOST, HOST_START) + } + + #[test] + fn tree_read_skips_child_whose_parent_pid_was_reused() { + for listing in LISTINGS { + let unrelated_parent = Process { + ppid: Some(1), + start: LISTED_AT + 50, + ..Lineage::ParentPid.child(1, 500) + }; + for after_11 in [Ok(unrelated_parent), Err(ReadError::Gone)] { + let read = read_tree_after_reuse(listing, after_11).expect("tree read"); + + assert_eq!(read.process_count, 2, "{listing:?} {after_11:?}"); + assert_eq!(read.cpu_nanos, 7 + 3, "{listing:?} {after_11:?}"); + } + } + } + + #[test] + fn tree_read_with_parent_unreadable_after_child_read_is_unavailable() { + for listing in LISTINGS { + let read = read_tree_after_reuse(listing, Err(ReadError::Unavailable("hidden"))); + + assert_eq!(read.unwrap_err(), "hidden", "{listing:?}"); + } + } + #[test] fn tree_read_with_unreadable_live_child_is_unavailable() { let with_child = |children, read| { From b2e92f20e748d9380740b6ca8096f8504aafc3be Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:34:02 -0700 Subject: [PATCH 020/102] fix: make a Linux snapshot unavailable when /proc hides a descendant With /proc mounted hidepid=invisible or hidepid=ptraceable, another user's process has no /proc entry, so the reader took a live descendant that changed user for an exited one and left it out of a successful snapshot. A missing /proc entry now counts as exited only when no process holds the PID, checked with signal 0; a PID that still names a process makes the snapshot unavailable. The kernel docs name kill -0 as the way to learn that a hidden PID exists. The fallback for a kernel without children files lists every process from /proc, where hidden processes are missing altogether. It now reads the /proc mount options and is unavailable under those two modes. With hidepid=noaccess, other users' processes stay listed and reading them fails, so the scan already reports the table as unreadable. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 121 +++++++++++++++++++- 1 file changed, 116 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index d362a979d..8548b799e 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -21,8 +21,9 @@ //! A live process in the tree that cannot be read, or a process table that //! the Linux fallback cannot read in full, makes the snapshot unavailable //! instead of leaving the process out. On Linux with `/proc` mounted -//! `hidepid=invisible`, another user's descendant is invisible and is not -//! counted. +//! `hidepid=invisible` or `hidepid=ptraceable`, a descendant that `/proc` +//! hides, such as one running as another user, makes the snapshot unavailable, +//! and the fallback for a kernel without children files is always unavailable. use std::time::SystemTime; @@ -302,6 +303,10 @@ mod platform { pub(super) const TABLE_UNREADABLE: &str = "the process table cannot be read in full"; pub(super) const TREE_CHANGING: &str = "the runtime's process tree kept changing during the read"; + pub(super) const PROCESS_HIDDEN: &str = + "/proc hides a process in the runtime's process tree (hidepid)"; + pub(super) const TABLE_HIDDEN: &str = + "/proc hides other users' processes (hidepid), so the process table cannot be read in full"; /// Times a process's children are read while waiting for two reads that agree. const MAX_CHILDREN_READS: usize = 4; @@ -373,7 +378,8 @@ mod platform { } fn stat(pid: u32) -> Result { - let text = std::fs::read_to_string(format!("/proc/{pid}/stat")).map_err(read_error)?; + let text = std::fs::read_to_string(format!("/proc/{pid}/stat")) + .map_err(|error| process_read_error(pid, error))?; parse_stat(&text).ok_or(ReadError::Unavailable(PROCESS_UNREADABLE)) } @@ -381,7 +387,9 @@ mod platform { /// exited during the read is skipped; its children move to another thread. fn children_once(pid: u32) -> Result, ReadError> { let mut children = Vec::new(); - for entry in std::fs::read_dir(format!("/proc/{pid}/task")).map_err(read_error)? { + let tasks = std::fs::read_dir(format!("/proc/{pid}/task")) + .map_err(|error| process_read_error(pid, error))?; + for entry in tasks { let tid = entry.map_err(read_error)?.file_name(); let path = format!("/proc/{pid}/task/{}/children", tid.to_string_lossy()); let text = match std::fs::read_to_string(path).map_err(read_error) { @@ -459,6 +467,52 @@ mod platform { } } + /// A process whose `/proc` entry is missing exited, unless its PID still + /// names a process: then `/proc` is mounted with `hidepid` and hides it. + pub(super) fn process_read_error(pid: u32, error: std::io::Error) -> ReadError { + match read_error(error) { + ReadError::Gone if pid_exists(pid) => ReadError::Unavailable(PROCESS_HIDDEN), + other => other, + } + } + + /// Whether a process holds `pid`. Signal 0 sends nothing: `ESRCH` means + /// no process has the PID, and `EPERM` means another user's process does. + fn pid_exists(pid: u32) -> bool { + // PID 0 would address this process group, so it is never checked. + let Some(pid) = libc::pid_t::try_from(pid).ok().filter(|pid| *pid > 0) else { + return false; + }; + // SAFETY: kill with signal 0 only checks the target; it sends nothing. + let status = unsafe { libc::kill(pid, 0) }; + status == 0 || std::io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) + } + + /// Whether the `/proc` mount in `mountinfo` leaves other users' processes + /// out of its listing: `hidepid=invisible` (2) or `hidepid=ptraceable` + /// (4). With `hidepid=noaccess` (1) they stay listed and reading them + /// fails, so the scan already reports the table as unreadable. The last + /// `/proc` mount listed is the one the path reaches. + pub(super) fn proc_hides_processes(mountinfo: &str) -> bool { + let proc_options = mountinfo.lines().rev().find_map(|line| { + // ` ... - ` + let (mount, filesystem) = line.split_once(" - ")?; + let mount_point = mount.split(' ').nth(4)?; + let mut filesystem = filesystem.split(' '); + let is_proc = filesystem.next()? == "proc"; + let options = filesystem.nth(1)?; + (mount_point == "/proc" && is_proc).then_some(options) + }); + proc_options.is_some_and(|options| { + options.split(',').any(|option| { + matches!( + option, + "hidepid=2" | "hidepid=invisible" | "hidepid=4" | "hidepid=ptraceable" + ) + }) + }) + } + pub(super) fn start_identity(pid: u32) -> Option { stat(pid).ok().map(|stat| stat.start) } @@ -476,8 +530,14 @@ mod platform { }) } - /// Children by parent PID, from one pass over every process. + /// Children by parent PID, from one pass over every process. A `/proc` + /// that leaves processes out of its listing makes the pass incomplete. pub(super) fn scan_children() -> Result { + let mountinfo = + std::fs::read_to_string("/proc/self/mountinfo").map_err(|_| TABLE_UNREADABLE)?; + if proc_hides_processes(&mountinfo) { + return Err(TABLE_HIDDEN); + } let by_parent = children_by_parent(all_pids()?, |pid| stat(pid).map(|stat| stat.ppid))?; Ok(ChildSource::Scanned(by_parent)) } @@ -1140,6 +1200,57 @@ mod tests { assert_eq!(incomplete.unwrap_err(), platform::TABLE_UNREADABLE); } + /// A missing `/proc` entry is a hidden process while its PID still names + /// a process, and an exited one otherwise. + #[cfg(target_os = "linux")] + #[test] + fn linux_missing_proc_entry_is_hidden_while_its_pid_is_live() { + let missing = || std::io::Error::from(std::io::ErrorKind::NotFound); + let mut reaped = std::process::Command::new("true") + .spawn() + .expect("spawn child"); + reaped.wait().expect("reap child"); + + assert_eq!( + platform::process_read_error(std::process::id(), missing()), + ReadError::Unavailable(platform::PROCESS_HIDDEN) + ); + assert_eq!( + platform::process_read_error(reaped.id(), missing()), + ReadError::Gone + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_proc_mount_that_hides_processes_is_detected() { + let proc_mount = |options: &str| { + format!("22 1 0:5 / /proc rw,nosuid,relatime shared:13 - proc proc {options}\n") + }; + let other_mount = "23 22 0:6 / /proc/sys/fs/binfmt_misc rw - binfmt_misc binfmt_misc rw\n"; + let hides = |mountinfo: &str| platform::proc_hides_processes(mountinfo); + + for options in [ + "rw,hidepid=2", + "rw,hidepid=invisible", + "rw,hidepid=4,gid=10", + "rw,hidepid=ptraceable", + ] { + assert!(hides(&(proc_mount(options) + other_mount)), "{options}"); + } + for options in [ + "rw", + "rw,hidepid=off", + "rw,hidepid=1", + "rw,hidepid=noaccess", + ] { + assert!(!hides(&(proc_mount(options) + other_mount)), "{options}"); + } + // A later mount on /proc covers the earlier one. + assert!(!hides(&(proc_mount("rw,hidepid=2") + &proc_mount("rw")))); + assert!(!hides("30 1 0:7 / /tmp/proc rw - proc proc rw,hidepid=2\n")); + } + /// The fallback for a kernel without children files reads the tree too. #[cfg(target_os = "linux")] #[test] From 5e39195c488344d9febe901c6f91206e56e8ba1f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:34:03 -0700 Subject: [PATCH 021/102] docs: state that a hidden descendant makes a snapshot unavailable The SDK guide said a descendant hidden by hidepid=invisible is not counted. The reader now makes the snapshot unavailable instead, and the fallback for a kernel without children lists is unavailable under that mount, so the guide states both, for hidepid=ptraceable too. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 13b5b4d38..f8edd255c 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -725,8 +725,11 @@ returning zeros or leaving a process out: environment provider, or operating system has no resource reader. The `http` and `native_plugin` adapter kinds and Windows raise this error. -On Linux with `/proc` mounted `hidepid=invisible`, a descendant owned by -another user is invisible and is not counted. +On Linux with `/proc` mounted `hidepid=invisible` or `hidepid=ptraceable`, the +snapshot is unavailable while `/proc` hides a descendant, such as one that runs +as another user. On a kernel without children lists, every snapshot is +unavailable under those mount options, because hidden processes are missing +from the process table that the snapshot reads to find the tree. Each `RunResult` from a local runtime also carries `resource_snapshot`, read once after the adapter returned the result. It is `None` when the read failed; From 2f76f8c39ca83c22563472b62a0f1b9089039cca Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:35:14 -0700 Subject: [PATCH 022/102] docs: regenerate the Rust reference for the hidepid module doc The resource_snapshot module doc now says a descendant hidden by hidepid makes a snapshot unavailable. Regenerated with Rust 1.94.0; only this page changes. Signed-off-by: mnajafian-nv --- .../nemo-fabric-core/resource-snapshot/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index e55480dcc..affbe0d0f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -16,7 +16,7 @@ A snapshot reads the process tree rooted at the adapter host process that NeMo F Linux reads `/proc//task//children` and `/proc//stat`. On a kernel without those children files, it reads every process's parent ID once per snapshot to find the tree. macOS reads `proc_listchildpids` and `proc_pid_rusage`. A snapshot exposes only its own fields: never command lines, environment variables, open files, or processes outside the tree. -A live process in the tree that cannot be read, or a process table that the Linux fallback cannot read in full, makes the snapshot unavailable instead of leaving the process out. On Linux with `/proc` mounted `hidepid=invisible`, another user's descendant is invisible and is not counted. +A live process in the tree that cannot be read, or a process table that the Linux fallback cannot read in full, makes the snapshot unavailable instead of leaving the process out. On Linux with `/proc` mounted `hidepid=invisible` or `hidepid=ptraceable`, a descendant that `/proc` hides, such as one running as another user, makes the snapshot unavailable, and the fallback for a kernel without children files is always unavailable. ## Structs From 6655b13eaff210549486a85ae897f146beba4900 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:54:49 -0700 Subject: [PATCH 023/102] test: cover run record write failures The recorder promises that a failed record write never replaces the invocation result, but only a failed create was tested. Add per-thread test hooks to the record writer that delay record creation or fail the start line, the end line or the sync, and use them to check that each failure keeps the result's status, output, error, usage and identity, adds one record_write_failed event, and leaves a raised invocation error unchanged. The hooks compile only in tests. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 143 ++++++++++++++++++++ crates/fabric-core/src/record/mod.rs | 2 + crates/fabric-core/src/record/writer.rs | 45 ++++++ 3 files changed, 190 insertions(+) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 7d8a2595a..276eba78b 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -298,6 +298,7 @@ fn end_artifacts(manifest: &ArtifactManifest) -> Vec { mod tests { use super::*; use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; + use crate::record::test_hooks::{self, Fault}; fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { let config: FabricConfig = serde_json::from_value(serde_json::json!({ @@ -474,4 +475,146 @@ mod tests { ); let _ = std::fs::remove_dir_all(folder); } + + fn failed_result_with_usage(invocation: &InvocationHandle) -> RunResult { + serde_json::from_value(serde_json::json!({ + "agent_name": "a", "harness": "h", "adapter_kind": "python", + "runtime_id": invocation.runtime_id, "invocation_id": invocation.invocation_id, + "request_id": invocation.request_id, "status": "failed", "output": "partial", + "error": { + "stage": "invoke", "code": "adapter_failure", + "message": "adapter gave up", "retryable": true, + }, + "usage": {"input_tokens": 3, "output_tokens": 5, "cost_usd": 0.25}, + })) + .expect("result") + } + + /// Start and finish one record with `fault` injected, and return the + /// finished outcome with the record file's lines. + fn finish_with_fault( + name: &str, + fault: Fault, + outcome: Result, + ) -> (Result, Vec) { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder(name); + let request = RunRequest::text("input"); + test_hooks::FAULT.with(|slot| slot.set(Some(fault))); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + let finished = recorder.finish(outcome); + test_hooks::FAULT.with(|slot| slot.set(None)); + let lines = std::fs::read_to_string(folder.join(RECORD_FILE_NAME)) + .expect("read record") + .lines() + .map(|line| serde_json::from_str(line).expect("record line")) + .collect(); + let _ = std::fs::remove_dir_all(folder); + (finished, lines) + } + + /// `finished` equals `original` apart from the artifacts and events that + /// recording adds, and carries one `record_write_failed` event. + fn assert_result_kept_with_failure_event(finished: &RunResult, original: &RunResult) { + let mut without_record = finished.clone(); + without_record.artifacts = original.artifacts.clone(); + without_record.events = original.events.clone(); + assert_eq!(&without_record, original); + let failures: Vec<_> = finished + .events + .iter() + .filter(|event| event.kind == RECORD_WRITE_FAILED) + .collect(); + assert_eq!(failures.len(), 1, "{:?}", finished.events); + assert_eq!( + failures[0].metadata["runtime_id"], + original.runtime_id.as_str() + ); + assert_eq!( + failures[0].metadata["invocation_id"], + original.invocation_id.as_str() + ); + } + + fn has_record_artifact(result: &RunResult) -> bool { + result + .artifacts + .artifacts + .iter() + .any(|artifact| artifact.kind == RECORD_ARTIFACT_KIND) + } + + #[test] + fn finish_after_a_failed_start_line_keeps_the_result_and_writes_nothing() { + let (_, _, invocation) = record_test_fixture(); + let original = failed_result_with_usage(&invocation); + + let (finished, lines) = + finish_with_fault("start-append", Fault::StartAppend, Ok(original.clone())); + + let finished = finished.expect("result"); + assert_result_kept_with_failure_event(&finished, &original); + assert!(!has_record_artifact(&finished)); + assert!(lines.is_empty(), "{lines:?}"); + } + + #[test] + fn finish_after_a_failed_end_line_keeps_the_result_and_the_start_line() { + let (_, _, invocation) = record_test_fixture(); + let original = failed_result_with_usage(&invocation); + + let (finished, lines) = + finish_with_fault("end-append", Fault::EndAppend, Ok(original.clone())); + + let finished = finished.expect("result"); + assert_result_kept_with_failure_event(&finished, &original); + assert!(has_record_artifact(&finished)); + let kinds: Vec<&Value> = lines.iter().map(|line| &line["kind"]).collect(); + assert_eq!(kinds, ["start"]); + } + + #[test] + fn finish_after_a_failed_sync_keeps_the_result_and_both_lines() { + let (_, _, invocation) = record_test_fixture(); + let original = failed_result_with_usage(&invocation); + + let (finished, lines) = finish_with_fault("sync", Fault::Sync, Ok(original.clone())); + + let finished = finished.expect("result"); + assert_result_kept_with_failure_event(&finished, &original); + assert!(has_record_artifact(&finished)); + let kinds: Vec<&Value> = lines.iter().map(|line| &line["kind"]).collect(); + assert_eq!(kinds, ["start", "end"]); + assert_eq!(lines[1]["usage"]["input_tokens"], 3); + } + + #[test] + fn finish_returns_the_invocation_error_when_recording_also_fails() { + for fault in [Fault::StartAppend, Fault::EndAppend, Fault::Sync] { + let error = FabricError::AdapterLifecycleOperation { + operation: "invoke", + runtime_id: "rt".to_string(), + code: "host_timeout".to_string(), + message: "adapter host did not answer".to_string(), + diagnostics: String::new(), + }; + let expected = error.to_string(); + + let (finished, _) = finish_with_fault("error", fault, Err(error)); + + let error = finished.expect_err("invocation error"); + assert_eq!(error.to_string(), expected, "{fault:?}"); + assert!( + matches!(&error, FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout"), + "{fault:?}: {error}" + ); + } + } } diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 32f867bb6..43a6019db 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -17,3 +17,5 @@ mod writer; pub(crate) use invocation::{InvocationRecorder, RecordStart}; pub(crate) use lines::RecordLine; +#[cfg(test)] +pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs index 1a849ac60..ea95049cf 100644 --- a/crates/fabric-core/src/record/writer.rs +++ b/crates/fabric-core/src/record/writer.rs @@ -27,6 +27,8 @@ impl RecordWriter { /// Create the record at `path`. `create_new` refuses an existing file or /// symbolic link at `path`, and on Unix the file is mode 0600. pub(crate) fn create(path: &Path) -> io::Result { + #[cfg(test)] + std::thread::sleep(test_hooks::CREATE_DELAY.with(std::cell::Cell::get)); let mut options = OpenOptions::new(); options.append(true).create_new(true); #[cfg(unix)] @@ -41,6 +43,8 @@ impl RecordWriter { /// Append `line` and its newline with one write. pub(crate) fn append(&mut self, line: &RecordLine) -> io::Result<()> { + #[cfg(test)] + test_hooks::fail_append(&line.body)?; let mut bytes = serde_json::to_vec(line).map_err(io::Error::other)?; bytes.push(b'\n'); self.file.write_all(&bytes) @@ -48,10 +52,51 @@ impl RecordWriter { /// Flush the file's data and metadata to stable storage. pub(crate) fn sync(&self) -> io::Result<()> { + #[cfg(test)] + test_hooks::fail_if(test_hooks::Fault::Sync)?; self.file.sync_all() } } +/// Per-thread hooks that make record I/O slow or fail in tests. +#[cfg(test)] +pub(crate) mod test_hooks { + use std::cell::Cell; + use std::io; + use std::time::Duration; + + use super::super::lines::LineBody; + + /// A record operation that a test makes fail. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum Fault { + StartAppend, + EndAppend, + Sync, + } + + thread_local! { + /// Delay added before each record file is created on this thread. + pub(crate) static CREATE_DELAY: Cell = const { Cell::new(Duration::ZERO) }; + /// Operation that fails on this thread. + pub(crate) static FAULT: Cell> = const { Cell::new(None) }; + } + + pub(super) fn fail_append(body: &LineBody) -> io::Result<()> { + match body { + LineBody::Start(_) => fail_if(Fault::StartAppend), + LineBody::End(_) => fail_if(Fault::EndAppend), + } + } + + pub(super) fn fail_if(fault: Fault) -> io::Result<()> { + if FAULT.with(Cell::get) == Some(fault) { + return Err(io::Error::other(format!("injected {fault:?} failure"))); + } + Ok(()) + } +} + #[cfg(test)] mod tests { use super::*; From f43809ed73de7c5e3b7608588e3aa5e7ef6dcc0f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:55:03 -0700 Subject: [PATCH 024/102] fix: start the run record before the adapter deadline Core set the adapter's deadline and then created the record and wrote its start line, while the host timeout starts only when the request is sent. Slow record storage therefore shortened the adapter's budget without shortening the host timeout, so recording could turn a successful invocation into a timeout. Create the record and write the start line before core sets the deadline and writes adapter-invocation.json. The start line records the timeout, not the deadline, so the deadline the adapter receives and the one in adapter-invocation.json stay the same value. A failed adapter-invocation.json write now happens after the record starts, so its end line reads artifact_write_failed. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 5 +- crates/fabric-core/src/runtime.rs | 53 ++++++++++++++++----- docs/sdk/python.mdx | 4 +- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 276eba78b..a38538da0 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -46,9 +46,10 @@ const RECORDED_CONTEXT_KEYS: [&str; 7] = [ "harbor_context_id", "harbor_trial_name", ]; -/// End-line error code for an artifact write that failed after the adapter returned. +/// End-line error code for a failed Fabric file write: `adapter-invocation.json` +/// before the request, or a result artifact after the adapter returned. const ARTIFACT_WRITE_FAILED: &str = "artifact_write_failed"; -/// End-line error code for any other Fabric error after the request was sent. +/// End-line error code for any other Fabric error after the record started. const FABRIC_ERROR: &str = "fabric_error"; /// What the start line needs from core's invocation path. diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index fce59395e..51e21dd5f 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1752,8 +1752,9 @@ fn run_local_host_invocation_with_timeout( } /// Run one invocation on the local host. When the plan turns the run record -/// on, `recorder` is set right before core sends the request to the host, so -/// every invocation that core tried to send has a record. +/// on, `recorder` is set before core sets the adapter deadline and writes +/// `adapter-invocation.json`, so every invocation that core tried to send has +/// a record. fn local_host_invocation_result( plan: &RunPlan, runtime: &RuntimeHandle, @@ -1802,6 +1803,18 @@ fn local_host_invocation_result( &host_guard.environment_env, &adapter_invocation.runtime_context.environment.env, )?; + // Record I/O runs before the adapter deadline is set, so slow record + // storage delays the request instead of shortening the adapter's budget. + if plan.config.runtime.record_enabled() && artifacts.root.is_some() { + *recorder = Some(InvocationRecorder::start(RecordStart { + plan, + runtime, + invocation: &invocation, + request: &request, + fabric_home: &fabric_home, + timeout: invoke_timeout, + })); + } adapter_invocation.runtime_context.deadline_millis = adapter_deadline_millis(invoke_timeout); let (lifecycle_request, adapter_payload) = match invocation_kind { @@ -1836,16 +1849,6 @@ fn local_host_invocation_result( } }; let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; - if plan.config.runtime.record_enabled() && artifacts.root.is_some() { - *recorder = Some(InvocationRecorder::start(RecordStart { - plan, - runtime, - invocation: &invocation, - request: &request, - fabric_home: &fabric_home, - timeout: invoke_timeout, - })); - } match exchange_lifecycle_message( &mut host_guard, &runtime.runtime_id, @@ -5726,6 +5729,32 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn record_preparation_does_not_shorten_the_adapter_deadline() { + let (root, mut plan) = record_plan("success"); + plan.config.runtime.timeout_seconds = Some(60.0); + let runtime = start_runtime(&plan).expect("start local host"); + let delay = Duration::from_millis(500); + crate::record::test_hooks::CREATE_DELAY.with(|slot| slot.set(delay)); + + let before = now_millis(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("slow record storage")); + let after = now_millis(); + crate::record::test_hooks::CREATE_DELAY.with(|slot| slot.set(Duration::ZERO)); + + let result = result.expect("invoke"); + let deadline = u128::from(result.output["deadline_millis"].as_u64().expect("deadline")); + // The adapter's 50-second budget starts after the record is created, + // so slow record storage does not eat into it. + assert!( + (before + delay.as_millis() + 50_000..=after + 50_000).contains(&deadline), + "deadline {deadline}, before {before}, after {after}" + ); + assert_eq!(record_files(&root).len(), 1); + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + #[test] fn record_warm_runtime_writes_one_record_per_invocation() { let (root, plan) = record_plan("success"); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 555e6ba5c..6ee3008a4 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -988,8 +988,8 @@ The file holds a start line and an end line. Every line carries `schema` adapter host error keeps its code, such as `host_timeout`, `host_crashed`, `host_io`, or `invalid_agent_run_result`. A result extension that the adapter's descriptor does not declare also reads `invalid_agent_run_result`. - A failed artifact write reads `artifact_write_failed`, and any other error - reads `fabric_error`. + A failed write of `adapter-invocation.json` or a result artifact reads + `artifact_write_failed`, and any other error reads `fabric_error`. The end line describes the invocation. A one-shot `Fabric.run` stops the runtime after the invocation; when that stop fails, the result reads `failed` From 100394a1fd0d8fd2e42b1716f83c8334fc24d1be Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:55:15 -0700 Subject: [PATCH 025/102] fix: keep run record model role and context key names whole The start line cut model role names and request context key names to 256 characters, and both are map keys. Two names that shared their first 255 characters became the same key, so one entry silently replaced the other. Config validation does not bound either name, so keep both names whole and cut only values. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 38 +++++++++++++++++++-- crates/fabric-core/src/record/lines.rs | 3 +- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index a38538da0..645a931cd 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -180,7 +180,7 @@ fn start_line(start: &RecordStart<'_>) -> StartLine { .iter() .map(|(role, model)| { ( - cut_text(role), + role.clone(), ModelName { provider: cut_text(&model.provider), model: cut_text(&model.model), @@ -194,7 +194,7 @@ fn start_line(start: &RecordStart<'_>) -> StartLine { .request .context .iter() - .map(|(key, value)| (cut_text(key), context_value(key, value))) + .map(|(key, value)| (key.clone(), context_value(key, value))) .collect(), override_keys: override_keys(start.request.overrides.as_ref()), } @@ -396,6 +396,40 @@ mod tests { ); } + #[test] + fn start_line_keeps_long_role_and_context_key_names_apart() { + let (mut plan, runtime, invocation) = record_test_fixture(); + let shared = "k".repeat(300); + let model = plan.config.models["default"].clone(); + for suffix in ["a", "b"] { + plan.config + .models + .insert(format!("{shared}{suffix}"), model.clone()); + } + let mut request = RunRequest::text("input"); + request.context = BTreeMap::from([ + (format!("{shared}a"), Value::from(1)), + (format!("{shared}b"), Value::from(2)), + ]); + let start = RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: Path::new("/unused"), + timeout: Duration::from_secs(5), + }; + + let line = start_line(&start); + + assert_eq!(line.models.len(), 3); + assert_eq!(line.context.len(), 2); + for suffix in ["a", "b"] { + assert!(line.models.contains_key(&format!("{shared}{suffix}"))); + assert!(line.context.contains_key(&format!("{shared}{suffix}"))); + } + } + fn temp_folder(name: &str) -> PathBuf { let folder = std::env::temp_dir().join(format!("fabric-recorder-{name}-{}", std::process::id())); diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index ebe9df282..964a79eda 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -15,7 +15,8 @@ use serde::Serialize; use crate::runtime::{ErrorStage, RunStatus}; -/// Longest string a line keeps from config or the request, in characters. +/// Longest value a line keeps from config or the request, in characters. +/// Model role and context key names are kept whole, so map keys stay distinct. const MAX_VALUE_CHARS: usize = 256; /// Longest adapter error code a line keeps. const MAX_CODE_CHARS: usize = 128; From 264e34ba469ebd05951045ab2d854bb0759cc780 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:55:23 -0700 Subject: [PATCH 026/102] docs: describe run recording as synchronous and best-effort The record docs said recording never changes the run and that the file is never created through a symbolic link. Both overstated the contract: record I/O runs on the invocation path, so slow storage delays the request and the return, and create_new refuses only a file or symbolic link at the record path itself. State that record I/O errors do not replace the invocation result, that recording is synchronous and best-effort with no time limit, that the record is created before the adapter deadline is set, and that creation refuses an existing file or symbolic link at run-record.jsonl. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 7 ++++--- crates/fabric-core/src/record/mod.rs | 6 ++++-- docs/sdk/python.mdx | 18 +++++++++++++----- .../references/results-and-errors.md | 5 ++++- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 645a931cd..45451bd2c 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -4,9 +4,10 @@ //! One invocation's record: the start line before the request, the end line //! after the invocation. //! -//! Recording never changes the run. A failed create, write or sync stops the -//! record for that invocation and adds a `record_write_failed` event to the -//! result; nothing here returns an error to the caller. +//! Record I/O errors do not replace the invocation result. A failed create, +//! write or sync stops the record for that invocation and adds a +//! `record_write_failed` event to the result; nothing here returns a record +//! error to the caller. use std::collections::BTreeMap; use std::io; diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 43a6019db..4b4b2deee 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -5,8 +5,10 @@ //! //! When `runtime.record.enabled` is set, core appends one start line before it //! sends the request to the adapter host and one end line after the invocation -//! returns or fails. Recording never changes the run: a failed write stops the -//! record and becomes a `record_write_failed` event on the result. +//! returns or fails. Recording is synchronous and best-effort: record I/O +//! errors do not replace the invocation result. A failed write stops the +//! record and becomes a `record_write_failed` event on the result, and slow +//! storage delays the invocation. //! //! Everything here is crate-private. The line format is public only through //! the `run-record-line` JSON Schema, version `v1alpha1`. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 6ee3008a4..3fa5363a9 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1004,11 +1004,19 @@ not covered by this rule. Each line is written with one append, so a killed process keeps every line whose write returned. NeMo Fabric syncs the file once, after the end line. A record without an end line belongs to an invocation that is still running or -that never finished; treat its outcome as unknown, not as failed. If a write -fails, the record stops, the result gains a `record_write_failed` event, and -the invocation continues unchanged. A record whose end line failed to write -therefore has no end line, and that event says why. The file is created with -mode `0600` on Linux and macOS, and never through a symbolic link. +that never finished; treat its outcome as unknown, not as failed. + +Recording is synchronous and best-effort. If a write or the sync fails, the +record stops. Record I/O errors do not replace the invocation result: a +returned result keeps its status, output, error, and usage and gains a +`record_write_failed` event, and a raised error stays the same error. A record +whose end line failed to write therefore has no end line, and that event says +why. NeMo Fabric creates the record and writes the start line before it sets +the adapter's deadline, so slow storage delays the request without shortening +that deadline. The end line and the sync run before the result returns, so +slow storage delays the return too. Recording sets no time limit on either +delay. The file is created with mode `0600` on Linux and macOS. Creation +refuses an existing file or symbolic link at `run-record.jsonl`. The line format is published as `schemas/sdk/run-record-line.schema.json` at version `v1alpha1`, which may change until a reader for run records ships. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index ebfeb1896..1c06367a3 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -57,7 +57,10 @@ result lists the file as the artifact with kind `run_record`. The file has a `start` line, written before the request, and an `end` line with status, elapsed time, error code, numeric usage, and artifact names. It holds no prompts, output, or error messages. A record without an end line has an -unknown outcome; do not count it as failed. The format is the `v1alpha1` +unknown outcome; do not count it as failed. Recording is synchronous and +best-effort: a failed record write adds a `record_write_failed` event and does +not replace the result, and slow storage delays the invocation. The format is +the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) and may change until a reader ships. From 73ebc0a8b5efdf8c1551f5cb9f14e1e18e07596b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 00:15:35 -0700 Subject: [PATCH 027/102] feat: record why each invocation stopped A run record's end line keeps the error code but no cause. A reader has to know core's and each adapter's codes to separate infrastructure failures from the agent's own, and an out-of-memory kill leaves no trace. The end line gains `stop`: a cause from Fabric's own closed list (fabric_failed, sandbox_failed, agent_exhausted_budget, unclassified, agent_failed), `reason: timeout` with the invoke timeout that applied, the time used and whether Fabric force-stopped the host, and the host's exit code or signal when it exited. Following the adapter contract, only a failed result reads agent_failed; every raised error other than a timeout or a host exit reads fabric_failed, except that an error from the exchange with the host during an out-of-memory rise reads sandbox_failed, because a kill can make the adapter raise. Both lines record the container's cgroup v2 oom_kill count when Fabric owns its cgroup. A rise sets a cause only with runtime.record.exclusive_container, which the start line records; set it only when nothing else runs in the container during an invocation, including other runtimes in the same process. The host's exit status is read only when a record is on. An ignored Linux test drives a real out-of-memory kill of the adapter host; it runs only in a memory-limited cgroup v2 container. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 26 +- crates/fabric-core/src/record/cgroup.rs | 151 +++++++++ crates/fabric-core/src/record/invocation.rs | 295 +++++++++++++++++- crates/fabric-core/src/record/lines.rs | 113 +++++++ crates/fabric-core/src/record/mod.rs | 5 +- crates/fabric-core/src/record/stop.rs | 155 +++++++++ crates/fabric-core/src/runtime.rs | 241 +++++++++++++- .../nemo_fabric.models.md | 1 + .../config/struct-recordconfig.mdx | 6 +- docs/sdk/python.mdx | 56 ++++ schemas/sdk/agent.schema.json | 4 + schemas/sdk/run-plan.schema.json | 4 + schemas/sdk/run-record-line.schema.json | 213 ++++++++++++- .../src/nemo_fabric/models.py | 9 + .../src/nemo_fabric/types.py | 14 +- .../references/results-and-errors.md | 6 +- tests/python/test_sdk_contract.py | 18 ++ 17 files changed, 1279 insertions(+), 38 deletions(-) create mode 100644 crates/fabric-core/src/record/cgroup.rs create mode 100644 crates/fabric-core/src/record/stop.rs diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 1cb2d7dcf..5cf456ae5 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -1083,6 +1083,12 @@ pub struct RecordConfig { /// artifact root, the `local` environment provider, and a `process` or /// `python` adapter. pub enabled: bool, + /// Whether nothing else runs in the container during an invocation, + /// including other runtimes in the same process, so a rise in the + /// container's out-of-memory kill count reads `sandbox_failed` in the + /// record. Unset means false. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub exclusive_container: bool, } fn default_input_schema() -> String { @@ -7425,7 +7431,10 @@ mod tests { #[test] fn record_enabled_without_an_artifact_root_fails_validation() { let mut config = typed_config("nvidia.fabric.hermes"); - config.runtime.record = Some(RecordConfig { enabled: true }); + config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container: false, + }); assert_eq!( record_error(&config), @@ -7445,7 +7454,10 @@ mod tests { fn record_enabled_with_a_non_local_provider_fails_validation() { let mut config = typed_config("nvidia.fabric.hermes"); config.runtime.artifacts = Some(PathBuf::from("runs")); - config.runtime.record = Some(RecordConfig { enabled: true }); + config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container: false, + }); config.environment.as_mut().expect("environment").provider = "docker".to_string(); assert_eq!( @@ -7463,7 +7475,10 @@ mod tests { let mut descriptor = load_adapter_descriptor(&path).expect("Claude descriptor"); let mut config = typed_config("nvidia.fabric.claude"); config.runtime.artifacts = Some(PathBuf::from("runs")); - config.runtime.record = Some(RecordConfig { enabled: true }); + config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container: false, + }); assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); for kind in [AdapterKind::Http, AdapterKind::NativePlugin] { @@ -7483,7 +7498,10 @@ mod tests { fn record_disabled_or_absent_needs_no_artifact_root() { let mut config = typed_config("nvidia.fabric.hermes"); assert_eq!(record_error(&config), None); - config.runtime.record = Some(RecordConfig { enabled: false }); + config.runtime.record = Some(RecordConfig { + enabled: false, + exclusive_container: false, + }); assert_eq!(record_error(&config), None); } diff --git a/crates/fabric-core/src/record/cgroup.rs b/crates/fabric-core/src/record/cgroup.rs new file mode 100644 index 000000000..d3931c32e --- /dev/null +++ b/crates/fabric-core/src/record/cgroup.rs @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The container's out-of-memory kill count: cgroup v2 `memory.events` +//! `oom_kill`, read at the start and end lines of an invocation. +//! +//! The count describes Fabric's container only when Fabric owns its cgroup: +//! `/proc/self/cgroup` has the single v2 line `0::/`, and the cgroup's +//! `memory.events` reads with an `oom_kill` line. The kernel creates +//! `memory.events` only in a non-root cgroup with the memory controller, so a +//! true root cgroup and rootless containers without memory delegation fail +//! that check. Every other case reads unavailable with the failed check's +//! reason. + +use std::path::{Path, PathBuf}; + +use super::lines::{OomKills, OomKillsUnavailable}; + +/// Reads cgroup v2 files under a root, `/` on a real system. +#[derive(Debug, Clone)] +pub(crate) struct CgroupReader { + root: PathBuf, +} + +impl CgroupReader { + /// The running system's `/proc/self/cgroup` and `/sys/fs/cgroup`. + pub(crate) fn system() -> Self { + Self { + root: PathBuf::from("/"), + } + } + + /// A fake cgroupfs: `/proc/self/cgroup` and `/sys/fs/cgroup`. + #[cfg(test)] + pub(crate) fn with_root(root: impl Into) -> Self { + Self { root: root.into() } + } + + /// The owned cgroup's `oom_kill` count, or why it is unavailable. + pub(crate) fn oom_kills(&self) -> OomKills { + match self.read_oom_kills() { + Ok(count) => OomKills::Count(count), + Err(reason) => OomKills::Unavailable { reason }, + } + } + + fn read_oom_kills(&self) -> Result { + if !cfg!(target_os = "linux") && self.root == Path::new("/") { + return Err(OomKillsUnavailable::Platform); + } + let membership = std::fs::read_to_string(self.root.join("proc/self/cgroup")) + .map_err(|_| OomKillsUnavailable::NoOwnedCgroup)?; + let lines: Vec<&str> = membership.lines().filter(|line| !line.is_empty()).collect(); + let [line] = lines.as_slice() else { + return Err(OomKillsUnavailable::CgroupV1); + }; + let Some(path) = line.strip_prefix("0::") else { + return Err(OomKillsUnavailable::CgroupV1); + }; + if path != "/" { + return Err(OomKillsUnavailable::NoOwnedCgroup); + } + let events = std::fs::read_to_string(self.root.join("sys/fs/cgroup/memory.events")) + .map_err(|_| OomKillsUnavailable::NoOwnedCgroup)?; + events + .lines() + .find_map(|line| line.strip_prefix("oom_kill ")) + .and_then(|count| count.trim().parse().ok()) + .ok_or(OomKillsUnavailable::NoOwnedCgroup) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + /// A fake cgroupfs root with `membership` in `/proc/self/cgroup` and, when + /// given, the cgroup's `memory.events`. + pub(crate) fn fake_root(name: &str, membership: &str, memory_events: Option<&str>) -> PathBuf { + let root = + std::env::temp_dir().join(format!("fabric-cgroup-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("proc/self")).expect("proc"); + std::fs::create_dir_all(root.join("sys/fs/cgroup")).expect("cgroup"); + std::fs::write(root.join("proc/self/cgroup"), membership).expect("membership"); + if let Some(events) = memory_events { + std::fs::write(root.join("sys/fs/cgroup/memory.events"), events).expect("events"); + } + root + } + + /// `memory.events` of a cgroup whose `oom_kill` count is `count`. + pub(crate) fn memory_events(count: u64) -> String { + format!("low 0\nhigh 0\nmax 2\noom 1\noom_kill {count}\noom_group_kill 0\n") + } + + fn unavailable(reason: OomKillsUnavailable) -> OomKills { + OomKills::Unavailable { reason } + } + + #[test] + fn oom_kills_owned_cgroup_v2_reads_memory_events() { + let root = fake_root("owned", "0::/\n", Some(&memory_events(1))); + + assert_eq!( + CgroupReader::with_root(&root).oom_kills(), + OomKills::Count(1) + ); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn oom_kills_cgroup_v1_reads_cgroup_v1() { + let root = fake_root( + "v1", + "12:memory:/docker/abc\n11:cpu:/docker/abc\n", + Some(&memory_events(1)), + ); + + assert_eq!( + CgroupReader::with_root(&root).oom_kills(), + unavailable(OomKillsUnavailable::CgroupV1) + ); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn oom_kills_named_cgroup_missing_events_or_no_oom_kill_line_reads_no_owned_cgroup() { + let named = fake_root( + "named", + "0::/user.slice/session-2.scope\n", + Some(&memory_events(1)), + ); + // Rootless containers without memory delegation, and a true root + // cgroup, have no `memory.events`. + let no_events = fake_root("no-events", "0::/\n", None); + let no_oom_kill = fake_root("no-oom-kill", "0::/\n", Some("low 0\nhigh 0\n")); + + for root in [&named, &no_events, &no_oom_kill] { + assert_eq!( + CgroupReader::with_root(root).oom_kills(), + unavailable(OomKillsUnavailable::NoOwnedCgroup), + "{}", + root.display() + ); + } + for root in [named, no_events, no_oom_kill] { + let _ = std::fs::remove_dir_all(root); + } + } +} diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 45451bd2c..2c18fa180 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -16,17 +16,20 @@ use std::time::{Duration, Instant, SystemTime}; use serde_json::Value; +use super::cgroup::CgroupReader; use super::lines::{ - ContextValue, EndArtifact, EndError, EndLine, EndUsage, LineBody, ModelName, NotRecorded, - RecordLine, RecordSchema, RecordVersion, StartLine, cut_text, recorded_code, + ContextValue, EndArtifact, EndError, EndLine, EndUsage, LimitType, LineBody, ModelName, + NotRecorded, OomKills, RecordLine, RecordSchema, RecordVersion, StartLine, Stop, StopLimit, + StopReason, cut_text, recorded_code, }; +use super::stop::{StopFacts, stop_cause}; use super::writer::{RECORD_FILE_NAME, RecordWriter}; use crate::config::RunPlan; use crate::error::{FabricError, Result}; use crate::runtime::{ - ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, INVALID_AGENT_RUN_RESULT, - InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, RuntimeHandle, - event_with_metadata, unique_artifact_name, + ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, HOST_CRASHED, HOST_TIMEOUT, + INVALID_AGENT_RUN_RESULT, InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, + RuntimeHandle, event_with_metadata, unique_artifact_name, }; use crate::time::rfc3339_millis; @@ -52,6 +55,9 @@ const RECORDED_CONTEXT_KEYS: [&str; 7] = [ const ARTIFACT_WRITE_FAILED: &str = "artifact_write_failed"; /// End-line error code for any other Fabric error after the record started. const FABRIC_ERROR: &str = "fabric_error"; +/// Error code an adapter reports when it stops at the deadline core gave it, +/// whether in a failed result or as a lifecycle error. +const ADAPTER_TIMEOUT_CODE: &str = "timeout"; /// What the start line needs from core's invocation path. pub(crate) struct RecordStart<'a> { @@ -63,6 +69,31 @@ pub(crate) struct RecordStart<'a> { pub fabric_home: &'a Path, /// The host timeout that applies to this invocation. pub timeout: Duration, + /// Reads the container's out-of-memory kill count. + pub cgroup: CgroupReader, +} + +/// The adapter host's wait status, when it exited. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct HostExit { + /// Exit code. + pub exit_code: Option, + /// Signal that ended the host (Unix only). + pub exit_signal: Option, +} + +impl HostExit { + /// Read a wait status. + pub(crate) fn from_status(status: std::process::ExitStatus) -> Self { + #[cfg(unix)] + let exit_signal = std::os::unix::process::ExitStatusExt::signal(&status); + #[cfg(not(unix))] + let exit_signal = None; + Self { + exit_code: status.code(), + exit_signal, + } + } } /// The record of one running invocation. @@ -76,12 +107,27 @@ pub(crate) struct InvocationRecorder { writer: Option, wrote_start: bool, failure: Option, + timeout: Duration, + cgroup: CgroupReader, + oom_kills_at_start: OomKills, + exclusive_container: bool, + failed_exchange: Option, +} + +/// What core saw when the exchange with the adapter host failed. +#[derive(Debug, Clone, Copy)] +struct FailedExchange { + /// Time from the start line to the failure, before core's own cleanup. + elapsed_ms: u64, + /// The host's wait status, when it had exited. + host_exit: Option, } impl InvocationRecorder { /// Create the record and write its start line. pub(crate) fn start(start: RecordStart<'_>) -> Self { let path = start.fabric_home.join(RECORD_FILE_NAME); + let oom_kills_at_start = start.cgroup.oom_kills(); let mut recorder = Self { runtime_id: start.invocation.runtime_id.clone(), invocation_id: start.invocation.invocation_id.clone(), @@ -91,11 +137,16 @@ impl InvocationRecorder { wrote_start: false, failure: None, path, + timeout: start.timeout, + cgroup: start.cgroup.clone(), + oom_kills_at_start, + exclusive_container: exclusive_container(start.plan), + failed_exchange: None, }; match RecordWriter::create(&recorder.path) { Ok(writer) => { recorder.writer = Some(writer); - recorder.append(LineBody::Start(start_line(&start))); + recorder.append(LineBody::Start(start_line(&start, oom_kills_at_start))); recorder.wrote_start = recorder.failure.is_none(); } Err(error) => recorder.failure = Some(error), @@ -103,14 +154,26 @@ impl InvocationRecorder { recorder } + /// Keep when the exchange with the adapter host failed and the host's + /// wait status, for the end line's `stop`. Core calls this before its + /// cleanup, such as stopping a timed-out host, so `failed_at` excludes it. + pub(crate) fn exchange_failed(&mut self, failed_at: Instant, host_exit: Option) { + self.failed_exchange = Some(FailedExchange { + elapsed_ms: millis(failed_at.saturating_duration_since(self.started)), + host_exit, + }); + } + /// Write and sync the end line for `outcome`, then return it. A result /// gains the `record` artifact when the start line was written, and a /// `record_write_failed` event when any write or the sync failed. pub(crate) fn finish(mut self, outcome: Result) -> Result { - let elapsed_ms = u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX); + let elapsed_ms = millis(self.started.elapsed()); + let oom_kills = self.cgroup.oom_kills(); + let stop = self.stop(&outcome, elapsed_ms, oom_kills); let end = match &outcome { - Ok(result) => result_end_line(result, elapsed_ms), - Err(error) => error_end_line(error, elapsed_ms), + Ok(result) => result_end_line(result, elapsed_ms, oom_kills, stop), + Err(error) => error_end_line(error, elapsed_ms, oom_kills, stop), }; self.append(LineBody::End(end)); if let Some(writer) = &self.writer @@ -148,6 +211,81 @@ impl InvocationRecorder { Ok(result) } + /// The end line's `stop`, or `None` when it holds nothing (a success). + fn stop( + &self, + outcome: &Result, + elapsed_ms: u64, + oom_kills_at_end: OomKills, + ) -> Option { + let oom_rose = self.exclusive_container + && matches!( + (self.oom_kills_at_start, oom_kills_at_end), + (OomKills::Count(start), OomKills::Count(end)) if end > start + ); + // `forced` is set exactly when a timeout stopped the invocation. + let (facts, forced) = match outcome { + Ok(result) => { + let failed = result.status == RunStatus::Failed; + let adapter_timeout = failed + && result + .error + .as_ref() + .is_some_and(|error| error.code == ADAPTER_TIMEOUT_CODE); + let facts = StopFacts { + status: result.status, + fabric_failed: false, + oom_rose, + timed_out: adapter_timeout, + host_exited: false, + agent_error: failed, + }; + (facts, adapter_timeout.then_some(false)) + } + // A raised error is never the agent's own failure: the adapter + // contract returns a target failure as a `failed` result. An + // out-of-memory rise outranks an error from the exchange itself, + // such as an adapter that raised after its harness was killed; + // core's own errors after a complete response keep rank 1. + Err(error) => { + let code = match error { + FabricError::AdapterLifecycleOperation { code, .. } => Some(code.as_str()), + _ => None, + }; + let forced = code == Some(HOST_TIMEOUT); + let timed_out = forced || code == Some(ADAPTER_TIMEOUT_CODE); + let host_exited = code == Some(HOST_CRASHED); + let exchange_error = self.failed_exchange.is_some(); + let facts = StopFacts { + status: RunStatus::Failed, + fabric_failed: !(timed_out || host_exited || (exchange_error && oom_rose)), + oom_rose, + timed_out, + host_exited, + agent_error: false, + }; + (facts, timed_out.then_some(forced)) + } + }; + let usage = self + .failed_exchange + .map_or(elapsed_ms, |failed| failed.elapsed_ms); + let host_exit = self.failed_exchange.and_then(|failed| failed.host_exit); + let stop = Stop { + cause: stop_cause(facts), + reason: forced.map(|_| StopReason::Timeout), + forced, + limit: forced.map(|_| StopLimit { + limit_type: LimitType::Time, + value: millis(self.timeout), + usage, + }), + exit_code: host_exit.and_then(|exit| exit.exit_code), + exit_signal: host_exit.and_then(|exit| exit.exit_signal), + }; + (stop != Stop::default()).then_some(stop) + } + /// Append one line. After the first failure the record stops. fn append(&mut self, body: LineBody) { let Some(writer) = self.writer.as_mut() else { @@ -169,7 +307,7 @@ impl InvocationRecorder { } } -fn start_line(start: &RecordStart<'_>) -> StartLine { +fn start_line(start: &RecordStart<'_>, container_oom_kills: OomKills) -> StartLine { StartLine { agent_name: cut_text(&start.runtime.agent_name), harness: cut_text(&start.runtime.harness), @@ -190,7 +328,7 @@ fn start_line(start: &RecordStart<'_>) -> StartLine { }) .collect(), fabric_version: crate::version().to_string(), - timeout_ms: u64::try_from(start.timeout.as_millis()).unwrap_or(u64::MAX), + timeout_ms: millis(start.timeout), context: start .request .context @@ -198,9 +336,23 @@ fn start_line(start: &RecordStart<'_>) -> StartLine { .map(|(key, value)| (key.clone(), context_value(key, value))) .collect(), override_keys: override_keys(start.request.overrides.as_ref()), + exclusive_container: exclusive_container(start.plan), + container_oom_kills, } } +fn exclusive_container(plan: &RunPlan) -> bool { + plan.config + .runtime + .record + .as_ref() + .is_some_and(|record| record.exclusive_container) +} + +fn millis(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + /// The sorted top-level key names of `overrides`; empty unless it is an object. fn override_keys(overrides: Option<&Value>) -> Vec { let Some(Value::Object(overrides)) = overrides else { @@ -228,17 +380,29 @@ fn context_value(key: &str, value: &Value) -> ContextValue { } } -fn result_end_line(result: &RunResult, elapsed_ms: u64) -> EndLine { +fn result_end_line( + result: &RunResult, + elapsed_ms: u64, + container_oom_kills: OomKills, + stop: Option, +) -> EndLine { EndLine { status: result.status, elapsed_ms, error: result.error.as_ref().map(result_error), usage: result.usage.as_ref().map(end_usage), artifacts: end_artifacts(&result.artifacts), + container_oom_kills, + stop, } } -fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { +fn error_end_line( + error: &FabricError, + elapsed_ms: u64, + container_oom_kills: OomKills, + stop: Option, +) -> EndLine { let (code, stage) = match error { FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), // After the record starts, only the check of the adapter's result @@ -257,6 +421,8 @@ fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { }), usage: None, artifacts: Vec::new(), + container_oom_kills, + stop, } } @@ -351,6 +517,7 @@ mod tests { request: &request, fabric_home: Path::new("/unused"), timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }; let line = serde_json::to_value(RecordLine { @@ -360,7 +527,7 @@ mod tests { invocation_id: "inv".to_string(), request_id: "req".to_string(), time: "2026-10-14T18:01:10.205Z".to_string(), - body: LineBody::Start(start_line(&start)), + body: LineBody::Start(start_line(&start, OomKills::Count(0))), }) .expect("start line"); @@ -419,9 +586,10 @@ mod tests { request: &request, fabric_home: Path::new("/unused"), timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }; - let line = start_line(&start); + let line = start_line(&start, OomKills::Count(0)); assert_eq!(line.models.len(), 3); assert_eq!(line.context.len(), 2); @@ -460,6 +628,7 @@ mod tests { request: &request, fabric_home: &folder, timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }); let mut result = succeeded_result(&invocation); result.artifacts.artifacts.push(ArtifactRef { @@ -496,6 +665,7 @@ mod tests { request: &request, fabric_home: &folder, timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }); let result = succeeded_result(&invocation); @@ -544,6 +714,7 @@ mod tests { request: &request, fabric_home: &folder, timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }); let finished = recorder.finish(outcome); test_hooks::FAULT.with(|slot| slot.set(None)); @@ -653,4 +824,98 @@ mod tests { ); } } + + /// Run one invocation whose exchange with the host fails with `code` after + /// the host died of SIGKILL, while a fake cgroup's `oom_kill` count rises + /// from 4 to 5, and return its start and end lines. + fn oom_kill_record(name: &str, exclusive_container: bool, code: &str) -> (Value, Value) { + use crate::config::RecordConfig; + use crate::record::cgroup::tests::{fake_root, memory_events}; + let (mut plan, runtime, invocation) = record_test_fixture(); + plan.config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container, + }); + let root = fake_root(name, "0::/\n", Some(&memory_events(4))); + let folder = root.join("invocation"); + std::fs::create_dir_all(&folder).expect("invocation folder"); + let request = RunRequest::text("input"); + let mut recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + cgroup: CgroupReader::with_root(&root), + }); + std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)) + .expect("oom kill"); + recorder.exchange_failed( + Instant::now(), + Some(HostExit { + exit_code: None, + exit_signal: Some(9), + }), + ); + + let error = recorder + .finish(Err(FabricError::AdapterLifecycleOperation { + operation: "invoke", + runtime_id: "rt".to_string(), + code: code.to_string(), + message: "host exited".to_string(), + diagnostics: String::new(), + })) + .expect_err("error passes through"); + assert!(error.to_string().contains(code)); + let text = std::fs::read_to_string(folder.join(RECORD_FILE_NAME)).expect("record"); + let lines: Vec = text + .lines() + .map(|line| serde_json::from_str(line).expect("json")) + .collect(); + let _ = std::fs::remove_dir_all(root); + let start = lines.first().expect("start line").clone(); + let end = lines.last().expect("end line").clone(); + assert_eq!( + (&start["kind"], &end["kind"]), + (&"start".into(), &"end".into()) + ); + (start, end) + } + + #[test] + fn oom_kill_rise_in_an_exclusive_container_reads_sandbox_failed() { + let (start, end) = oom_kill_record("exclusive", true, HOST_CRASHED); + + assert_eq!(start["container_oom_kills"], 4); + assert_eq!(start["exclusive_container"], true); + assert_eq!(end["container_oom_kills"], 5); + assert_eq!( + end["stop"], + serde_json::json!({"cause": "sandbox_failed", "exit_signal": 9}) + ); + } + + #[test] + fn oom_kill_rise_in_a_shared_container_sets_no_sandbox_cause() { + let (start, end) = oom_kill_record("shared", false, HOST_CRASHED); + + assert_eq!(start["exclusive_container"], false); + assert_eq!(end["container_oom_kills"], 5); + assert_eq!( + end["stop"], + serde_json::json!({"cause": "unclassified", "exit_signal": 9}) + ); + } + + #[test] + fn oom_kill_rise_outranks_an_adapter_error_from_the_exchange() { + let (_, end) = oom_kill_record("adapter-error", true, "lifecycle_adapter_invoke_failed"); + + assert_eq!( + end["stop"], + serde_json::json!({"cause": "sandbox_failed", "exit_signal": 9}) + ); + } } diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index 964a79eda..886df0f2c 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -96,6 +96,11 @@ pub(crate) struct StartLine { /// recorded. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub override_keys: Vec, + /// `runtime.record.exclusive_container`: whether a rise in + /// `container_oom_kills` can set the `sandbox_failed` cause. + pub exclusive_container: bool, + /// The container's cgroup v2 `oom_kill` count when the invocation started. + pub container_oom_kills: OomKills, } /// One configured model. @@ -147,6 +152,114 @@ pub(crate) struct EndLine { /// Result artifacts, with paths relative to the artifact root. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub artifacts: Vec, + /// The container's cgroup v2 `oom_kill` count when the invocation ended. + pub container_oom_kills: OomKills, + /// Why the invocation stopped; absent when it holds nothing, as on success. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stop: Option, +} + +/// The cgroup v2 `memory.events` `oom_kill` count of the container Fabric +/// runs in, or why it could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(untagged)] +pub(crate) enum OomKills { + /// The count. + Count(u64), + /// The count could not be read. + Unavailable { + /// Why the count could not be read. + reason: OomKillsUnavailable, + }, +} + +/// Why the container's `oom_kill` count could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum OomKillsUnavailable { + /// The operating system has no cgroup v2 source. + Platform, + /// The host uses cgroup v1. + CgroupV1, + /// Fabric does not run at the root of its own cgroup, or the cgroup files + /// cannot be read. + NoOwnedCgroup, +} + +/// Why an invocation stopped. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct Stop { + /// Fabric's failure cause; absent on a cancel alone. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// What stopped the invocation, when a limit did. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// Whether Fabric force-stopped the adapter host. After a forced stop, + /// `exit_signal` is the signal Fabric sent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub forced: Option, + /// The limit that ended the invocation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, + /// The adapter host's exit code, when it exited. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exit_code: Option, + /// The signal that ended the adapter host, when one did. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exit_signal: Option, +} + +/// Fabric's own closed list of failure causes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum StopCause { + /// Core or the adapter host raised an error other than a timeout or a + /// host exit, such as an invalid adapter result or an adapter crash + /// inside the host. An error from the exchange with the host during an + /// out-of-memory rise in an exclusive container reads `SandboxFailed`. + FabricFailed, + /// The container's `oom_kill` count rose and + /// `runtime.record.exclusive_container` is set. + SandboxFailed, + /// A limit ended the invocation, such as Fabric's timeout. + AgentExhaustedBudget, + /// The adapter host exited, or nothing else matched. + Unclassified, + /// The adapter returned a failed result. + AgentFailed, +} + +/// What stopped an invocation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum StopReason { + /// The invocation reached its time limit. + Timeout, +} + +/// The limit that ended an invocation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct StopLimit { + /// The kind of limit. + #[serde(rename = "type")] + pub limit_type: LimitType, + /// The configured limit, in milliseconds for `time`: the invoke timeout + /// that applied, `runtime.timeout_seconds` or 1 hour when it is unset. + /// Core gives the adapter an earlier deadline, the timeout less + /// 10 seconds and at least half of it. + pub value: u64, + /// What the invocation used when it stopped, in the same unit. For a + /// forced stop, the time until core's timeout fired. + pub usage: u64, +} + +/// Kinds of limit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum LimitType { + /// Wall-clock time. + Time, } /// The end line's error. diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 4b4b2deee..571584a4a 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -13,11 +13,14 @@ //! Everything here is crate-private. The line format is public only through //! the `run-record-line` JSON Schema, version `v1alpha1`. +mod cgroup; mod invocation; mod lines; +mod stop; mod writer; -pub(crate) use invocation::{InvocationRecorder, RecordStart}; +pub(crate) use cgroup::CgroupReader; +pub(crate) use invocation::{HostExit, InvocationRecorder, RecordStart}; pub(crate) use lines::RecordLine; #[cfg(test)] pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/record/stop.rs b/crates/fabric-core/src/record/stop.rs new file mode 100644 index 000000000..c7dd127e3 --- /dev/null +++ b/crates/fabric-core/src/record/stop.rs @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Why an invocation stopped: the end line's `stop.cause`, in a fixed order +//! where the first match wins. A cause never changes `status`. + +use super::lines::StopCause; +use crate::runtime::RunStatus; + +/// What core knows about how an invocation ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct StopFacts { + /// Final status. + pub status: RunStatus, + /// Core or the adapter host raised an error other than a timeout or a + /// host exit, and the error did not come from the exchange with the host + /// during an out-of-memory rise. + pub fabric_failed: bool, + /// The container's `oom_kill` count rose and the run owns its container. + pub oom_rose: bool, + /// Fabric's timeout fired, or the adapter stopped at the deadline core gave it. + pub timed_out: bool, + /// The adapter host exited before it answered. + pub host_exited: bool, + /// The adapter returned a failed result. + pub agent_error: bool, +} + +/// The stop cause for `facts`: none on success and on a cancel alone. +pub(crate) fn stop_cause(facts: StopFacts) -> Option { + if facts.status == RunStatus::Succeeded { + return None; + } + if facts.fabric_failed { + return Some(StopCause::FabricFailed); + } + if facts.oom_rose { + return Some(StopCause::SandboxFailed); + } + if facts.timed_out { + return Some(StopCause::AgentExhaustedBudget); + } + if facts.status == RunStatus::Cancelled { + return None; + } + if facts.host_exited { + return Some(StopCause::Unclassified); + } + if facts.agent_error { + return Some(StopCause::AgentFailed); + } + Some(StopCause::Unclassified) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn failed() -> StopFacts { + StopFacts { + status: RunStatus::Failed, + fabric_failed: false, + oom_rose: false, + timed_out: false, + host_exited: false, + agent_error: false, + } + } + + #[test] + fn stop_cause_succeeded_run_has_no_cause_even_after_an_oom_rise() { + let facts = StopFacts { + status: RunStatus::Succeeded, + oom_rose: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), None); + } + + #[test] + fn stop_cause_core_error_during_an_oom_kill_reads_fabric_failed() { + let facts = StopFacts { + fabric_failed: true, + oom_rose: true, + host_exited: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), Some(StopCause::FabricFailed)); + } + + #[test] + fn stop_cause_oom_rise_with_a_host_exit_reads_sandbox_failed() { + let facts = StopFacts { + oom_rose: true, + host_exited: true, + timed_out: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), Some(StopCause::SandboxFailed)); + } + + #[test] + fn stop_cause_timeout_reads_agent_exhausted_budget() { + let facts = StopFacts { + timed_out: true, + agent_error: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), Some(StopCause::AgentExhaustedBudget)); + } + + #[test] + fn stop_cause_cancel_alone_has_no_cause_but_an_oom_rise_still_counts() { + let cancelled = StopFacts { + status: RunStatus::Cancelled, + ..failed() + }; + + assert_eq!(stop_cause(cancelled), None); + assert_eq!( + stop_cause(StopFacts { + oom_rose: true, + ..cancelled + }), + Some(StopCause::SandboxFailed) + ); + } + + #[test] + fn stop_cause_host_exit_reads_unclassified_before_an_agent_error() { + let facts = StopFacts { + host_exited: true, + agent_error: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), Some(StopCause::Unclassified)); + } + + #[test] + fn stop_cause_agent_error_reads_agent_failed_and_nothing_reads_unclassified() { + assert_eq!( + stop_cause(StopFacts { + agent_error: true, + ..failed() + }), + Some(StopCause::AgentFailed) + ); + assert_eq!(stop_cause(failed()), Some(StopCause::Unclassified)); + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 51e21dd5f..37a035a5b 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -31,10 +31,14 @@ use crate::config::{ validate_tool_definitions, validate_workflow, }; use crate::error::{FabricError, Result}; -use crate::record::{InvocationRecorder, RecordStart}; +use crate::record::{CgroupReader, HostExit, InvocationRecorder, RecordStart}; static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; +/// Lifecycle error code for an invocation that outlived core's timeout. +pub(crate) const HOST_TIMEOUT: &str = "host_timeout"; +/// Lifecycle error code for an adapter host that exited or closed its pipes. +pub(crate) const HOST_CRASHED: &str = "host_crashed"; /// Lifecycle error code for an adapter result that core rejects. pub(crate) const INVALID_AGENT_RUN_RESULT: &str = "invalid_agent_run_result"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; @@ -47,6 +51,8 @@ const LOCAL_HOST_INVOKE_TIMEOUT: Duration = Duration::from_secs(60 * 60); const LOCAL_HOST_ADAPTER_RESERVE: Duration = Duration::from_secs(10); const LOCAL_HOST_STOP_TIMEOUT: Duration = Duration::from_secs(10); const LOCAL_HOST_EXIT_GRACE: Duration = Duration::from_secs(2); +// How long a run record waits for a crashed host's exit status. +const LOCAL_HOST_CRASH_WAIT: Duration = Duration::from_secs(1); const LOCAL_HOST_DIAGNOSTIC_LIMIT: usize = 16 * 1024; #[cfg(test)] pub(crate) const OPENAI_STREAM_HOST: &str = "127.0.0.1"; @@ -1546,7 +1552,7 @@ impl RuntimeAdapter for LocalHostAdapter { let host_crashed = matches!( &result, Err(FabricError::AdapterLifecycleOperation { code, .. }) - if code == "host_crashed" + if code == HOST_CRASHED ); if !host_crashed { result?; @@ -1813,6 +1819,7 @@ fn local_host_invocation_result( request: &request, fabric_home: &fabric_home, timeout: invoke_timeout, + cgroup: CgroupReader::system(), })); } adapter_invocation.runtime_context.deadline_millis = @@ -1869,12 +1876,21 @@ fn local_host_invocation_result( )) } Err(error) => { + let failed_at = Instant::now(); if matches!( &error, - FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" + FabricError::AdapterLifecycleOperation { code, .. } if code == HOST_TIMEOUT ) { invalidate_timed_out_local_host(&runtime.runtime_id, &host, &mut host_guard); } + // Only a record reads the exit status, so a run without one never waits. + if let Some(recorder) = recorder.as_mut() { + let crashed = matches!( + &error, + FabricError::AdapterLifecycleOperation { code, .. } if code == HOST_CRASHED + ); + recorder.exchange_failed(failed_at, local_host_exit(&mut host_guard, crashed)); + } Err(error) } } @@ -2039,6 +2055,25 @@ fn local_host_invocation_result( }) } +/// The host's wait status after a failed exchange, when it has exited. A +/// crashed host gets up to [`LOCAL_HOST_CRASH_WAIT`] for its exit to become +/// observable. +fn local_host_exit(host: &mut LocalAdapterHost, crashed: bool) -> Option { + let deadline = Instant::now() + + if crashed { + LOCAL_HOST_CRASH_WAIT + } else { + Duration::ZERO + }; + loop { + match host.child.try_wait() { + Ok(Some(status)) => return Some(HostExit::from_status(status)), + Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(10)), + Ok(None) | Err(_) => return None, + } + } +} + fn invalidate_timed_out_local_host( runtime_id: &str, expected_host: &Arc>, @@ -2148,7 +2183,7 @@ fn stop_service_host(record: &mut LocalServiceRecord) -> Result<()> { let removal = remove_local_host_files(&record.host); let host_crashed = matches!( &result, - Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "host_crashed" + Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == HOST_CRASHED ); if !host_crashed { result?; @@ -2438,7 +2473,7 @@ fn exchange_lifecycle_message( return Err(lifecycle_error( operation, runtime_id, - "host_crashed", + HOST_CRASHED, format!( "persistent local adapter host exited before {} ({status})", operation.as_str() @@ -2456,7 +2491,7 @@ fn exchange_lifecycle_message( let code = match (source.kind(), host.child.try_wait()) { // A closed lifecycle pipe is terminal even when the child exit // status is not observable yet. - (ErrorKind::BrokenPipe, _) | (_, Ok(Some(_))) => "host_crashed", + (ErrorKind::BrokenPipe, _) | (_, Ok(Some(_))) => HOST_CRASHED, _ => "host_io", }; return Err(lifecycle_error( @@ -2477,7 +2512,7 @@ fn exchange_lifecycle_message( return Err(lifecycle_error( operation, runtime_id, - "host_timeout", + HOST_TIMEOUT, format!( "persistent local adapter host did not complete {} within {} ms", operation.as_str(), @@ -2490,7 +2525,7 @@ fn exchange_lifecycle_message( return Err(lifecycle_error( operation, runtime_id, - "host_crashed", + HOST_CRASHED, format!( "persistent local adapter host exited while processing {}", operation.as_str() @@ -3951,6 +3986,17 @@ for line in sys.stdin: if MODE == "invoke_failure": response("invoke", error=failure("invoke", "fake_invoke", "invoke rejected")) continue + if MODE == "invoke_deadline_error": + response("invoke", error=failure("invoke", "timeout", "deadline reached")) + continue + if MODE == "exit_3": + sys.exit(3) + if MODE == "self_kill": + os.kill(os.getpid(), 9) + if MODE == "allocate_until_killed": + blocks = [] + while True: + blocks.append(b"x" * (16 * 1024 * 1024)) invocation = message["payload"] expected_fields = ( {"runtime_context", "request", "stream"} @@ -3990,7 +4036,15 @@ for line in sys.stdin: "context_env": invocation["runtime_context"]["environment"].get("env", {}), "deadline_millis": invocation["runtime_context"].get("deadline_millis"), } - if MODE in {"adapter_reported_failure", "blank_error_code", "free_text_error_code"}: + if MODE == "cancelled": + result = {"status": "cancelled", "output": output} + elif MODE == "adapter_deadline": + result = { + "status": "failed", + "output": output, + "error": {"code": "timeout", "message": "deadline reached", "retryable": False}, + } + elif MODE in {"adapter_reported_failure", "blank_error_code", "free_text_error_code"}: codes = { "blank_error_code": " \t", "free_text_error_code": "Rate limit for key sk-live-123 exceeded", @@ -5508,7 +5562,10 @@ for line in sys.stdin: fn record_plan(mode: &str) -> (PathBuf, RunPlan) { let (root, mut plan) = local_host_plan(mode); - plan.config.runtime.record = Some(crate::config::RecordConfig { enabled: true }); + plan.config.runtime.record = Some(crate::config::RecordConfig { + enabled: true, + exclusive_container: false, + }); (root, plan) } @@ -5725,6 +5782,21 @@ for line in sys.stdin: end["error"], serde_json::json!({"code": "host_timeout", "stage": "invoke", "retryable": false}) ); + let stop = &end["stop"]; + assert_eq!(stop["cause"], "agent_exhausted_budget"); + assert_eq!(stop["reason"], "timeout"); + assert_eq!(stop["forced"], true); + assert_eq!(stop["limit"]["value"], 100); + let usage = stop["limit"]["usage"].as_u64().expect("usage"); + // Usage stops when the timeout fires; the end line's elapsed time also + // covers core's stop of the hung host. + assert!(usage >= 100, "{stop}"); + assert!( + usage < end["elapsed_ms"].as_u64().expect("elapsed"), + "{end}" + ); + #[cfg(unix)] + assert_eq!(stop["exit_signal"], 9, "{stop}"); let _ = stop_runtime(&plan, &runtime); let _ = fs::remove_dir_all(root); } @@ -5774,4 +5846,153 @@ for line in sys.stdin: } let _ = fs::remove_dir_all(root); } + + fn end_line_of(root: &Path) -> Value { + let (text, lines) = only_record(root); + let end = lines.last().expect("end line").clone(); + assert_eq!(end["kind"], "end", "{text}"); + end + } + + fn failed_invocation_end_line(mode: &str) -> (FabricError, Value) { + let (root, plan) = record_plan(mode); + let runtime = start_runtime(&plan).expect("start local host"); + let error = invoke_runtime(&plan, &runtime, RunRequest::text("fail")) + .expect_err("lifecycle error still raises"); + let end = end_line_of(&root); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + (error, end) + } + + fn result_end_line(mode: &str) -> (RunResult, Value) { + let (root, plan) = record_plan(mode); + let result = run_plan(&plan, RunRequest::text("run")).expect("result"); + let end = end_line_of(&root); + let _ = fs::remove_dir_all(root); + (result, end) + } + + #[test] + fn stop_succeeded_invocation_has_no_stop() { + let (result, end) = result_end_line("success"); + + assert_eq!(result.status, RunStatus::Succeeded); + assert!(end.get("stop").is_none(), "{end}"); + assert!(end.get("container_oom_kills").is_some(), "{end}"); + } + + #[test] + fn stop_host_exit_3_reads_unclassified_with_the_exit_code() { + let (error, end) = failed_invocation_end_line("exit_3"); + + assert!(error.to_string().contains("host_crashed"), "{error}"); + assert_eq!(end["error"]["code"], "host_crashed"); + assert_eq!( + end["stop"], + serde_json::json!({"cause": "unclassified", "exit_code": 3}) + ); + } + + #[cfg(unix)] + #[test] + fn stop_host_killed_by_a_signal_reads_unclassified_with_the_signal() { + let (_, end) = failed_invocation_end_line("self_kill"); + + assert_eq!( + end["stop"], + serde_json::json!({"cause": "unclassified", "exit_signal": 9}) + ); + } + + #[test] + fn stop_invalid_adapter_result_reads_fabric_failed() { + let (error, end) = failed_invocation_end_line("blank_error_code"); + + assert!( + error.to_string().contains("invalid_agent_run_result"), + "{error}" + ); + assert_eq!(end["stop"], serde_json::json!({"cause": "fabric_failed"})); + } + + #[test] + fn stop_host_reported_invoke_failure_reads_fabric_failed() { + let (_, end) = failed_invocation_end_line("invoke_failure"); + + assert_eq!(end["error"]["code"], "fake_invoke"); + assert_eq!(end["stop"], serde_json::json!({"cause": "fabric_failed"})); + } + + #[test] + fn stop_adapter_deadline_reported_as_a_lifecycle_error_reads_an_unforced_time_limit() { + let (_, end) = failed_invocation_end_line("invoke_deadline_error"); + + assert_eq!(end["error"]["code"], "timeout"); + let stop = &end["stop"]; + assert_eq!(stop["cause"], "agent_exhausted_budget"); + assert_eq!(stop["forced"], false); + assert_eq!(stop["limit"]["value"], 3_600_000); + } + + /// The out-of-memory path end to end. It needs Linux with cgroup v2 and + /// a memory-limited container in which Fabric owns the root cgroup, so it + /// is ignored by default; the PR body gives the `docker run` command. + #[cfg(target_os = "linux")] + #[test] + #[ignore = "needs a memory-limited cgroup v2 container"] + fn stop_oom_kill_in_an_exclusive_container_reads_sandbox_failed() { + let (root, mut plan) = record_plan("allocate_until_killed"); + plan.config.runtime.record = Some(crate::config::RecordConfig { + enabled: true, + exclusive_container: true, + }); + let runtime = start_runtime(&plan).expect("start local host"); + + invoke_runtime(&plan, &runtime, RunRequest::text("allocate")) + .expect_err("the kernel kills the host"); + let (_, lines) = only_record(&root); + + let count = |kind| line_of_kind(&lines, kind)["container_oom_kills"].as_u64(); + let (start, end) = ( + count("start").expect("start count"), + count("end").expect("end count"), + ); + assert!(end > start, "{lines:?}"); + let stop = &line_of_kind(&lines, "end")["stop"]; + assert_eq!(stop["cause"], "sandbox_failed", "{stop}"); + assert_eq!(stop["exit_signal"], 9, "{stop}"); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn stop_failed_result_reads_agent_failed() { + let (result, end) = result_end_line("adapter_reported_failure"); + + assert_eq!(result.status, RunStatus::Failed); + assert_eq!(end["stop"], serde_json::json!({"cause": "agent_failed"})); + } + + #[test] + fn stop_cancelled_result_has_no_stop() { + let (result, end) = result_end_line("cancelled"); + + assert_eq!(result.status, RunStatus::Cancelled); + assert_eq!(end["status"], "cancelled"); + assert!(end.get("stop").is_none(), "{end}"); + } + + #[test] + fn stop_adapter_deadline_reads_an_unforced_time_limit() { + let (_, end) = result_end_line("adapter_deadline"); + + let stop = &end["stop"]; + assert_eq!(stop["cause"], "agent_exhausted_budget"); + assert_eq!(stop["reason"], "timeout"); + assert_eq!(stop["forced"], false); + assert_eq!(stop["limit"]["type"], "time"); + assert_eq!(stop["limit"]["value"], 3_600_000); + assert!(stop["limit"]["usage"].is_u64(), "{stop}"); + } } diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index 703cff7b4..c12d9e71d 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -518,6 +518,7 @@ The model defines the following fields: | Field | Type | Required | Default | Constraints | Description | | --- | --- | --- | --- | --- | --- | | `enabled` | `StrictBool` | Yes | — | `Strict(strict=True)` | — | +| `exclusive_container` | `StrictBool \| None` | No | `None` | — | Whether nothing else runs in the container during an invocation, including other runtimes in the same process, so a rise in the container's out-of-memory kill count reads sandbox_failed in the record. Unset means false. | --- diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx index d33e42889..b72fc30ee 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n}"}} />
+
bool,\n    pub exclusive_container: bool,\n}"}} />
Per-invocation run record settings. @@ -21,6 +21,10 @@ When enabled, core appends a start line and an end line for each local invocatio Whether to write a run record for each invocation. Requires an artifact root, the `local` environment provider, and a `process` or `python` adapter. +### `exclusive_container: bool` + +Whether nothing else runs in the container during an invocation, including other runtimes in the same process, so a rise in the container's out-of-memory kill count reads `sandbox_failed` in the record. Unset means false. + ## Trait Implementations ### `impl Clone for RecordConfig` diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 3fa5363a9..59e092b21 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -995,6 +995,62 @@ The end line describes the invocation. A one-shot `Fabric.run` stops the runtime after the invocation; when that stop fails, the result reads `failed` with stage `stop`, and the record still holds the invocation's own end line. +The end line's `stop` says why the invocation stopped. It is absent on +success. `stop.cause` comes from NeMo Fabric's own closed list. Three names are +Harbor's failure causes: `sandbox_failed`, `agent_exhausted_budget`, and +`unclassified`. A Harbor runner maps `fabric_failed` to `harbor_failed` and +`agent_failed` to `unclassified`. The first rule that matches sets the cause: + +1. `fabric_failed`: NeMo Fabric or the adapter host raised an error that is + not a timeout or a host exit. Examples are an invalid adapter result and an + adapter that crashed inside the adapter host. The adapter contract returns + the agent's own failure as a `failed` result, never as a raised error. An + error from the exchange with the adapter host during an out-of-memory rise + in an exclusive container reads `sandbox_failed` instead, because a kill + can make the adapter raise; NeMo Fabric's own errors after a complete + response stay `fabric_failed`. +2. `sandbox_failed`: the container's out-of-memory kill count rose during the + invocation, and `runtime.record.exclusive_container` is `True`. +3. `agent_exhausted_budget`: the invocation reached its time limit. +4. A cancelled invocation that matched none of the above has no cause. +5. `unclassified`: the adapter host exited before it answered. +6. `agent_failed`: the adapter returned a `failed` result. +7. `unclassified`: anything else. + +An adapter that sets its own error code for a harness crash or a setup error +in a `failed` result, such as `claude_process_failed`, reads `agent_failed`. +NeMo Fabric does not interpret adapter-specific codes. + +A timeout sets `stop.reason` to `timeout` and `stop.limit` to +`{"type": "time", "value": , "usage": }`. `value` is the invoke +timeout that applied: `runtime.timeout_seconds`, or 1 hour when it is unset. +Two timeouts reach this rule: + +- NeMo Fabric's own timeout fired (error code `host_timeout`). `forced` is + `true`, and `usage` is the time until the timeout fired. `elapsed_ms` also + covers NeMo Fabric stopping the hung adapter host, so it is larger. + `exit_signal` is then the signal NeMo Fabric sent, not a kernel kill. +- The adapter stopped at the deadline NeMo Fabric gave it and reported error + code `timeout`, in a `failed` result or as an error. `forced` is `false`. + That deadline is earlier than `value`: the timeout less 10 seconds, and at + least half of it. + +When the adapter host exited, `stop.exit_code` or `stop.exit_signal` holds its +wait status. + +On Linux with cgroup v2, when NeMo Fabric runs at the root of its own +container's cgroup, both lines record the container's `memory.events` +`oom_kill` count as `container_oom_kills`. Otherwise the field reads +`{"reason": "platform" | "cgroup_v1" | "no_owned_cgroup"}`. The start line +also records `exclusive_container`. Set `exclusive_container` in +`runtime.record` to `True` only when nothing else runs in the container during +an invocation, including other runtimes in the same process. Otherwise a kill +can belong to another run, so the count is recorded but sets no cause: + +```python +record = RecordConfig(enabled=True, exclusive_container=True) +``` + `run-record.jsonl` never holds prompts, model output, error messages, tool payloads, adapter metadata, or context values outside the keys above. The same folder also holds `adapter-invocation.json` and `stdout.txt`, which NeMo Fabric diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index 7d6c5e994..6ba4ad3ee 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -565,6 +565,10 @@ "enabled": { "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", "type": "boolean" + }, + "exclusive_container": { + "description": "Whether nothing else runs in the container during an invocation,\nincluding other runtimes in the same process, so a rise in the\ncontainer's out-of-memory kill count reads `sandbox_failed` in the\nrecord. Unset means false.", + "type": "boolean" } }, "required": [ diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index 5d3211b4e..03499b493 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -1933,6 +1933,10 @@ "enabled": { "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", "type": "boolean" + }, + "exclusive_container": { + "description": "Whether nothing else runs in the container during an invocation,\nincluding other runtimes in the same process, so a rise in the\ncontainer's out-of-memory kill count reads `sandbox_failed` in the\nrecord. Unset means false.", + "type": "boolean" } }, "required": [ diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index 7b0f8642e..f9c976bfb 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -84,6 +84,10 @@ }, "type": "array" }, + "container_oom_kills": { + "$ref": "#/$defs/OomKills", + "description": "The container's cgroup v2 `oom_kill` count when the invocation ended." + }, "elapsed_ms": { "description": "Monotonic time from the start line to the end line, in milliseconds.", "format": "uint64", @@ -105,6 +109,17 @@ "$ref": "#/$defs/RunStatus", "description": "Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`." }, + "stop": { + "anyOf": [ + { + "$ref": "#/$defs/Stop" + }, + { + "type": "null" + } + ], + "description": "Why the invocation stopped; absent when it holds nothing, as on success." + }, "usage": { "anyOf": [ { @@ -119,7 +134,8 @@ }, "required": [ "status", - "elapsed_ms" + "elapsed_ms", + "container_oom_kills" ], "type": "object" }, @@ -226,6 +242,16 @@ } ] }, + "LimitType": { + "description": "Kinds of limit.", + "oneOf": [ + { + "const": "time", + "description": "Wall-clock time.", + "type": "string" + } + ] + }, "ModelName": { "description": "One configured model.", "properties": { @@ -258,6 +284,50 @@ ], "type": "object" }, + "OomKills": { + "anyOf": [ + { + "description": "The count.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + { + "description": "The count could not be read.", + "properties": { + "reason": { + "$ref": "#/$defs/OomKillsUnavailable", + "description": "Why the count could not be read." + } + }, + "required": [ + "reason" + ], + "type": "object" + } + ], + "description": "The cgroup v2 `memory.events` `oom_kill` count of the container Fabric\nruns in, or why it could not be read." + }, + "OomKillsUnavailable": { + "description": "Why the container's `oom_kill` count could not be read.", + "oneOf": [ + { + "const": "platform", + "description": "The operating system has no cgroup v2 source.", + "type": "string" + }, + { + "const": "cgroup_v1", + "description": "The host uses cgroup v1.", + "type": "string" + }, + { + "const": "no_owned_cgroup", + "description": "Fabric does not run at the root of its own cgroup, or the cgroup files\ncannot be read.", + "type": "string" + } + ] + }, "RecordSchema": { "description": "The record's schema name.", "oneOf": [ @@ -312,6 +382,10 @@ "description": "Config name (`metadata.name`).", "type": "string" }, + "container_oom_kills": { + "$ref": "#/$defs/OomKills", + "description": "The container's cgroup v2 `oom_kill` count when the invocation started." + }, "context": { "additionalProperties": { "$ref": "#/$defs/ContextValue" @@ -319,6 +393,10 @@ "description": "`RunRequest.context`. Allowlisted identity and Harbor link keys keep a\nstring or integer value; every other key is listed by name only.", "type": "object" }, + "exclusive_container": { + "description": "`runtime.record.exclusive_container`: whether a rise in\n`container_oom_kills` can set the `sandbox_failed` cause.", + "type": "boolean" + }, "fabric_version": { "description": "NeMo Fabric version that wrote the record.", "type": "string" @@ -352,9 +430,140 @@ "agent_name", "harness", "fabric_version", - "timeout_ms" + "timeout_ms", + "exclusive_container", + "container_oom_kills" ], "type": "object" + }, + "Stop": { + "description": "Why an invocation stopped.", + "properties": { + "cause": { + "anyOf": [ + { + "$ref": "#/$defs/StopCause" + }, + { + "type": "null" + } + ], + "description": "Fabric's failure cause; absent on a cancel alone." + }, + "exit_code": { + "description": "The adapter host's exit code, when it exited.", + "format": "int32", + "type": [ + "integer", + "null" + ] + }, + "exit_signal": { + "description": "The signal that ended the adapter host, when one did.", + "format": "int32", + "type": [ + "integer", + "null" + ] + }, + "forced": { + "description": "Whether Fabric force-stopped the adapter host. After a forced stop,\n`exit_signal` is the signal Fabric sent.", + "type": [ + "boolean", + "null" + ] + }, + "limit": { + "anyOf": [ + { + "$ref": "#/$defs/StopLimit" + }, + { + "type": "null" + } + ], + "description": "The limit that ended the invocation." + }, + "reason": { + "anyOf": [ + { + "$ref": "#/$defs/StopReason" + }, + { + "type": "null" + } + ], + "description": "What stopped the invocation, when a limit did." + } + }, + "type": "object" + }, + "StopCause": { + "description": "Fabric's own closed list of failure causes.", + "oneOf": [ + { + "const": "fabric_failed", + "description": "Core or the adapter host raised an error other than a timeout or a\nhost exit, such as an invalid adapter result or an adapter crash\ninside the host. An error from the exchange with the host during an\nout-of-memory rise in an exclusive container reads `SandboxFailed`.", + "type": "string" + }, + { + "const": "sandbox_failed", + "description": "The container's `oom_kill` count rose and\n`runtime.record.exclusive_container` is set.", + "type": "string" + }, + { + "const": "agent_exhausted_budget", + "description": "A limit ended the invocation, such as Fabric's timeout.", + "type": "string" + }, + { + "const": "unclassified", + "description": "The adapter host exited, or nothing else matched.", + "type": "string" + }, + { + "const": "agent_failed", + "description": "The adapter returned a failed result.", + "type": "string" + } + ] + }, + "StopLimit": { + "description": "The limit that ended an invocation.", + "properties": { + "type": { + "$ref": "#/$defs/LimitType", + "description": "The kind of limit." + }, + "usage": { + "description": "What the invocation used when it stopped, in the same unit. For a\nforced stop, the time until core's timeout fired.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "value": { + "description": "The configured limit, in milliseconds for `time`: the invoke timeout\nthat applied, `runtime.timeout_seconds` or 1 hour when it is unset.\nCore gives the adapter an earlier deadline, the timeout less\n10 seconds and at least half of it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "type", + "value", + "usage" + ], + "type": "object" + }, + "StopReason": { + "description": "What stopped an invocation.", + "oneOf": [ + { + "const": "timeout", + "description": "The invocation reached its time limit.", + "type": "string" + } + ] } }, "$schema": "https://json-schema.org/draft/2020-12/schema", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index 66ca0ba7d..a378da50e 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -200,6 +200,15 @@ class RecordConfig(FabricBaseModel): model_config = ConfigDict(extra="forbid") enabled: StrictBool + exclusive_container: StrictBool | None = Field( + default=None, + description=( + "Whether nothing else runs in the container during an invocation, " + "including other runtimes in the same process, so a rise in the " + "container's out-of-memory kill count reads sandbox_failed in the " + "record. Unset means false." + ), + ) class RuntimeConfig(FabricBaseModel): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 0456b68ff..34b218914 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -372,7 +372,8 @@ class _RuntimeConfig(_ConfigMapping): artifacts: Optional artifact-root path. timeout_seconds: Optional invocation deadline in seconds. max_turns: Optional harness turn limit. - record: Optional run record settings, ``{"enabled": bool}``. + record: Optional run record settings, ``{"enabled": bool}`` with an + optional boolean ``exclusive_container``. extra_fields: Preserved extension fields not recognized by this SDK. """ @@ -430,11 +431,16 @@ def __init__( values["max_turns"] = max_turns if record is not None: record = _mapping(record, "runtime record") - if set(record) != {"enabled"} or not isinstance(record["enabled"], bool): + if ( + "enabled" not in record + or not set(record) <= {"enabled", "exclusive_container"} + or not all(isinstance(value, bool) for value in record.values()) + ): raise FabricConfigError( - "runtime record must be an object with only a boolean enabled field" + "runtime record must be an object with a boolean enabled field " + "and an optional boolean exclusive_container field" ) - values["record"] = {"enabled": record["enabled"]} + values["record"] = dict(record) super().__init__(values, extra_fields=extra_fields) @classmethod diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 1c06367a3..45e8d823b 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -55,7 +55,11 @@ default and needs an artifact root, the `local` environment provider, and a `process` or `python` adapter. The result lists the file as the artifact with kind `run_record`. The file has a `start` line, written before the request, and an `end` line with status, -elapsed time, error code, numeric usage, and artifact names. It holds no +elapsed time, error code, numeric usage, artifact names, and `stop`, which +gives NeMo Fabric's failure cause (`fabric_failed`, `sandbox_failed`, +`agent_exhausted_budget`, `unclassified`, or `agent_failed`), the time limit +for a timeout, and the adapter host's exit status. Only a `failed` result +reads `agent_failed`; an error NeMo Fabric raises never does. It holds no prompts, output, or error messages. A record without an end line has an unknown outcome; do not count it as failed. Recording is synchronous and best-effort: a failed record write adds a `record_write_failed` event and does diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index e527e819a..a20f9f2da 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -721,6 +721,24 @@ def test_runtime_record_round_trips_and_rejects_other_shapes(): _FabricConfigSnapshot.from_mapping(raw) +def test_runtime_record_exclusive_container_round_trips_and_rejects_non_boolean(): + record = RecordConfig(enabled=True, exclusive_container=True) + + assert record.to_mapping() == {"enabled": True, "exclusive_container": True} + assert RecordConfig(enabled=True).to_mapping() == {"enabled": True} + with pytest.raises(ValidationError): + RecordConfig(enabled=True, exclusive_container="yes") + raw = _plan()["config"] + raw["runtime"] = {"record": {"enabled": True, "exclusive_container": True}} + assert _FabricConfigSnapshot.from_mapping(raw).runtime.record == { + "enabled": True, + "exclusive_container": True, + } + raw["runtime"]["record"]["exclusive_container"] = "yes" + with pytest.raises(FabricConfigError, match="optional boolean exclusive_container"): + _FabricConfigSnapshot.from_mapping(raw) + + def test_max_turns_matches_rust_u32_range(): maximum = (1 << 32) - 1 From 41670fff7d74a1a959d14e2b80fdb90a81569afe Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 05:56:36 -0700 Subject: [PATCH 028/102] test: drop impossible host exits from two stop rule tests InvocationRecorder::stop never sets fabric_failed together with host_exited, and a timeout and a host exit come from one error code, so they are never both true either. Two rule tests used those states, which taught the rule from cases that cannot happen. The second test now checks the real boundary: an out-of-memory rise outranks a timeout. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/stop.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/fabric-core/src/record/stop.rs b/crates/fabric-core/src/record/stop.rs index c7dd127e3..53da0ef13 100644 --- a/crates/fabric-core/src/record/stop.rs +++ b/crates/fabric-core/src/record/stop.rs @@ -83,7 +83,6 @@ mod tests { let facts = StopFacts { fabric_failed: true, oom_rose: true, - host_exited: true, ..failed() }; @@ -91,10 +90,9 @@ mod tests { } #[test] - fn stop_cause_oom_rise_with_a_host_exit_reads_sandbox_failed() { + fn stop_cause_oom_rise_outranks_a_timeout() { let facts = StopFacts { oom_rose: true, - host_exited: true, timed_out: true, ..failed() }; From ff6141cfd80733f933f155d4f5c7b7715ca12c59 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 05:56:37 -0700 Subject: [PATCH 029/102] docs: say when a forced stop records the signal Fabric sent terminate_local_host waits 2 seconds for the host to exit on its own before it kills it, so a host that exits in that window records its own status. On Windows the Job Object stop gives exit code 1 and no signal. State both instead of saying exit_signal is always the signal Fabric sent. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/lines.rs | 5 +++-- docs/sdk/python.mdx | 4 +++- schemas/sdk/run-record-line.schema.json | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index 886df0f2c..3e02f1789 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -195,8 +195,9 @@ pub(crate) struct Stop { /// What stopped the invocation, when a limit did. #[serde(default, skip_serializing_if = "Option::is_none")] pub reason: Option, - /// Whether Fabric force-stopped the adapter host. After a forced stop, - /// `exit_signal` is the signal Fabric sent. + /// Whether Fabric force-stopped the adapter host. After a forced stop on + /// Linux and macOS, `exit_signal` is usually the signal Fabric sent (9), + /// unless the host exited by itself during the 2-second grace period. #[serde(default, skip_serializing_if = "Option::is_none")] pub forced: Option, /// The limit that ended the invocation. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 59e092b21..60eca0840 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1029,7 +1029,9 @@ Two timeouts reach this rule: - NeMo Fabric's own timeout fired (error code `host_timeout`). `forced` is `true`, and `usage` is the time until the timeout fired. `elapsed_ms` also covers NeMo Fabric stopping the hung adapter host, so it is larger. - `exit_signal` is then the signal NeMo Fabric sent, not a kernel kill. + On Linux and macOS, `exit_signal` is then usually the signal NeMo Fabric + sent (9), unless the host exited by itself during the 2-second grace period. + On Windows, a forced stop gives `exit_code` 1 and no signal. - The adapter stopped at the deadline NeMo Fabric gave it and reported error code `timeout`, in a `failed` result or as an error. `forced` is `false`. That deadline is earlier than `value`: the timeout less 10 seconds, and at diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index f9c976bfb..99839a0fb 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -467,7 +467,7 @@ ] }, "forced": { - "description": "Whether Fabric force-stopped the adapter host. After a forced stop,\n`exit_signal` is the signal Fabric sent.", + "description": "Whether Fabric force-stopped the adapter host. After a forced stop on\nLinux and macOS, `exit_signal` is usually the signal Fabric sent (9),\nunless the host exited by itself during the 2-second grace period.", "type": [ "boolean", "null" From 59cee92e161d69ecde1dc82adbe6f81ffcff85bb Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:52:38 -0700 Subject: [PATCH 030/102] fix: read the end oom_kill count when the exchange ends finish read the container's oom_kill count after core had released the host lock and processed the response. In that window another caller on the same runtime can start the next exchange, and its out-of-memory kill would enter this record's rise and could set sandbox_failed on the wrong invocation. Read the count under the host lock as soon as exchange_lifecycle_message returns, before a timed-out host is stopped, and carry it to finish in an ExchangeEnd value together with the exchange's elapsed time and, after a failure, the host's wait status. ExchangeEnd replaces exchange_failed and is set on success too. After a host_crashed error the count is read again once the exit is observed and the larger value is kept, because the kernel counts a kill after it sends SIGKILL. File writes and the sync stay outside the lock. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 177 ++++++++++++++++---- crates/fabric-core/src/runtime.rs | 25 ++- docs/sdk/python.mdx | 6 +- 3 files changed, 169 insertions(+), 39 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 2c18fa180..d101e9466 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -111,18 +111,37 @@ pub(crate) struct InvocationRecorder { cgroup: CgroupReader, oom_kills_at_start: OomKills, exclusive_container: bool, - failed_exchange: Option, + exchange: Option, } -/// What core saw when the exchange with the adapter host failed. -#[derive(Debug, Clone, Copy)] -struct FailedExchange { - /// Time from the start line to the failure, before core's own cleanup. +/// What core saw when the exchange with the adapter host ended. Core reads +/// it under the host lock, before it stops a timed-out host and before a +/// later invocation can run, so every value describes this invocation. +#[derive(Debug, Clone)] +pub(crate) struct ExchangeEnd { + /// Time from the start line to the end of the exchange, before core's + /// own cleanup. elapsed_ms: u64, - /// The host's wait status, when it had exited. + /// The container's out-of-memory kill count at that moment, or the + /// larger count read after a crashed host's exit. + oom_kills: OomKills, + /// Whether the exchange raised an error. + failed: bool, + /// The host's wait status after a failed exchange, when it had exited. host_exit: Option, } +impl ExchangeEnd { + /// Mark the exchange failed, with the host's wait status when it exited. + pub(crate) fn with_host_exit(self, host_exit: Option) -> Self { + Self { + failed: true, + host_exit, + ..self + } + } +} + impl InvocationRecorder { /// Create the record and write its start line. pub(crate) fn start(start: RecordStart<'_>) -> Self { @@ -141,7 +160,7 @@ impl InvocationRecorder { cgroup: start.cgroup.clone(), oom_kills_at_start, exclusive_container: exclusive_container(start.plan), - failed_exchange: None, + exchange: None, }; match RecordWriter::create(&recorder.path) { Ok(writer) => { @@ -154,14 +173,33 @@ impl InvocationRecorder { recorder } - /// Keep when the exchange with the adapter host failed and the host's - /// wait status, for the end line's `stop`. Core calls this before its - /// cleanup, such as stopping a timed-out host, so `failed_at` excludes it. - pub(crate) fn exchange_failed(&mut self, failed_at: Instant, host_exit: Option) { - self.failed_exchange = Some(FailedExchange { - elapsed_ms: millis(failed_at.saturating_duration_since(self.started)), - host_exit, - }); + /// The end values at `ended_at`, when the exchange with the host + /// returned: the container's out-of-memory kill count, read now. Core + /// calls this under the host lock. + pub(crate) fn read_exchange_end(&self, ended_at: Instant) -> ExchangeEnd { + ExchangeEnd { + elapsed_ms: millis(ended_at.saturating_duration_since(self.started)), + oom_kills: self.cgroup.oom_kills(), + failed: false, + host_exit: None, + } + } + + /// `end` with the larger of its out-of-memory kill count and a count + /// read now. Core calls this after a crashed host's exit is observed: + /// the kernel counts a kill after it sends SIGKILL, so a host killed for + /// another process's allocation can exit before the count rises. + pub(crate) fn recount_oom_kills(&self, end: ExchangeEnd) -> ExchangeEnd { + let oom_kills = match (end.oom_kills, self.cgroup.oom_kills()) { + (OomKills::Count(at_end), OomKills::Count(now)) => OomKills::Count(at_end.max(now)), + (at_end, _) => at_end, + }; + ExchangeEnd { oom_kills, ..end } + } + + /// Keep what the exchange saw for the end line. + pub(crate) fn exchange_ended(&mut self, end: ExchangeEnd) { + self.exchange = Some(end); } /// Write and sync the end line for `outcome`, then return it. A result @@ -169,7 +207,11 @@ impl InvocationRecorder { /// `record_write_failed` event when any write or the sync failed. pub(crate) fn finish(mut self, outcome: Result) -> Result { let elapsed_ms = millis(self.started.elapsed()); - let oom_kills = self.cgroup.oom_kills(); + // Core always records the exchange's end; reading here is a fallback. + let oom_kills = self + .exchange + .as_ref() + .map_or_else(|| self.cgroup.oom_kills(), |exchange| exchange.oom_kills); let stop = self.stop(&outcome, elapsed_ms, oom_kills); let end = match &outcome { Ok(result) => result_end_line(result, elapsed_ms, oom_kills, stop), @@ -218,6 +260,7 @@ impl InvocationRecorder { elapsed_ms: u64, oom_kills_at_end: OomKills, ) -> Option { + let exchange = self.exchange.as_ref(); let oom_rose = self.exclusive_container && matches!( (self.oom_kills_at_start, oom_kills_at_end), @@ -255,7 +298,7 @@ impl InvocationRecorder { let forced = code == Some(HOST_TIMEOUT); let timed_out = forced || code == Some(ADAPTER_TIMEOUT_CODE); let host_exited = code == Some(HOST_CRASHED); - let exchange_error = self.failed_exchange.is_some(); + let exchange_error = exchange.is_some_and(|exchange| exchange.failed); let facts = StopFacts { status: RunStatus::Failed, fabric_failed: !(timed_out || host_exited || (exchange_error && oom_rose)), @@ -267,10 +310,8 @@ impl InvocationRecorder { (facts, timed_out.then_some(forced)) } }; - let usage = self - .failed_exchange - .map_or(elapsed_ms, |failed| failed.elapsed_ms); - let host_exit = self.failed_exchange.and_then(|failed| failed.host_exit); + let usage = exchange.map_or(elapsed_ms, |exchange| exchange.elapsed_ms); + let host_exit = exchange.and_then(|exchange| exchange.host_exit); let stop = Stop { cause: stop_cause(facts), reason: forced.map(|_| StopReason::Timeout), @@ -827,8 +868,15 @@ mod tests { /// Run one invocation whose exchange with the host fails with `code` after /// the host died of SIGKILL, while a fake cgroup's `oom_kill` count rises - /// from 4 to 5, and return its start and end lines. - fn oom_kill_record(name: &str, exclusive_container: bool, code: &str) -> (Value, Value) { + /// from 4 to 5, and return its start and end lines. With `counted_late`, + /// the count rises only after the exchange's end was read, and the end is + /// recounted as core does for a crashed host. + fn oom_kill_record( + name: &str, + exclusive_container: bool, + code: &str, + counted_late: bool, + ) -> (Value, Value) { use crate::config::RecordConfig; use crate::record::cgroup::tests::{fake_root, memory_events}; let (mut plan, runtime, invocation) = record_test_fixture(); @@ -849,15 +897,22 @@ mod tests { timeout: Duration::from_secs(5), cgroup: CgroupReader::with_root(&root), }); - std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)) - .expect("oom kill"); - recorder.exchange_failed( - Instant::now(), - Some(HostExit { + let count_the_kill = + || std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)); + if !counted_late { + count_the_kill().expect("oom kill"); + } + let mut end = recorder + .read_exchange_end(Instant::now()) + .with_host_exit(Some(HostExit { exit_code: None, exit_signal: Some(9), - }), - ); + })); + if counted_late { + count_the_kill().expect("oom kill"); + end = recorder.recount_oom_kills(end); + } + recorder.exchange_ended(end); let error = recorder .finish(Err(FabricError::AdapterLifecycleOperation { @@ -886,7 +941,7 @@ mod tests { #[test] fn oom_kill_rise_in_an_exclusive_container_reads_sandbox_failed() { - let (start, end) = oom_kill_record("exclusive", true, HOST_CRASHED); + let (start, end) = oom_kill_record("exclusive", true, HOST_CRASHED, false); assert_eq!(start["container_oom_kills"], 4); assert_eq!(start["exclusive_container"], true); @@ -899,7 +954,7 @@ mod tests { #[test] fn oom_kill_rise_in_a_shared_container_sets_no_sandbox_cause() { - let (start, end) = oom_kill_record("shared", false, HOST_CRASHED); + let (start, end) = oom_kill_record("shared", false, HOST_CRASHED, false); assert_eq!(start["exclusive_container"], false); assert_eq!(end["container_oom_kills"], 5); @@ -911,11 +966,67 @@ mod tests { #[test] fn oom_kill_rise_outranks_an_adapter_error_from_the_exchange() { - let (_, end) = oom_kill_record("adapter-error", true, "lifecycle_adapter_invoke_failed"); + let (_, end) = oom_kill_record( + "adapter-error", + true, + "lifecycle_adapter_invoke_failed", + false, + ); assert_eq!( end["stop"], serde_json::json!({"cause": "sandbox_failed", "exit_signal": 9}) ); } + + #[test] + fn oom_kill_counted_after_the_crashed_host_exited_still_reads_sandbox_failed() { + let (_, end) = oom_kill_record("counted-late", true, HOST_CRASHED, true); + + assert_eq!(end["container_oom_kills"], 5); + assert_eq!( + end["stop"], + serde_json::json!({"cause": "sandbox_failed", "exit_signal": 9}) + ); + } + + #[test] + fn oom_kill_counted_after_the_exchange_ended_stays_out_of_the_record() { + use crate::config::RecordConfig; + use crate::record::cgroup::tests::{fake_root, memory_events}; + let (mut plan, runtime, invocation) = record_test_fixture(); + plan.config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container: true, + }); + let root = fake_root("after-exchange", "0::/\n", Some(&memory_events(4))); + let folder = root.join("invocation"); + std::fs::create_dir_all(&folder).expect("invocation folder"); + let request = RunRequest::text("input"); + let mut recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + cgroup: CgroupReader::with_root(&root), + }); + recorder.exchange_ended(recorder.read_exchange_end(Instant::now())); + // Core released the host lock, so a later invocation on the same + // runtime can be killed before this result is finished. + std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)) + .expect("later oom kill"); + let mut result = succeeded_result(&invocation); + result.status = RunStatus::Failed; + + recorder.finish(Ok(result)).expect("result"); + + let text = std::fs::read_to_string(folder.join(RECORD_FILE_NAME)).expect("record"); + let _ = std::fs::remove_dir_all(root); + let end: Value = + serde_json::from_str(text.lines().last().expect("end line")).expect("json"); + assert_eq!(end["container_oom_kills"], 4, "{end}"); + assert_eq!(end["stop"], serde_json::json!({"cause": "agent_failed"})); + } } diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 37a035a5b..d9c2b1a92 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1856,13 +1856,23 @@ fn local_host_invocation_result( } }; let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; - match exchange_lifecycle_message( + let exchange = exchange_lifecycle_message( &mut host_guard, &runtime.runtime_id, &lifecycle_request, invoke_timeout, - ) { + ); + let ended_at = Instant::now(); + // Read while the host lock is held and before a timed-out host is + // stopped, so the end values count this invocation and no later one. + let exchange_end = recorder + .as_ref() + .map(|recorder| recorder.read_exchange_end(ended_at)); + match exchange { Ok(output) => { + if let (Some(recorder), Some(end)) = (recorder.as_mut(), exchange_end) { + recorder.exchange_ended(end); + } let stderr = take_local_host_stderr(&mut host_guard); Ok(( output, @@ -1876,7 +1886,6 @@ fn local_host_invocation_result( )) } Err(error) => { - let failed_at = Instant::now(); if matches!( &error, FabricError::AdapterLifecycleOperation { code, .. } if code == HOST_TIMEOUT @@ -1884,12 +1893,18 @@ fn local_host_invocation_result( invalidate_timed_out_local_host(&runtime.runtime_id, &host, &mut host_guard); } // Only a record reads the exit status, so a run without one never waits. - if let Some(recorder) = recorder.as_mut() { + if let (Some(recorder), Some(end)) = (recorder.as_mut(), exchange_end) { let crashed = matches!( &error, FabricError::AdapterLifecycleOperation { code, .. } if code == HOST_CRASHED ); - recorder.exchange_failed(failed_at, local_host_exit(&mut host_guard, crashed)); + let end = end.with_host_exit(local_host_exit(&mut host_guard, crashed)); + let end = if crashed { + recorder.recount_oom_kills(end) + } else { + end + }; + recorder.exchange_ended(end); } Err(error) } diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 60eca0840..57ad9a1c6 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1043,7 +1043,11 @@ wait status. On Linux with cgroup v2, when NeMo Fabric runs at the root of its own container's cgroup, both lines record the container's `memory.events` `oom_kill` count as `container_oom_kills`. Otherwise the field reads -`{"reason": "platform" | "cgroup_v1" | "no_owned_cgroup"}`. The start line +`{"reason": "platform" | "cgroup_v1" | "no_owned_cgroup"}`. The end line's +count is read when the adapter host answers, crashes, or reaches the timeout, +before NeMo Fabric stops a hung host and before the next invocation on the +runtime can start. After a crash, NeMo Fabric reads the count again once the +host's exit is seen and keeps the larger value. The start line also records `exclusive_container`. Set `exclusive_container` in `runtime.record` to `True` only when nothing else runs in the container during an invocation, including other runtimes in the same process. Otherwise a kill From 51077a454b757a49815e40de4e59a639a4f0d7ad Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:52:51 -0700 Subject: [PATCH 031/102] test: pin no stop cause for a cancelled result with a timeout code The adapter deadline code counts as a timeout only in a failed result or a raised error, so a cancelled result that carries error code timeout has no stop cause: cancellation takes precedence. Add a contract test from a real local host for that case and state the exception in the stop rules. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 15 +++++++++++++++ docs/sdk/python.mdx | 5 ++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d9c2b1a92..95adae888 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -4053,6 +4053,12 @@ for line in sys.stdin: } if MODE == "cancelled": result = {"status": "cancelled", "output": output} + elif MODE == "cancelled_at_deadline": + result = { + "status": "cancelled", + "output": output, + "error": {"code": "timeout", "message": "deadline reached", "retryable": False}, + } elif MODE == "adapter_deadline": result = { "status": "failed", @@ -5998,6 +6004,15 @@ for line in sys.stdin: assert!(end.get("stop").is_none(), "{end}"); } + #[test] + fn stop_cancelled_result_with_a_timeout_error_code_has_no_stop() { + let (result, end) = result_end_line("cancelled_at_deadline"); + + assert_eq!(result.status, RunStatus::Cancelled); + assert_eq!(end["error"]["code"], "timeout"); + assert!(end.get("stop").is_none(), "{end}"); + } + #[test] fn stop_adapter_deadline_reads_an_unforced_time_limit() { let (_, end) = result_end_line("adapter_deadline"); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 57ad9a1c6..9ab2b38a6 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1012,7 +1012,10 @@ Harbor's failure causes: `sandbox_failed`, `agent_exhausted_budget`, and 2. `sandbox_failed`: the container's out-of-memory kill count rose during the invocation, and `runtime.record.exclusive_container` is `True`. 3. `agent_exhausted_budget`: the invocation reached its time limit. -4. A cancelled invocation that matched none of the above has no cause. +4. A cancelled invocation that matched none of the above has no cause. A + `cancelled` result with error code `timeout` also has no cause: the + adapter's deadline code counts as a timeout only in a `failed` result or a + raised error. 5. `unclassified`: the adapter host exited before it answered. 6. `agent_failed`: the adapter returned a `failed` result. 7. `unclassified`: anything else. From b13430a8c6431ccb4cc3df23ef1ad1dbe9b94236 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:52:52 -0700 Subject: [PATCH 032/102] docs: qualify forced stop exit status and the Harbor cause mapping The Windows exit code 1 and the Unix signal 9 appear only when Fabric terminates the host; a host that exits by itself during the 2-second grace period records its own exit status while forced stays true. The fabric_failed and agent_failed mapping is guidance for a Harbor consumer of the record; nothing in Fabric reads the record for Harbor. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 9ab2b38a6..5882c63ad 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -998,8 +998,9 @@ with stage `stop`, and the record still holds the invocation's own end line. The end line's `stop` says why the invocation stopped. It is absent on success. `stop.cause` comes from NeMo Fabric's own closed list. Three names are Harbor's failure causes: `sandbox_failed`, `agent_exhausted_budget`, and -`unclassified`. A Harbor runner maps `fabric_failed` to `harbor_failed` and -`agent_failed` to `unclassified`. The first rule that matches sets the cause: +`unclassified`. A Harbor consumer of the record can map `fabric_failed` to +`harbor_failed` and `agent_failed` to `unclassified`. The first rule that +matches sets the cause: 1. `fabric_failed`: NeMo Fabric or the adapter host raised an error that is not a timeout or a host exit. Examples are an invalid adapter result and an @@ -1032,9 +1033,10 @@ Two timeouts reach this rule: - NeMo Fabric's own timeout fired (error code `host_timeout`). `forced` is `true`, and `usage` is the time until the timeout fired. `elapsed_ms` also covers NeMo Fabric stopping the hung adapter host, so it is larger. - On Linux and macOS, `exit_signal` is then usually the signal NeMo Fabric - sent (9), unless the host exited by itself during the 2-second grace period. - On Windows, a forced stop gives `exit_code` 1 and no signal. + When NeMo Fabric terminates the host, `exit_signal` is the signal it sent + (9) on Linux and macOS, and on Windows `exit_code` is 1 with no signal. A + host that exits by itself during the 2-second grace period records its own + exit status instead, while `forced` stays `true`. - The adapter stopped at the deadline NeMo Fabric gave it and reported error code `timeout`, in a `failed` result or as an error. `forced` is `false`. That deadline is earlier than `value`: the timeout less 10 seconds, and at From 2fcc672192012db9e6d6c8e257eb22ee3131cdec Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:57:52 -0700 Subject: [PATCH 033/102] test: lower the exited-child CPU threshold to two clock ticks The exited-child snapshot test required 0.1 s of CPU from a shell loop of 100,000 iterations. On a fast Linux runner with dash and 10 ms clock ticks the loop can finish in less, so the test could fail with a correct reader. The host only sleeps, so 0.02 s (two ticks) still proves that the exited child's CPU time is counted, with the same workload. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 8548b799e..61755580b 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -834,9 +834,12 @@ mod tests { assert_eq!(running.process_count, 2, "child never seen: {running:?}"); assert_eq!(exited.process_count, 1, "child still running: {exited:?}"); + // The host only sleeps, so two 10 ms clock ticks can come only from + // the exited child. A fast shell can finish the loop in well under + // 100 ms. assert!( - exited.cpu_time_seconds >= 0.1, - "exited child's CPU missing: {exited:?}" + exited.cpu_time_seconds >= 0.02, + "exited child's CPU (at least two clock ticks) missing: {exited:?}" ); } From 5ef35935dcd73ae16bc498bc1f9eca0a7f4b2184 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:26:46 -0700 Subject: [PATCH 034/102] feat: add sample lines and resource snapshots to run records A run record's start and end lines say nothing about the work in between, so a reader cannot tell a long-running invocation from a quiet one, or see how much CPU and memory the invocation used. One process-wide thread, started with the first recorded invocation, appends a sample line every 10 seconds to each active record with the elapsed time and the runtime's resource snapshot. The thread holds only a weak reference to its ticker. A snapshot read that panics is written as unavailable. A forked child sees the parent's started thread but has none, so its start lines omit sample_interval_ms and its records get no sample lines. The start and end lines carry a snapshot too. Core reads the end snapshot under the host lock together with the end oom_kill count in ExchangeEnd, right after the adapter host answers or the timeout fires, and reuses it for RunResult.resource_snapshot. A snapshot that cannot be read is written as its error code. The record file closes at the end line. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/file.rs | 96 +++++ crates/fabric-core/src/record/invocation.rs | 404 +++++++++++++----- crates/fabric-core/src/record/lines.rs | 76 +++- crates/fabric-core/src/record/mod.rs | 14 +- crates/fabric-core/src/record/sampler.rs | 198 +++++++++ crates/fabric-core/src/record/writer.rs | 1 + crates/fabric-core/src/runtime.rs | 63 ++- docs/sdk/python.mdx | 42 +- schemas/sdk/run-record-line.schema.json | 138 +++++- .../references/results-and-errors.md | 27 +- 10 files changed, 933 insertions(+), 126 deletions(-) create mode 100644 crates/fabric-core/src/record/file.rs create mode 100644 crates/fabric-core/src/record/sampler.rs diff --git a/crates/fabric-core/src/record/file.rs b/crates/fabric-core/src/record/file.rs new file mode 100644 index 000000000..373d8908c --- /dev/null +++ b/crates/fabric-core/src/record/file.rs @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The open record file that the invocation and the sample thread share. + +use std::io; +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::SystemTime; + +use super::lines::{LineBody, RecordLine, RecordSchema, RecordVersion}; +use super::writer::RecordWriter; +use crate::time::rfc3339_millis; + +/// One invocation's record file. After the first failed write it stops. +#[derive(Debug)] +pub(crate) struct RecordFile { + pub runtime_id: String, + pub invocation_id: String, + pub request_id: String, + writer: Option, + /// The first create, write or sync failure. + pub failure: Option, + /// Set when the end line is written; no sample line follows it. + ended: bool, +} + +impl RecordFile { + /// Create the file at `path`. A failed create is kept in `failure`, and + /// every later append is skipped. + pub(crate) fn create( + path: &Path, + runtime_id: String, + invocation_id: String, + request_id: String, + ) -> Self { + let (writer, failure) = match RecordWriter::create(path) { + Ok(writer) => (Some(writer), None), + Err(error) => (None, Some(error)), + }; + Self { + runtime_id, + invocation_id, + request_id, + writer, + failure, + ended: false, + } + } + + /// Append one line. + pub(crate) fn append(&mut self, body: LineBody) { + let Some(writer) = self.writer.as_mut() else { + return; + }; + let line = RecordLine { + schema: RecordSchema::RunRecord, + version: RecordVersion::V1Alpha1, + runtime_id: self.runtime_id.clone(), + invocation_id: self.invocation_id.clone(), + request_id: self.request_id.clone(), + time: rfc3339_millis(SystemTime::now()), + body, + }; + if let Err(error) = writer.append(&line) { + self.failure = Some(error); + self.writer = None; + } + } + + /// Append a sample line, unless the end line is already written. + pub(crate) fn append_sample(&mut self, body: LineBody) { + if !self.ended { + self.append(body); + } + } + + /// Append the end line, sync the file and close it. No sample line + /// follows, and a sample tick that still holds this record keeps no open + /// file handle. + pub(crate) fn append_end(&mut self, body: LineBody) { + self.ended = true; + self.append(body); + if let Some(writer) = self.writer.take() + && let Err(error) = writer.sync() + { + self.failure = Some(error); + } + } +} + +/// Lock a shared record file. A panic elsewhere while it was held leaves the +/// file usable: every write is one append. +pub(crate) fn lock(file: &Mutex) -> MutexGuard<'_, RecordFile> { + file.lock().unwrap_or_else(|error| error.into_inner()) +} diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index d101e9466..042b332f1 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! One invocation's record: the start line before the request, the end line -//! after the invocation. +//! One invocation's record: the start line before the request, sample lines +//! from the process-wide ticker while it runs, and the end line after it. //! //! Record I/O errors do not replace the invocation result. A failed create, //! write or sync stops the record for that invocation and adds a @@ -10,28 +10,30 @@ //! error to the caller. use std::collections::BTreeMap; -use std::io; use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant, SystemTime}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; use serde_json::Value; use super::cgroup::CgroupReader; +use super::file::{RecordFile, lock}; use super::lines::{ ContextValue, EndArtifact, EndError, EndLine, EndUsage, LimitType, LineBody, ModelName, - NotRecorded, OomKills, RecordLine, RecordSchema, RecordVersion, StartLine, Stop, StopLimit, - StopReason, cut_text, recorded_code, + NotRecorded, OomKills, Resources, StartLine, Stop, StopLimit, StopReason, cut_text, + recorded_code, }; +use super::sampler::{ReadResources, Registration, SampleTicker}; use super::stop::{StopFacts, stop_cause}; -use super::writer::{RECORD_FILE_NAME, RecordWriter}; +use super::writer::RECORD_FILE_NAME; use crate::config::RunPlan; use crate::error::{FabricError, Result}; +use crate::resource_snapshot::RuntimeResourceSnapshot; use crate::runtime::{ ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, HOST_CRASHED, HOST_TIMEOUT, INVALID_AGENT_RUN_RESULT, InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, RuntimeHandle, event_with_metadata, unique_artifact_name, }; -use crate::time::rfc3339_millis; /// Name of the result artifact that points at the record. const RECORD_ARTIFACT_NAME: &str = "record"; @@ -71,6 +73,10 @@ pub(crate) struct RecordStart<'a> { pub timeout: Duration, /// Reads the container's out-of-memory kill count. pub cgroup: CgroupReader, + /// Reads the runtime's resource snapshot. + pub read_resources: ReadResources, + /// Appends the sample lines. + pub ticker: Arc, } /// The adapter host's wait status, when it exited. @@ -97,21 +103,17 @@ impl HostExit { } /// The record of one running invocation. -#[derive(Debug)] pub(crate) struct InvocationRecorder { - runtime_id: String, - invocation_id: String, - request_id: String, + file: Arc>, path: PathBuf, started: Instant, - writer: Option, wrote_start: bool, - failure: Option, timeout: Duration, cgroup: CgroupReader, oom_kills_at_start: OomKills, exclusive_container: bool, exchange: Option, + sampling: Option, } /// What core saw when the exchange with the adapter host ended. Core reads @@ -122,6 +124,8 @@ pub(crate) struct ExchangeEnd { /// Time from the start line to the end of the exchange, before core's /// own cleanup. elapsed_ms: u64, + /// The runtime's resource snapshot at that moment. + resources: Resources, /// The container's out-of-memory kill count at that moment, or the /// larger count read after a crashed host's exit. oom_kills: OomKills, @@ -143,42 +147,58 @@ impl ExchangeEnd { } impl InvocationRecorder { - /// Create the record and write its start line. + /// Create the record, write its start line, and register it with the + /// sample ticker. pub(crate) fn start(start: RecordStart<'_>) -> Self { let path = start.fabric_home.join(RECORD_FILE_NAME); + let started = Instant::now(); let oom_kills_at_start = start.cgroup.oom_kills(); - let mut recorder = Self { - runtime_id: start.invocation.runtime_id.clone(), - invocation_id: start.invocation.invocation_id.clone(), - request_id: start.invocation.request_id.clone(), - started: Instant::now(), - writer: None, - wrote_start: false, - failure: None, + let mut file = RecordFile::create( + &path, + start.invocation.runtime_id.clone(), + start.invocation.invocation_id.clone(), + start.invocation.request_id.clone(), + ); + let sample_interval = start.ticker.start(); + file.append(LineBody::Start(start_line( + &start, + sample_interval, + oom_kills_at_start, + ))); + let wrote_start = file.failure.is_none(); + let file = Arc::new(Mutex::new(file)); + let sampling = (wrote_start && sample_interval.is_some()).then(|| { + start.ticker.register( + Arc::clone(&file), + started, + Arc::clone(&start.read_resources), + ) + }); + Self { + file, path, + started, + wrote_start, timeout: start.timeout, cgroup: start.cgroup.clone(), oom_kills_at_start, exclusive_container: exclusive_container(start.plan), exchange: None, - }; - match RecordWriter::create(&recorder.path) { - Ok(writer) => { - recorder.writer = Some(writer); - recorder.append(LineBody::Start(start_line(&start, oom_kills_at_start))); - recorder.wrote_start = recorder.failure.is_none(); - } - Err(error) => recorder.failure = Some(error), + sampling, } - recorder } /// The end values at `ended_at`, when the exchange with the host - /// returned: the container's out-of-memory kill count, read now. Core - /// calls this under the host lock. - pub(crate) fn read_exchange_end(&self, ended_at: Instant) -> ExchangeEnd { + /// returned: `snapshot` and the container's out-of-memory kill count, + /// read now. Core calls this under the host lock. + pub(crate) fn read_exchange_end( + &self, + ended_at: Instant, + snapshot: &Result, + ) -> ExchangeEnd { ExchangeEnd { elapsed_ms: millis(ended_at.saturating_duration_since(self.started)), + resources: Resources::from(snapshot), oom_kills: self.cgroup.oom_kills(), failed: false, host_exit: None, @@ -206,23 +226,29 @@ impl InvocationRecorder { /// gains the `record` artifact when the start line was written, and a /// `record_write_failed` event when any write or the sync failed. pub(crate) fn finish(mut self, outcome: Result) -> Result { + self.sampling = None; let elapsed_ms = millis(self.started.elapsed()); // Core always records the exchange's end; reading here is a fallback. - let oom_kills = self - .exchange - .as_ref() - .map_or_else(|| self.cgroup.oom_kills(), |exchange| exchange.oom_kills); + let (resources, oom_kills) = match self.exchange.as_ref() { + Some(exchange) => (exchange.resources.clone(), exchange.oom_kills), + None => (Resources::UNAVAILABLE, self.cgroup.oom_kills()), + }; let stop = self.stop(&outcome, elapsed_ms, oom_kills); + let end = EndValues { + elapsed_ms, + resources, + oom_kills, + stop, + }; let end = match &outcome { - Ok(result) => result_end_line(result, elapsed_ms, oom_kills, stop), - Err(error) => error_end_line(error, elapsed_ms, oom_kills, stop), + Ok(result) => result_end_line(result, end), + Err(error) => error_end_line(error, end), }; - self.append(LineBody::End(end)); - if let Some(writer) = &self.writer - && let Err(error) = writer.sync() - { - self.failure = Some(error); - } + let mut file = lock(&self.file); + file.append_end(LineBody::End(end)); + let failure = file.failure.take(); + let (runtime_id, invocation_id) = (file.runtime_id.clone(), file.invocation_id.clone()); + drop(file); let mut result = outcome?; if self.wrote_start { let name = unique_artifact_name(&result.artifacts, RECORD_ARTIFACT_NAME); @@ -234,19 +260,13 @@ impl InvocationRecorder { metadata: BTreeMap::new(), }); } - if let Some(error) = &self.failure { + if let Some(error) = failure { result.events.push(event_with_metadata( RECORD_WRITE_FAILED, format!("run record write failed; the record stops here: {error}"), BTreeMap::from([ - ( - "runtime_id".to_string(), - Value::from(self.runtime_id.as_str()), - ), - ( - "invocation_id".to_string(), - Value::from(self.invocation_id.as_str()), - ), + ("runtime_id".to_string(), Value::from(runtime_id)), + ("invocation_id".to_string(), Value::from(invocation_id)), ]), )); } @@ -326,29 +346,21 @@ impl InvocationRecorder { }; (stop != Stop::default()).then_some(stop) } +} - /// Append one line. After the first failure the record stops. - fn append(&mut self, body: LineBody) { - let Some(writer) = self.writer.as_mut() else { - return; - }; - let line = RecordLine { - schema: RecordSchema::RunRecord, - version: RecordVersion::V1Alpha1, - runtime_id: self.runtime_id.clone(), - invocation_id: self.invocation_id.clone(), - request_id: self.request_id.clone(), - time: rfc3339_millis(SystemTime::now()), - body, - }; - if let Err(error) = writer.append(&line) { - self.failure = Some(error); - self.writer = None; - } - } +/// The end line's values that do not come from the outcome. +struct EndValues { + elapsed_ms: u64, + resources: Resources, + oom_kills: OomKills, + stop: Option, } -fn start_line(start: &RecordStart<'_>, container_oom_kills: OomKills) -> StartLine { +fn start_line( + start: &RecordStart<'_>, + sample_interval: Option, + container_oom_kills: OomKills, +) -> StartLine { StartLine { agent_name: cut_text(&start.runtime.agent_name), harness: cut_text(&start.runtime.harness), @@ -370,6 +382,8 @@ fn start_line(start: &RecordStart<'_>, container_oom_kills: OomKills) -> StartLi .collect(), fabric_version: crate::version().to_string(), timeout_ms: millis(start.timeout), + sample_interval_ms: sample_interval.map(millis), + resources: Resources::from(&(start.read_resources)()), context: start .request .context @@ -421,29 +435,20 @@ fn context_value(key: &str, value: &Value) -> ContextValue { } } -fn result_end_line( - result: &RunResult, - elapsed_ms: u64, - container_oom_kills: OomKills, - stop: Option, -) -> EndLine { +fn result_end_line(result: &RunResult, end: EndValues) -> EndLine { EndLine { status: result.status, - elapsed_ms, + elapsed_ms: end.elapsed_ms, error: result.error.as_ref().map(result_error), usage: result.usage.as_ref().map(end_usage), artifacts: end_artifacts(&result.artifacts), - container_oom_kills, - stop, + resources: end.resources, + container_oom_kills: end.oom_kills, + stop: end.stop, } } -fn error_end_line( - error: &FabricError, - elapsed_ms: u64, - container_oom_kills: OomKills, - stop: Option, -) -> EndLine { +fn error_end_line(error: &FabricError, end: EndValues) -> EndLine { let (code, stage) = match error { FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), // After the record starts, only the check of the adapter's result @@ -454,7 +459,7 @@ fn error_end_line( }; EndLine { status: RunStatus::Failed, - elapsed_ms, + elapsed_ms: end.elapsed_ms, error: Some(EndError { code: recorded_code(code), stage, @@ -462,8 +467,9 @@ fn error_end_line( }), usage: None, artifacts: Vec::new(), - container_oom_kills, - stop, + resources: end.resources, + container_oom_kills: end.oom_kills, + stop: end.stop, } } @@ -507,8 +513,25 @@ fn end_artifacts(manifest: &ArtifactManifest) -> Vec { mod tests { use super::*; use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; + use crate::record::lines::{RecordLine, RecordSchema, RecordVersion}; use crate::record::test_hooks::{self, Fault}; + fn snapshot() -> RuntimeResourceSnapshot { + RuntimeResourceSnapshot { + runtime_id: "rt".to_string(), + started_at: "2026-10-14T18:01:10.205Z".to_string(), + observed_at: "2026-10-14T18:01:11.205Z".to_string(), + resident_memory_bytes: 4096, + process_count: 2, + cpu_time_seconds: 0.5, + } + } + + /// A ticker whose thread never ticks during a test. + fn idle_ticker() -> Arc { + SampleTicker::new(Duration::from_secs(3600)) + } + fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { let config: FabricConfig = serde_json::from_value(serde_json::json!({ "schema_version": "fabric.agent/v1alpha1", @@ -559,6 +582,8 @@ mod tests { fabric_home: Path::new("/unused"), timeout: Duration::from_secs(5), cgroup: CgroupReader::system(), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }; let line = serde_json::to_value(RecordLine { @@ -568,7 +593,11 @@ mod tests { invocation_id: "inv".to_string(), request_id: "req".to_string(), time: "2026-10-14T18:01:10.205Z".to_string(), - body: LineBody::Start(start_line(&start, OomKills::Count(0))), + body: LineBody::Start(start_line( + &start, + Some(Duration::from_secs(10)), + OomKills::Count(0), + )), }) .expect("start line"); @@ -594,6 +623,8 @@ mod tests { 256 ); assert_eq!(line["timeout_ms"], 5_000); + assert_eq!(line["sample_interval_ms"], 10_000); + assert_eq!(line["resources"]["process_count"], 2); assert_eq!( line["models"], serde_json::json!({"default": {"provider": "nvidia", "model": "nemotron"}}) @@ -628,9 +659,11 @@ mod tests { fabric_home: Path::new("/unused"), timeout: Duration::from_secs(5), cgroup: CgroupReader::system(), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }; - let line = start_line(&start, OomKills::Count(0)); + let line = start_line(&start, None, OomKills::Count(0)); assert_eq!(line.models.len(), 3); assert_eq!(line.context.len(), 2); @@ -670,6 +703,8 @@ mod tests { fabric_home: &folder, timeout: Duration::from_secs(5), cgroup: CgroupReader::system(), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }); let mut result = succeeded_result(&invocation); result.artifacts.artifacts.push(ArtifactRef { @@ -707,6 +742,8 @@ mod tests { fabric_home: &folder, timeout: Duration::from_secs(5), cgroup: CgroupReader::system(), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }); let result = succeeded_result(&invocation); @@ -756,6 +793,8 @@ mod tests { fabric_home: &folder, timeout: Duration::from_secs(5), cgroup: CgroupReader::system(), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }); let finished = recorder.finish(outcome); test_hooks::FAULT.with(|slot| slot.set(None)); @@ -896,6 +935,8 @@ mod tests { fabric_home: &folder, timeout: Duration::from_secs(5), cgroup: CgroupReader::with_root(&root), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }); let count_the_kill = || std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)); @@ -903,7 +944,7 @@ mod tests { count_the_kill().expect("oom kill"); } let mut end = recorder - .read_exchange_end(Instant::now()) + .read_exchange_end(Instant::now(), &Ok(snapshot())) .with_host_exit(Some(HostExit { exit_code: None, exit_signal: Some(9), @@ -1011,8 +1052,10 @@ mod tests { fabric_home: &folder, timeout: Duration::from_secs(5), cgroup: CgroupReader::with_root(&root), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }); - recorder.exchange_ended(recorder.read_exchange_end(Instant::now())); + recorder.exchange_ended(recorder.read_exchange_end(Instant::now(), &Ok(snapshot()))); // Core released the host lock, so a later invocation on the same // runtime can be killed before this result is finished. std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)) @@ -1029,4 +1072,169 @@ mod tests { assert_eq!(end["container_oom_kills"], 4, "{end}"); assert_eq!(end["stop"], serde_json::json!({"cause": "agent_failed"})); } + + fn record_line_validator() -> jsonschema::Validator { + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunRecordLine) + .expect("record line schema"); + jsonschema::validator_for(&schema).expect("valid schema") + } + + /// Start a record in a fresh folder named `name`, sampled by `ticker`. + fn start_recorder( + name: &str, + ticker: &Arc, + read: ReadResources, + ) -> (PathBuf, InvocationRecorder) { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder(name); + let request = RunRequest::text("input"); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), + read_resources: read, + ticker: Arc::clone(ticker), + }); + (folder, recorder) + } + + fn read_lines(path: &Path) -> Vec { + std::fs::read_to_string(path) + .expect("record") + .lines() + .map(|line| serde_json::from_str(line).expect("json line")) + .collect() + } + + fn samples(path: &Path) -> Vec { + read_lines(path) + .into_iter() + .filter(|line| line["kind"] == "sample") + .collect() + } + + fn wait_for_samples(path: &Path, count: usize) { + // Wait for lines, not a fixed time, so a loaded runner cannot flake it. + let deadline = Instant::now() + Duration::from_secs(10); + while samples(path).len() < count && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + samples(path).len() >= count, + "fewer than {count} sample lines in {}", + path.display() + ); + } + + #[test] + fn two_records_share_one_ticker_and_samples_stop_at_the_end_line() { + let ticker = SampleTicker::new(Duration::from_millis(20)); + let (first_folder, mut first) = + start_recorder("share-1", &ticker, Arc::new(|| Ok(snapshot()))); + let (second_folder, second) = + start_recorder("share-2", &ticker, Arc::new(|| Ok(snapshot()))); + let first_path = first_folder.join(RECORD_FILE_NAME); + let second_path = second_folder.join(RECORD_FILE_NAME); + + wait_for_samples(&first_path, 2); + wait_for_samples(&second_path, 2); + let end = first + .read_exchange_end(Instant::now(), &Ok(snapshot())) + .with_host_exit(None); + first.exchange_ended(end); + let error = FabricError::AdapterLifecycleOperation { + operation: "invoke", + runtime_id: "rt".to_string(), + code: HOST_CRASHED.to_string(), + message: "host exited".to_string(), + diagnostics: String::new(), + }; + first.finish(Err(error)).expect_err("error passes through"); + std::thread::sleep(Duration::from_millis(60)); + + let lines = read_lines(&first_path); + let validator = record_line_validator(); + for line in lines.iter().chain(&read_lines(&second_path)) { + assert!(validator.is_valid(line), "{line}"); + } + assert_eq!(lines.last().expect("end")["kind"], "end"); + assert_eq!(lines.last().expect("end")["resources"]["process_count"], 2); + let samples = samples(&first_path); + assert!(samples.len() >= 2, "{lines:?}"); + assert!(samples[0]["elapsed_ms"].is_u64()); + assert_eq!(samples[0]["resources"]["process_count"], 2); + assert_eq!(lines[0]["sample_interval_ms"], 20); + drop(second); + let _ = std::fs::remove_dir_all(first_folder); + let _ = std::fs::remove_dir_all(second_folder); + } + + #[test] + fn sample_line_with_a_snapshot_error_keeps_only_the_error_code() { + let ticker = SampleTicker::new(Duration::from_millis(20)); + let (folder, recorder) = start_recorder( + "snapshot-error", + &ticker, + Arc::new(|| { + Err(FabricError::ResourceObservationUnavailable { + runtime_id: "rt".to_string(), + reason: "adapter host process has exited".to_string(), + }) + }), + ); + let path = folder.join(RECORD_FILE_NAME); + + wait_for_samples(&path, 1); + drop(recorder); + + let lines = read_lines(&path); + let unavailable = serde_json::json!({"error": "resource_observation_unavailable"}); + assert_eq!(samples(&path)[0]["resources"], unavailable); + assert_eq!(lines[0]["resources"], unavailable); + assert!( + !std::fs::read_to_string(&path) + .expect("record") + .contains("exited") + ); + let _ = std::fs::remove_dir_all(folder); + } + + #[test] + fn a_panicking_snapshot_read_still_writes_sample_lines_without_the_panic_text() { + let ticker = SampleTicker::new(Duration::from_millis(20)); + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let reads = Arc::clone(&calls); + let (folder, recorder) = start_recorder( + "panicking-read", + &ticker, + Arc::new(move || { + // The start line's read succeeds; every sample read panics. + if reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed) > 0 { + panic!("snapshot reader bug"); + } + Ok(snapshot()) + }), + ); + let path = folder.join(RECORD_FILE_NAME); + + wait_for_samples(&path, 2); + drop(recorder); + + let samples = samples(&path); + assert!(samples.len() >= 2, "{samples:?}"); + assert_eq!( + samples[1]["resources"], + serde_json::json!({"error": "resource_observation_unavailable"}) + ); + assert!( + !std::fs::read_to_string(&path) + .expect("record") + .contains("reader bug") + ); + let _ = std::fs::remove_dir_all(folder); + } } diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index 3e02f1789..405d46f15 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -13,8 +13,12 @@ use std::collections::BTreeMap; use schemars::JsonSchema; use serde::Serialize; +use crate::error::{FabricError, Result}; +use crate::resource_snapshot::RuntimeResourceSnapshot; use crate::runtime::{ErrorStage, RunStatus}; +/// Interval between sample lines while an invocation runs. +pub(crate) const SAMPLE_INTERVAL_MS: u64 = 10_000; /// Longest value a line keeps from config or the request, in characters. /// Model role and context key names are kept whole, so map keys stay distinct. const MAX_VALUE_CHARS: usize = 256; @@ -67,6 +71,9 @@ pub(crate) enum RecordVersion { pub(crate) enum LineBody { /// Written once, before core sends the request to the adapter host. Start(StartLine), + /// Written about every 10 s while the invocation runs, when the start + /// line has `sample_interval_ms`. + Sample(SampleLine), /// Written once, after the invocation returned a result or failed. End(EndLine), } @@ -88,6 +95,12 @@ pub(crate) struct StartLine { pub fabric_version: String, /// Host invocation timeout that applied, in milliseconds. pub timeout_ms: u64, + /// Interval between sample lines, in milliseconds. Absent when this + /// process has no sample thread, so the record gets no sample lines. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sample_interval_ms: Option, + /// The runtime's process values when the invocation started. + pub resources: Resources, /// `RunRequest.context`. Allowlisted identity and Harbor link keys keep a /// string or integer value; every other key is listed by name only. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] @@ -152,13 +165,74 @@ pub(crate) struct EndLine { /// Result artifacts, with paths relative to the artifact root. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub artifacts: Vec, - /// The container's cgroup v2 `oom_kill` count when the invocation ended. + /// The runtime's process values when the invocation ended, read under + /// the runtime's host lock right after the adapter host answered or the + /// timeout fired. + pub resources: Resources, + /// The container's cgroup v2 `oom_kill` count when the invocation ended, + /// read with `resources`. pub container_oom_kills: OomKills, /// Why the invocation stopped; absent when it holds nothing, as on success. #[serde(default, skip_serializing_if = "Option::is_none")] pub stop: Option, } +/// Fields of a sample line. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub(crate) struct SampleLine { + /// Monotonic time since the start line, in milliseconds. + pub elapsed_ms: u64, + /// The runtime's process values now. + pub resources: Resources, +} + +/// A runtime resource snapshot, or the error code that explains why none +/// could be read. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +#[serde(untagged)] +pub(crate) enum Resources { + /// The snapshot. + Snapshot(RuntimeResourceSnapshot), + /// No snapshot could be read. + Error { + /// Why no snapshot could be read. + error: ResourcesError, + }, +} + +impl Resources { + /// No snapshot, for a read that did not happen or did not finish. + pub(crate) const UNAVAILABLE: Self = Self::Error { + error: ResourcesError::ResourceObservationUnavailable, + }; +} + +impl From<&Result> for Resources { + /// Keep the snapshot, or only the error's code: never its message. + fn from(read: &Result) -> Self { + match read { + Ok(snapshot) => Self::Snapshot(snapshot.clone()), + Err(FabricError::ResourceObservationUnsupported { .. }) => Self::Error { + error: ResourcesError::ResourceObservationUnsupported, + }, + Err(_) => Self::UNAVAILABLE, + } + } +} + +/// Error codes for a snapshot that could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ResourcesError { + /// The runtime stopped or is stopping, its adapter host exited, a live + /// process in its tree or the process table cannot be read, or the + /// process tree kept changing during the read. + ResourceObservationUnavailable, + /// The adapter kind, environment provider, or operating system has no + /// resource reader. + ResourceObservationUnsupported, +} + /// The cgroup v2 `memory.events` `oom_kill` count of the container Fabric /// runs in, or why it could not be read. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 571584a4a..d02e45414 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -4,23 +4,27 @@ //! The opt-in per-invocation run record, `run-record.jsonl`. //! //! When `runtime.record.enabled` is set, core appends one start line before it -//! sends the request to the adapter host and one end line after the invocation -//! returns or fails. Recording is synchronous and best-effort: record I/O -//! errors do not replace the invocation result. A failed write stops the -//! record and becomes a `record_write_failed` event on the result, and slow -//! storage delays the invocation. +//! sends the request to the adapter host, a sample line about every 10 seconds +//! while the invocation runs, and one end line after it returns or fails. +//! Recording is synchronous and best-effort: record I/O errors do not replace +//! the invocation result. A failed write stops the record and becomes a +//! `record_write_failed` event on the result, and slow storage delays the +//! invocation. //! //! Everything here is crate-private. The line format is public only through //! the `run-record-line` JSON Schema, version `v1alpha1`. mod cgroup; +mod file; mod invocation; mod lines; +mod sampler; mod stop; mod writer; pub(crate) use cgroup::CgroupReader; pub(crate) use invocation::{HostExit, InvocationRecorder, RecordStart}; pub(crate) use lines::RecordLine; +pub(crate) use sampler::SampleTicker; #[cfg(test)] pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/record/sampler.rs b/crates/fabric-core/src/record/sampler.rs new file mode 100644 index 000000000..6c257a60c --- /dev/null +++ b/crates/fabric-core/src/record/sampler.rs @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One process-wide thread that appends a sample line to every active record. +//! +//! The thread starts with the first recorded invocation and serves every +//! runtime in the process, so N parallel runtimes cost one wake-up per +//! interval, not N threads. Each tick reads each record's runtime snapshot, +//! which never waits for the runtime's host lock. + +use std::collections::BTreeMap; +use std::panic::{self, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, LazyLock, Mutex, MutexGuard, Once, Weak}; +use std::thread; +use std::time::{Duration, Instant}; + +use super::file::{RecordFile, lock}; +use super::lines::{LineBody, Resources, SAMPLE_INTERVAL_MS, SampleLine}; +use crate::error::Result; +use crate::resource_snapshot::RuntimeResourceSnapshot; + +/// Reads a runtime's resource snapshot without waiting for its host lock. +pub(crate) type ReadResources = Arc Result + Send + Sync>; + +static GLOBAL: LazyLock> = + LazyLock::new(|| SampleTicker::new(Duration::from_millis(SAMPLE_INTERVAL_MS))); + +/// One active record the ticker samples. +struct ActiveRecord { + file: Arc>, + started: Instant, + read: ReadResources, +} + +impl ActiveRecord { + /// Read the snapshot, then append the sample line. The read runs without + /// the file lock, so a slow read never delays the end line. A read that + /// panics is written as unavailable, so the record keeps its heartbeat. + fn sample(&self) { + let resources = panic::catch_unwind(AssertUnwindSafe(|| Resources::from(&(self.read)()))) + .unwrap_or(Resources::UNAVAILABLE); + let sample = SampleLine { + elapsed_ms: u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX), + resources, + }; + lock(&self.file).append_sample(LineBody::Sample(sample)); + } +} + +/// Appends sample lines to active records at a fixed interval. +pub(crate) struct SampleTicker { + interval: Duration, + records: Mutex>>, + next_id: AtomicU64, + thread_started: Once, + running: AtomicBool, + /// The process that spawned the thread. A forked child inherits + /// `running` but not the thread. + spawned_by: AtomicU32, +} + +impl SampleTicker { + /// A ticker with its own interval; [`Self::start`] starts its thread. + pub(crate) fn new(interval: Duration) -> Arc { + Arc::new(Self { + interval, + records: Mutex::new(BTreeMap::new()), + next_id: AtomicU64::new(0), + thread_started: Once::new(), + running: AtomicBool::new(false), + spawned_by: AtomicU32::new(0), + }) + } + + /// The process-wide ticker, sampling every [`SAMPLE_INTERVAL_MS`]. + pub(crate) fn global() -> Arc { + GLOBAL.clone() + } + + /// Start the thread once, and return the sample interval when it runs + /// in this process. `None` means the operating system refused the thread, + /// or this process is a forked child of the one that started it, so + /// records get no sample lines. + pub(crate) fn start(self: &Arc) -> Option { + self.thread_started.call_once(|| { + // The thread holds a weak reference and ends when the ticker is + // dropped. The process-wide ticker is never dropped. + let ticker = Arc::downgrade(self); + let interval = self.interval; + let spawned = thread::Builder::new() + .name("fabric-record-samples".to_string()) + .spawn(move || run(&ticker, interval)); + self.spawned_by.store(std::process::id(), Ordering::Relaxed); + self.running.store(spawned.is_ok(), Ordering::Release); + }); + self.interval_in(std::process::id()) + } + + /// The sample interval when the thread runs in process `pid`. + fn interval_in(&self, pid: u32) -> Option { + (self.running.load(Ordering::Acquire) && self.spawned_by.load(Ordering::Relaxed) == pid) + .then_some(self.interval) + } + + /// Sample `file` until the returned registration is dropped. + pub(crate) fn register( + self: &Arc, + file: Arc>, + started: Instant, + read: ReadResources, + ) -> Registration { + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + self.lock_records().insert( + id, + Arc::new(ActiveRecord { + file, + started, + read, + }), + ); + Registration { + ticker: Arc::clone(self), + id, + } + } + + /// Append one sample line to every active record. + fn tick(&self) { + let active: Vec> = self.lock_records().values().cloned().collect(); + for record in active { + record.sample(); + } + } + + fn lock_records(&self) -> MutexGuard<'_, BTreeMap>> { + self.records + .lock() + .unwrap_or_else(|error| error.into_inner()) + } +} + +/// The ticker thread: tick on a fixed schedule until the ticker is gone. +/// Ticks are scheduled from the first deadline, so the time a tick takes +/// does not add up across ticks. +fn run(ticker: &Weak, interval: Duration) { + let mut next = Instant::now() + interval; + loop { + thread::sleep(next.saturating_duration_since(Instant::now())); + let Some(ticker) = ticker.upgrade() else { + return; + }; + ticker.tick(); + drop(ticker); + next += interval; + let now = Instant::now(); + if next < now { + // A tick longer than the interval skips the missed ticks. + next = now + interval; + } + } +} + +/// Keeps a record in the ticker; dropping it stops sampling that record. +pub(crate) struct Registration { + ticker: Arc, + id: u64, +} + +impl Drop for Registration { + fn drop(&mut self) { + self.ticker.lock_records().remove(&self.id); + } +} + +impl std::fmt::Debug for Registration { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("Registration") + .field("id", &self.id) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn start_returns_no_interval_in_a_process_that_did_not_spawn_the_thread() { + let ticker = SampleTicker::new(Duration::from_secs(3600)); + + assert_eq!(ticker.start(), Some(Duration::from_secs(3600))); + // A forked child keeps the parent's memory, so it sees a started + // thread, but it has another PID and no thread. + assert_eq!(ticker.interval_in(std::process::id().wrapping_add(1)), None); + } +} diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs index ea95049cf..4ac4f3bbd 100644 --- a/crates/fabric-core/src/record/writer.rs +++ b/crates/fabric-core/src/record/writer.rs @@ -86,6 +86,7 @@ pub(crate) mod test_hooks { match body { LineBody::Start(_) => fail_if(Fault::StartAppend), LineBody::End(_) => fail_if(Fault::EndAppend), + LineBody::Sample(_) => Ok(()), } } diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d6520830e..a4fba6bef 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -31,7 +31,7 @@ use crate::config::{ validate_tool_definitions, validate_workflow, }; use crate::error::{FabricError, Result}; -use crate::record::{CgroupReader, HostExit, InvocationRecorder, RecordStart}; +use crate::record::{CgroupReader, HostExit, InvocationRecorder, RecordStart, SampleTicker}; use crate::resource_snapshot::{LocalHostProcess, RuntimeResourceSnapshot, snapshot_local_host}; static NEXT_ID: AtomicU64 = AtomicU64::new(1); @@ -1876,6 +1876,11 @@ fn local_host_invocation_result( fabric_home: &fabric_home, timeout: invoke_timeout, cgroup: CgroupReader::system(), + read_resources: { + let runtime = runtime.clone(); + Arc::new(move || local_runtime_resource_snapshot(&runtime)) + }, + ticker: SampleTicker::global(), })); } adapter_invocation.runtime_context.deadline_millis = @@ -1921,17 +1926,19 @@ fn local_host_invocation_result( let ended_at = Instant::now(); // Read while the host lock is held and before a timed-out host is // stopped, so the end values count this invocation and no later one. + let end_snapshot = (exchange.is_ok() || recorder.is_some()) + .then(|| local_runtime_resource_snapshot(runtime)); let exchange_end = recorder .as_ref() - .map(|recorder| recorder.read_exchange_end(ended_at)); + .zip(end_snapshot.as_ref()) + .map(|(recorder, snapshot)| recorder.read_exchange_end(ended_at, snapshot)); match exchange { Ok(output) => { if let (Some(recorder), Some(end)) = (recorder.as_mut(), exchange_end) { recorder.exchange_ended(end); } let stderr = take_local_host_stderr(&mut host_guard); - // Read while the host lock is held, so no later invocation is counted. - let resource_snapshot = local_runtime_resource_snapshot(runtime).ok(); + let resource_snapshot = end_snapshot.and_then(Result::ok); Ok(( output, resource_snapshot, @@ -1951,7 +1958,8 @@ fn local_host_invocation_result( ) { invalidate_timed_out_local_host(&runtime.runtime_id, &host, &mut host_guard); } - // Only a record reads the exit status, so a run without one never waits. + // Only a record reads the exit status, so a run without one + // never waits. After a forced stop it is the signal core sent. if let (Some(recorder), Some(end)) = (recorder.as_mut(), exchange_end) { let crashed = matches!( &error, @@ -6291,4 +6299,49 @@ for line in sys.stdin: assert_eq!(stop["limit"]["value"], 3_600_000); assert!(stop["limit"]["usage"].is_u64(), "{stop}"); } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn record_start_and_end_lines_carry_the_runtime_snapshot() { + let (root, plan) = record_plan("success"); + + let result = run_plan(&plan, RunRequest::text("snapshot")).expect("run"); + let (_, lines) = only_record(&root); + + let (start, end) = (line_of_kind(&lines, "start"), line_of_kind(&lines, "end")); + assert_eq!(start["sample_interval_ms"], 10_000); + for line in [start, end] { + assert_eq!(line["resources"]["runtime_id"], result.runtime_id.as_str()); + assert!(line["resources"]["process_count"].as_u64().expect("count") >= 1); + } + assert_eq!( + end["resources"], + serde_json::to_value(result.resource_snapshot.expect("result snapshot")) + .expect("snapshot JSON") + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn record_end_line_of_a_timed_out_invocation_keeps_the_hung_host_snapshot() { + let (root, plan) = record_plan("invoke_timeout"); + let runtime = start_runtime(&plan).expect("start local host"); + + run_local_host_adapter_with_timeout( + &plan, + &runtime, + RunRequest::text("hang"), + Duration::from_millis(100), + ) + .expect_err("timeout raises"); + let (_, lines) = only_record(&root); + + let end = line_of_kind(&lines, "end"); + assert_eq!(end["error"]["code"], "host_timeout"); + assert_eq!(end["resources"]["runtime_id"], runtime.runtime_id.as_str()); + assert!(end["resources"]["process_count"].as_u64().expect("count") >= 1); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } } diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 016802bab..72000f93e 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1033,18 +1033,22 @@ result lists it as the `record` artifact with kind `run_record`: record = next(a for a in result.artifacts.artifacts if a.kind == "run_record") ``` -The file holds a start line and an end line. Every line carries `schema` +The file holds a start line, a sample line about every 10 seconds while the +invocation runs when the start line has `sample_interval_ms`, and an end +line. Every line carries `schema` (`nemo.fabric.run_record`), `version` (`v1alpha1`), `kind`, `runtime_id`, `invocation_id`, `request_id`, and `time`. - The `start` line is written before the request is sent. It holds the agent name, harness, adapter ID, each configured model's provider and model name, - the NeMo Fabric version, and the timeout in milliseconds. From + the NeMo Fabric version, the timeout and the sample interval in + milliseconds, and the runtime's resource snapshot. From `RunRequest.context`, it keeps the string or integer values of `task_id`, `task_revision`, `attempt`, `step`, `harbor_session_id`, `harbor_context_id`, and `harbor_trial_name`, cut to 256 characters. Every other context key appears by name only as `{"recorded": false}`. `override_keys` lists the top-level `RunRequest.overrides` key names, sorted, without their values. +- A `sample` line holds `elapsed_ms` and the runtime's resource snapshot. - The `end` line is written after the invocation returns or fails. It holds `status`, `elapsed_ms`, the error's `code`, `stage`, and `retryable` flag, the numeric `RunUsage` counters, and each result artifact's name, kind, @@ -1055,7 +1059,28 @@ The file holds a start line and an end line. Every line carries `schema` `host_io`, or `invalid_agent_run_result`. A result extension that the adapter's descriptor does not declare also reads `invalid_agent_run_result`. A failed write of `adapter-invocation.json` or a result artifact reads - `artifact_write_failed`, and any other error reads `fabric_error`. + `artifact_write_failed`, and any other error reads `fabric_error`. The end + line also holds the runtime's snapshot when the invocation ended, read right + after the adapter host answered or the timeout fired. + +The start, sample, and end lines store the snapshot as `resources`, with the +same fields as `RuntimeResourceSnapshot` (see +[Runtime Resource Snapshots](#runtime-resource-snapshots)). When no snapshot +can be read, `resources` is +`{"error": "resource_observation_unavailable"}` or +`{"error": "resource_observation_unsupported"}`. CPU time is cumulative since +the adapter host started. When both the start and end lines hold a snapshot, +the invocation's CPU time is the end line's `cpu_time_seconds` minus the start +line's. The difference leaves out descendants that left the process tree. + +One NeMo Fabric thread writes the sample lines for every recorded invocation +in the process. When the operating system refuses that thread, the start line +has no `sample_interval_ms` and the record gets no sample lines. A process +that forks after its first recorded invocation, such as Python +`multiprocessing` with the `fork` start method, has no sample thread in the +child, so in a forked child the start line has no `sample_interval_ms`. Use +the `spawn` or `forkserver` start method with recording; `fork` after threads +exist is unsafe. The end line describes the invocation. A one-shot `Fabric.run` stops the runtime after the invocation; when that stop fails, the result reads `failed` @@ -1135,9 +1160,14 @@ writes for every invocation whether or not recording is on; those files are not covered by this rule. Each line is written with one append, so a killed process keeps every line -whose write returned. NeMo Fabric syncs the file once, after the end line. A -record without an end line belongs to an invocation that is still running or -that never finished; treat its outcome as unknown, not as failed. +whose write returned. NeMo Fabric syncs the file once, after the end line. +When the start line has `sample_interval_ms`, a record without an end line +whose file changed within three sample intervals (30 seconds at the default) +belongs to a running invocation, and one that has not changed for longer +never finished, or its record stopped on a failed write (the result's +`record_write_failed` event says which). Without `sample_interval_ms`, a +record without an end line has an unknown outcome. Never count a record +without an end line as failed. Recording is synchronous and best-effort. If a write or the sync fails, the record stops. Record I/O errors do not replace the invocation result: a diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index 99839a0fb..7f3ec6f4b 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -86,7 +86,7 @@ }, "container_oom_kills": { "$ref": "#/$defs/OomKills", - "description": "The container's cgroup v2 `oom_kill` count when the invocation ended." + "description": "The container's cgroup v2 `oom_kill` count when the invocation ended,\nread with `resources`." }, "elapsed_ms": { "description": "Monotonic time from the start line to the end line, in milliseconds.", @@ -105,6 +105,10 @@ ], "description": "Error code, stage and retryable flag, without the message." }, + "resources": { + "$ref": "#/$defs/Resources", + "description": "The runtime's process values when the invocation ended, read under\nthe runtime's host lock right after the adapter host answered or the\ntimeout fired." + }, "status": { "$ref": "#/$defs/RunStatus", "description": "Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`." @@ -135,6 +139,7 @@ "required": [ "status", "elapsed_ms", + "resources", "container_oom_kills" ], "type": "object" @@ -348,6 +353,43 @@ } ] }, + "Resources": { + "anyOf": [ + { + "$ref": "#/$defs/RuntimeResourceSnapshot", + "description": "The snapshot." + }, + { + "description": "No snapshot could be read.", + "properties": { + "error": { + "$ref": "#/$defs/ResourcesError", + "description": "Why no snapshot could be read." + } + }, + "required": [ + "error" + ], + "type": "object" + } + ], + "description": "A runtime resource snapshot, or the error code that explains why none\ncould be read." + }, + "ResourcesError": { + "description": "Error codes for a snapshot that could not be read.", + "oneOf": [ + { + "const": "resource_observation_unavailable", + "description": "The runtime stopped or is stopping, its adapter host exited, a live\nprocess in its tree or the process table cannot be read, or the\nprocess tree kept changing during the read.", + "type": "string" + }, + { + "const": "resource_observation_unsupported", + "description": "The adapter kind, environment provider, or operating system has no\nresource reader.", + "type": "string" + } + ] + }, "RunStatus": { "description": "Runtime completion status.", "oneOf": [ @@ -368,6 +410,72 @@ } ] }, + "RuntimeResourceSnapshot": { + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", + "properties": { + "cpu_time_seconds": { + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", + "format": "double", + "minimum": 0.0, + "type": "number" + }, + "observed_at": { + "description": "When Fabric read these values, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + }, + "process_count": { + "description": "Live processes in the tree, including the adapter host. Exited processes\nthat are not yet reaped are not counted.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "resident_memory_bytes": { + "description": "Sum of the resident set sizes of the live processes, in bytes. Pages\nshared between processes count once per process.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "runtime_id": { + "description": "Runtime this snapshot describes.", + "type": "string" + }, + "started_at": { + "description": "When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + } + }, + "required": [ + "runtime_id", + "started_at", + "observed_at", + "resident_memory_bytes", + "process_count", + "cpu_time_seconds" + ], + "type": "object" + }, + "SampleLine": { + "description": "Fields of a sample line.", + "properties": { + "elapsed_ms": { + "description": "Monotonic time since the start line, in milliseconds.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "resources": { + "$ref": "#/$defs/Resources", + "description": "The runtime's process values now." + } + }, + "required": [ + "elapsed_ms", + "resources" + ], + "type": "object" + }, "StartLine": { "description": "Fields of the start line.", "properties": { @@ -419,6 +527,19 @@ }, "type": "array" }, + "resources": { + "$ref": "#/$defs/Resources", + "description": "The runtime's process values when the invocation started." + }, + "sample_interval_ms": { + "description": "Interval between sample lines, in milliseconds. Absent when this\nprocess has no sample thread, so the record gets no sample lines.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, "timeout_ms": { "description": "Host invocation timeout that applied, in milliseconds.", "format": "uint64", @@ -431,6 +552,7 @@ "harness", "fabric_version", "timeout_ms", + "resources", "exclusive_container", "container_oom_kills" ], @@ -583,6 +705,20 @@ ], "type": "object" }, + { + "$ref": "#/$defs/SampleLine", + "description": "Written about every 10 s while the invocation runs, when the start\nline has `sample_interval_ms`.", + "properties": { + "kind": { + "const": "sample", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, { "$ref": "#/$defs/EndLine", "description": "Written once, after the invocation returned a result or failed.", diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index d02191a48..aa559eb71 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -53,19 +53,26 @@ Set `RuntimeConfig(artifacts=..., record=RecordConfig(enabled=True))` to have NeMo Fabric write `run-record.jsonl` for each invocation under `/.fabric///`. Recording is off by default and needs an artifact root, the `local` environment provider, and a -`process` or `python` adapter. The -result lists the file as the artifact with kind `run_record`. The file has a -`start` line, written before the request, and an `end` line with status, -elapsed time, error code, numeric usage, artifact names, and `stop`, which +`process` or `python` adapter. The result lists the file as the artifact with +kind `run_record`. The file has a `start` line, written before the request, a +`sample` line about every 10 seconds when the start line has +`sample_interval_ms`, and an `end` line with status, elapsed time, error +code, numeric usage, artifact names, and `stop`, which gives NeMo Fabric's failure cause (`fabric_failed`, `sandbox_failed`, `agent_exhausted_budget`, `unclassified`, or `agent_failed`), the time limit for a timeout, and the adapter host's exit status. Only a `failed` result -reads `agent_failed`; an error NeMo Fabric raises never does. It holds no -prompts, output, or error messages. A record without an end line has an -unknown outcome; do not count it as failed. Recording is synchronous and -best-effort: a failed record write adds a `record_write_failed` event and does -not replace the result, and slow storage delays the invocation. The format is -the `v1alpha1` +reads `agent_failed`; an error NeMo Fabric raises never does. Each line +stores the runtime's resource snapshot as `resources`. When both the start and +end lines hold one, the invocation's CPU time is end minus start, leaving out +descendants that left the process tree. It holds no prompts, output, or error +messages. When the start line has `sample_interval_ms`, a record without an +end line whose file changed within three sample intervals belongs to a running +invocation, and one that has not changed for longer never finished, or its +record stopped on a failed write (the result's `record_write_failed` event +says which). Without `sample_interval_ms`, its outcome is unknown. Never count a record without an +end line as failed. Recording is synchronous and best-effort: a failed record +write adds a `record_write_failed` event and does not replace the result, and +slow storage delays the invocation. The format is the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) and may change until a reader ships. From bc7da5cc76f7611669e4c33bef9efaafe2b78b67 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:33:28 -0700 Subject: [PATCH 035/102] fix: close a record's sampling window when its exchange ends A record stayed registered with the sample ticker until finish, which runs after core releases the host lock and processes the result. In that window another caller can start the next invocation on the same runtime, and the earlier record kept sampling the runtime while the later invocation ran, so its sample lines described the wrong invocation. Core now takes the record's sampling registration under the host lock and drops it when the exchange ends. Dropping it removes the record from the ticker and marks the file closed for sampling, so a sample whose read finishes later is discarded. The registration is declared after the host lock guard, so an early return also closes the window before the lock is released. The regression test ends invocation A's exchange, runs invocation B on the same runtime before A's result is finished, and waits for three more ticks; before this change A's record gained a sample on every one. Tests can give records started on their thread a fast ticker. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/file.rs | 19 ++-- crates/fabric-core/src/record/invocation.rs | 7 ++ crates/fabric-core/src/record/mod.rs | 2 + crates/fabric-core/src/record/sampler.rs | 28 +++++- crates/fabric-core/src/runtime.rs | 104 ++++++++++++++++++++ 5 files changed, 152 insertions(+), 8 deletions(-) diff --git a/crates/fabric-core/src/record/file.rs b/crates/fabric-core/src/record/file.rs index 373d8908c..be2aa26ce 100644 --- a/crates/fabric-core/src/record/file.rs +++ b/crates/fabric-core/src/record/file.rs @@ -21,8 +21,9 @@ pub(crate) struct RecordFile { writer: Option, /// The first create, write or sync failure. pub failure: Option, - /// Set when the end line is written; no sample line follows it. - ended: bool, + /// Set when the sampling window closes, at the exchange's end or at the + /// end line; no sample line follows it. + sampling_closed: bool, } impl RecordFile { @@ -44,7 +45,7 @@ impl RecordFile { request_id, writer, failure, - ended: false, + sampling_closed: false, } } @@ -68,18 +69,24 @@ impl RecordFile { } } - /// Append a sample line, unless the end line is already written. + /// Append a sample line, unless the sampling window is closed. pub(crate) fn append_sample(&mut self, body: LineBody) { - if !self.ended { + if !self.sampling_closed { self.append(body); } } + /// Close the sampling window. A sample whose read finishes later is + /// discarded. + pub(crate) fn close_sampling(&mut self) { + self.sampling_closed = true; + } + /// Append the end line, sync the file and close it. No sample line /// follows, and a sample tick that still holds this record keeps no open /// file handle. pub(crate) fn append_end(&mut self, body: LineBody) { - self.ended = true; + self.sampling_closed = true; self.append(body); if let Some(writer) = self.writer.take() && let Err(error) = writer.sync() diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 042b332f1..a202a7f84 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -217,6 +217,13 @@ impl InvocationRecorder { ExchangeEnd { oom_kills, ..end } } + /// The record's sampling registration. Core holds it under the host lock + /// and drops it when the exchange ends, which closes the sampling window + /// before a later invocation on the runtime can start. + pub(crate) fn take_sampling(&mut self) -> Option { + self.sampling.take() + } + /// Keep what the exchange saw for the end line. pub(crate) fn exchange_ended(&mut self, end: ExchangeEnd) { self.exchange = Some(end); diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index d02e45414..5bf3dd1ee 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -27,4 +27,6 @@ pub(crate) use invocation::{HostExit, InvocationRecorder, RecordStart}; pub(crate) use lines::RecordLine; pub(crate) use sampler::SampleTicker; #[cfg(test)] +pub(crate) use sampler::test_hooks as sampler_test_hooks; +#[cfg(test)] pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/record/sampler.rs b/crates/fabric-core/src/record/sampler.rs index 6c257a60c..a5cdc487e 100644 --- a/crates/fabric-core/src/record/sampler.rs +++ b/crates/fabric-core/src/record/sampler.rs @@ -75,6 +75,10 @@ impl SampleTicker { /// The process-wide ticker, sampling every [`SAMPLE_INTERVAL_MS`]. pub(crate) fn global() -> Arc { + #[cfg(test)] + if let Some(ticker) = test_hooks::TICKER.with(|slot| slot.borrow().clone()) { + return ticker; + } GLOBAL.clone() } @@ -114,7 +118,7 @@ impl SampleTicker { self.lock_records().insert( id, Arc::new(ActiveRecord { - file, + file: Arc::clone(&file), started, read, }), @@ -122,6 +126,7 @@ impl SampleTicker { Registration { ticker: Arc::clone(self), id, + file, } } @@ -161,15 +166,19 @@ fn run(ticker: &Weak, interval: Duration) { } } -/// Keeps a record in the ticker; dropping it stops sampling that record. +/// Keeps a record in the ticker. Dropping it closes the record's sampling +/// window: the ticker stops sampling the record, and a sample whose read is +/// still running is discarded. pub(crate) struct Registration { ticker: Arc, id: u64, + file: Arc>, } impl Drop for Registration { fn drop(&mut self) { self.ticker.lock_records().remove(&self.id); + lock(&self.file).close_sampling(); } } @@ -182,6 +191,21 @@ impl std::fmt::Debug for Registration { } } +/// A per-thread ticker that replaces the process-wide one in tests. +#[cfg(test)] +pub(crate) mod test_hooks { + use std::cell::RefCell; + use std::sync::Arc; + + use super::SampleTicker; + + thread_local! { + /// Ticker for records started on this thread, instead of the + /// process-wide one. + pub(crate) static TICKER: RefCell>> = const { RefCell::new(None) }; + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index a4fba6bef..cfecec717 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1883,6 +1883,12 @@ fn local_host_invocation_result( ticker: SampleTicker::global(), })); } + // Dropping `sampling` closes the record's sampling window. It is + // declared after `host_guard`, so an early return also closes the + // window before the host lock is released. + let sampling = recorder + .as_mut() + .and_then(InvocationRecorder::take_sampling); adapter_invocation.runtime_context.deadline_millis = adapter_deadline_millis(invoke_timeout); let (lifecycle_request, adapter_payload) = match invocation_kind { @@ -1932,6 +1938,9 @@ fn local_host_invocation_result( .as_ref() .zip(end_snapshot.as_ref()) .map(|(recorder, snapshot)| recorder.read_exchange_end(ended_at, snapshot)); + // No later sample belongs to this invocation: the next invocation on + // this runtime can start as soon as the host lock is released. + drop(sampling); match exchange { Ok(output) => { if let (Some(recorder), Some(end)) = (recorder.as_mut(), exchange_end) { @@ -6344,4 +6353,99 @@ for line in sys.stdin: let _ = stop_runtime(&plan, &runtime); let _ = fs::remove_dir_all(root); } + + fn sample_lines(record: &Path) -> usize { + fs::read_to_string(record) + .expect("read record") + .lines() + .filter(|line| line.contains(r#""kind":"sample""#)) + .count() + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn record_gets_no_sample_after_its_exchange_ends_while_a_later_invocation_runs() { + let (root, plan) = record_plan("success"); + let ticker = SampleTicker::new(Duration::from_millis(10)); + crate::record::sampler_test_hooks::TICKER + .with(|slot| *slot.borrow_mut() = Some(Arc::clone(&ticker))); + let runtime = start_runtime(&plan).expect("start local host"); + + // Invocation A's exchange ends, and its result is not finished yet. + let mut first = None; + let first_outcome = local_host_invocation_result( + &plan, + &runtime, + RunRequest::text("first"), + LocalHostInvocation::Invoke, + Duration::from_secs(30), + &mut first, + ); + let first = first.expect("first record"); + let first_record = record_files(&root).pop().expect("first record file"); + let samples_at_exchange_end = sample_lines(&first_record); + // Invocation B runs on the same runtime while A is unfinished. + run_local_host_adapter_with_timeout( + &plan, + &runtime, + RunRequest::text("second"), + Duration::from_secs(30), + ) + .expect("second invocation"); + // A record that stays open on the same ticker shows that the ticker + // ticked after both exchanges. + let sentinel_home = root.join("sentinel"); + fs::create_dir_all(&sentinel_home).expect("sentinel folder"); + let sentinel_invocation = InvocationHandle { + invocation_id: "invocation-sentinel".to_string(), + request_id: "request-sentinel".to_string(), + runtime_id: runtime.runtime_id.clone(), + }; + let sentinel = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &sentinel_invocation, + request: &RunRequest::text("sentinel"), + fabric_home: &sentinel_home, + timeout: Duration::from_secs(30), + cgroup: CgroupReader::system(), + read_resources: { + let runtime = runtime.clone(); + Arc::new(move || local_runtime_resource_snapshot(&runtime)) + }, + ticker: Arc::clone(&ticker), + }); + let sentinel_record = sentinel_home.join("run-record.jsonl"); + let deadline = Instant::now() + Duration::from_secs(10); + while sample_lines(&sentinel_record) < 3 && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + assert!( + sample_lines(&sentinel_record) >= 3, + "the ticker never ticked" + ); + + first.finish(first_outcome).expect("first result"); + drop(sentinel); + crate::record::sampler_test_hooks::TICKER.with(|slot| *slot.borrow_mut() = None); + + let lines: Vec = fs::read_to_string(&first_record) + .expect("read record") + .lines() + .map(|line| serde_json::from_str(line).expect("record line")) + .collect(); + let sample_times: Vec<&Value> = lines + .iter() + .filter(|line| line["kind"] == "sample") + .map(|line| &line["elapsed_ms"]) + .collect(); + assert_eq!( + sample_times.len(), + samples_at_exchange_end, + "sample elapsed_ms: {sample_times:?}" + ); + assert_eq!(lines.last().expect("end line")["kind"], "end"); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } } From 8e7eda7269964accfb186c3297727042eb155ab5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:33:36 -0700 Subject: [PATCH 036/102] test: block a sample read across the end line instead of sleeping The shared-ticker test slept 60 ms after the end line and checked that no sample followed, which passes whenever the ticker happens not to tick in that window. The new test blocks the first sample read, writes the end line while the read is in flight, releases the read, and waits until a second record on the same ticker gets its next sample, so the blocked tick has finished before the record is read. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 45 ++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index a202a7f84..e693ac6bf 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1161,7 +1161,6 @@ mod tests { diagnostics: String::new(), }; first.finish(Err(error)).expect_err("error passes through"); - std::thread::sleep(Duration::from_millis(60)); let lines = read_lines(&first_path); let validator = record_line_validator(); @@ -1180,6 +1179,50 @@ mod tests { let _ = std::fs::remove_dir_all(second_folder); } + #[test] + fn a_sample_read_in_flight_when_the_record_ends_writes_nothing_after_the_end_line() { + use std::sync::Barrier; + use std::sync::atomic::{AtomicUsize, Ordering}; + let ticker = SampleTicker::new(Duration::from_millis(10)); + let (entered, release) = (Arc::new(Barrier::new(2)), Arc::new(Barrier::new(2))); + let calls = Arc::new(AtomicUsize::new(0)); + let read: ReadResources = { + let (entered, release) = (Arc::clone(&entered), Arc::clone(&release)); + Arc::new(move || { + // Call 0 is the start line's read; call 1 is the first sample + // read, which waits until the test has finished the record. + if calls.fetch_add(1, Ordering::SeqCst) == 1 { + entered.wait(); + release.wait(); + } + Ok(snapshot()) + }) + }; + let (folder, recorder) = start_recorder("in-flight", &ticker, read); + // A second record on the same ticker, sampled after the first in + // each tick, shows when the blocked tick has finished. + let (sentinel_folder, sentinel) = + start_recorder("in-flight-sentinel", &ticker, Arc::new(|| Ok(snapshot()))); + let path = folder.join(RECORD_FILE_NAME); + let sentinel_path = sentinel_folder.join(RECORD_FILE_NAME); + + entered.wait(); + let (_, _, invocation) = record_test_fixture(); + recorder + .finish(Ok(succeeded_result(&invocation))) + .expect("result"); + let sentinel_samples = samples(&sentinel_path).len(); + release.wait(); + wait_for_samples(&sentinel_path, sentinel_samples + 1); + + let lines = read_lines(&path); + assert_eq!(lines.last().expect("end line")["kind"], "end", "{lines:?}"); + assert!(samples(&path).is_empty(), "{lines:?}"); + drop(sentinel); + let _ = std::fs::remove_dir_all(folder); + let _ = std::fs::remove_dir_all(sentinel_folder); + } + #[test] fn sample_line_with_a_snapshot_error_keeps_only_the_error_code() { let ticker = SampleTicker::new(Duration::from_millis(20)); From f7c688684cf1cfda7b9e3f2a25bda8e99e0bf95e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:34:52 -0700 Subject: [PATCH 037/102] docs: say what a recent sample and a stale run record show The guides read a record whose file changed within three sample intervals as a running invocation and an older one as never finished. Neither follows: the one sample thread serves records one after another, so a slow snapshot read delays every record's samples, a suspended process can resume, and a sample says Fabric observed the record, not that the agent made progress. A recent sample now means recently observed; after three intervals the record is stale and its outcome is unknown until an end line exists, and a stale record is never failed. Also say that end minus start CPU time is a partial observed delta whose decrease reads as unknown, that the end oom_kill count read with the snapshot can be replaced by a larger count after a crashed host's exit and is best effort, that sampling does not take the host lock rather than that it costs nothing, that sample lines stop when the exchange ends, and that a forked child stops writing samples, without a general claim about fork safety. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/lines.rs | 12 ++++-- crates/fabric-core/src/record/mod.rs | 3 +- crates/fabric-core/src/record/sampler.rs | 5 ++- docs/sdk/python.mdx | 42 ++++++++++--------- schemas/sdk/run-record-line.schema.json | 4 +- .../references/results-and-errors.md | 24 ++++++----- 6 files changed, 52 insertions(+), 38 deletions(-) diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index 405d46f15..a7f70f5a9 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -71,8 +71,10 @@ pub(crate) enum RecordVersion { pub(crate) enum LineBody { /// Written once, before core sends the request to the adapter host. Start(StartLine), - /// Written about every 10 s while the invocation runs, when the start - /// line has `sample_interval_ms`. + /// Written about every 10 s until the exchange with the adapter host + /// ends, when the start line has `sample_interval_ms`. A recent sample + /// shows that Fabric observed the record recently, not that the agent is + /// making progress. Sample(SampleLine), /// Written once, after the invocation returned a result or failed. End(EndLine), @@ -169,8 +171,10 @@ pub(crate) struct EndLine { /// the runtime's host lock right after the adapter host answered or the /// timeout fired. pub resources: Resources, - /// The container's cgroup v2 `oom_kill` count when the invocation ended, - /// read with `resources`. + /// The container's cgroup v2 `oom_kill` count, read with `resources`. + /// After a crashed host's exit, a larger count read then replaces it. + /// Best effort: the kernel can count a kill after the host's exit is + /// seen. pub container_oom_kills: OomKills, /// Why the invocation stopped; absent when it holds nothing, as on success. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 5bf3dd1ee..b205d64ad 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -5,7 +5,8 @@ //! //! When `runtime.record.enabled` is set, core appends one start line before it //! sends the request to the adapter host, a sample line about every 10 seconds -//! while the invocation runs, and one end line after it returns or fails. +//! until the exchange with the host ends, and one end line after the +//! invocation returns or fails. //! Recording is synchronous and best-effort: record I/O errors do not replace //! the invocation result. A failed write stops the record and becomes a //! `record_write_failed` event on the result, and slow storage delays the diff --git a/crates/fabric-core/src/record/sampler.rs b/crates/fabric-core/src/record/sampler.rs index a5cdc487e..23f4f9427 100644 --- a/crates/fabric-core/src/record/sampler.rs +++ b/crates/fabric-core/src/record/sampler.rs @@ -6,7 +6,8 @@ //! The thread starts with the first recorded invocation and serves every //! runtime in the process, so N parallel runtimes cost one wake-up per //! interval, not N threads. Each tick reads each record's runtime snapshot, -//! which never waits for the runtime's host lock. +//! which never takes the runtime's host lock. Records are sampled one after +//! another, so one slow read delays the samples of every record. use std::collections::BTreeMap; use std::panic::{self, AssertUnwindSafe}; @@ -36,7 +37,7 @@ struct ActiveRecord { impl ActiveRecord { /// Read the snapshot, then append the sample line. The read runs without /// the file lock, so a slow read never delays the end line. A read that - /// panics is written as unavailable, so the record keeps its heartbeat. + /// panics is written as unavailable, so the record keeps its sample lines. fn sample(&self) { let resources = panic::catch_unwind(AssertUnwindSafe(|| Resources::from(&(self.read)()))) .unwrap_or(Resources::UNAVAILABLE); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 72000f93e..11aa774a0 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1033,9 +1033,9 @@ result lists it as the `record` artifact with kind `run_record`: record = next(a for a in result.artifacts.artifacts if a.kind == "run_record") ``` -The file holds a start line, a sample line about every 10 seconds while the -invocation runs when the start line has `sample_interval_ms`, and an end -line. Every line carries `schema` +The file holds a start line, a sample line about every 10 seconds until the +adapter host answers or the timeout fires when the start line has +`sample_interval_ms`, and an end line. Every line carries `schema` (`nemo.fabric.run_record`), `version` (`v1alpha1`), `kind`, `runtime_id`, `invocation_id`, `request_id`, and `time`. @@ -1070,17 +1070,18 @@ can be read, `resources` is `{"error": "resource_observation_unavailable"}` or `{"error": "resource_observation_unsupported"}`. CPU time is cumulative since the adapter host started. When both the start and end lines hold a snapshot, -the invocation's CPU time is the end line's `cpu_time_seconds` minus the start -line's. The difference leaves out descendants that left the process tree. +the end line's `cpu_time_seconds` minus the start line's is a partial observed +CPU delta for the invocation: it leaves out descendants that left the process +tree before the end snapshot. Read a decrease as unknown, never as zero or a +negative CPU time. One NeMo Fabric thread writes the sample lines for every recorded invocation -in the process. When the operating system refuses that thread, the start line -has no `sample_interval_ms` and the record gets no sample lines. A process -that forks after its first recorded invocation, such as Python -`multiprocessing` with the `fork` start method, has no sample thread in the -child, so in a forked child the start line has no `sample_interval_ms`. Use -the `spawn` or `forkserver` start method with recording; `fork` after threads -exist is unsafe. +in the process, one record after another. Sampling does not take the host +lock. When the operating system refuses that thread, the start line has no +`sample_interval_ms` and the record gets no sample lines. A forked child stops +writing samples: in a process forked after its first recorded invocation, such +as a Python `multiprocessing` child with the `fork` start method, the start +line has no `sample_interval_ms` and the record gets no sample lines. The end line describes the invocation. A one-shot `Fabric.run` stops the runtime after the invocation; when that stop fails, the result reads `failed` @@ -1161,13 +1162,16 @@ not covered by this rule. Each line is written with one append, so a killed process keeps every line whose write returned. NeMo Fabric syncs the file once, after the end line. -When the start line has `sample_interval_ms`, a record without an end line -whose file changed within three sample intervals (30 seconds at the default) -belongs to a running invocation, and one that has not changed for longer -never finished, or its record stopped on a failed write (the result's -`record_write_failed` event says which). Without `sample_interval_ms`, a -record without an end line has an unknown outcome. Never count a record -without an end line as failed. +When the start line has `sample_interval_ms`, a recent sample shows that NeMo +Fabric observed the record recently, not that the agent is making progress. +After three sample intervals (30 seconds at the default) without an update, +the record is stale, and the invocation's outcome is unknown until an end +line exists. A stale record is never counted as failed: one slow snapshot read +delays samples for every record in the process, and a suspended process can +resume. A record also stops changing when a write fails; the result's +`record_write_failed` event says so. Without `sample_interval_ms`, a record +without an end line has an unknown outcome. Never count a record without an +end line as failed. Recording is synchronous and best-effort. If a write or the sync fails, the record stops. Record I/O errors do not replace the invocation result: a diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index 7f3ec6f4b..f14f319a7 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -86,7 +86,7 @@ }, "container_oom_kills": { "$ref": "#/$defs/OomKills", - "description": "The container's cgroup v2 `oom_kill` count when the invocation ended,\nread with `resources`." + "description": "The container's cgroup v2 `oom_kill` count, read with `resources`.\nAfter a crashed host's exit, a larger count read then replaces it.\nBest effort: the kernel can count a kill after the host's exit is\nseen." }, "elapsed_ms": { "description": "Monotonic time from the start line to the end line, in milliseconds.", @@ -707,7 +707,7 @@ }, { "$ref": "#/$defs/SampleLine", - "description": "Written about every 10 s while the invocation runs, when the start\nline has `sample_interval_ms`.", + "description": "Written about every 10 s until the exchange with the adapter host\nends, when the start line has `sample_interval_ms`. A recent sample\nshows that Fabric observed the record recently, not that the agent is\nmaking progress.", "properties": { "kind": { "const": "sample", diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index aa559eb71..a64699e3d 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -55,22 +55,26 @@ NeMo Fabric write `run-record.jsonl` for each invocation under default and needs an artifact root, the `local` environment provider, and a `process` or `python` adapter. The result lists the file as the artifact with kind `run_record`. The file has a `start` line, written before the request, a -`sample` line about every 10 seconds when the start line has -`sample_interval_ms`, and an `end` line with status, elapsed time, error +`sample` line about every 10 seconds until the adapter host answers or the +timeout fires, when the start line has `sample_interval_ms`, and an `end` line with status, elapsed time, error code, numeric usage, artifact names, and `stop`, which gives NeMo Fabric's failure cause (`fabric_failed`, `sandbox_failed`, `agent_exhausted_budget`, `unclassified`, or `agent_failed`), the time limit for a timeout, and the adapter host's exit status. Only a `failed` result reads `agent_failed`; an error NeMo Fabric raises never does. Each line stores the runtime's resource snapshot as `resources`. When both the start and -end lines hold one, the invocation's CPU time is end minus start, leaving out -descendants that left the process tree. It holds no prompts, output, or error -messages. When the start line has `sample_interval_ms`, a record without an -end line whose file changed within three sample intervals belongs to a running -invocation, and one that has not changed for longer never finished, or its -record stopped on a failed write (the result's `record_write_failed` event -says which). Without `sample_interval_ms`, its outcome is unknown. Never count a record without an -end line as failed. Recording is synchronous and best-effort: a failed record +end lines hold one, end minus start is a partial observed CPU delta that +leaves out descendants that left the process tree; read a decrease as +unknown, never as zero or negative. It holds no prompts, output, or error +messages. A recent sample shows that NeMo Fabric observed the record +recently, not that the agent is making progress. After three sample intervals +without an update, the record is stale and the invocation's outcome is +unknown until an end line exists; never count a stale record as failed. One +slow snapshot read delays samples for every record in the process, and a +suspended process can resume. A record also stops changing when a write +fails; the result's `record_write_failed` event says so. Without +`sample_interval_ms`, a record without an end line has an unknown outcome. +Recording is synchronous and best-effort: a failed record write adds a `record_write_failed` event and does not replace the result, and slow storage delays the invocation. The format is the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) From 7e2be97645241fe1c162dfd27e638e5540297f7c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:44:46 -0700 Subject: [PATCH 038/102] test: build the sampling window test helpers only where the test runs The regression test for the sampling window runs on Linux and macOS, so on Windows its record helper and the ticker hook re-export were unused and clippy warned. Gate both to the same platforms. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/mod.rs | 2 +- crates/fabric-core/src/runtime.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index b205d64ad..8bf3551b0 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -27,7 +27,7 @@ pub(crate) use cgroup::CgroupReader; pub(crate) use invocation::{HostExit, InvocationRecorder, RecordStart}; pub(crate) use lines::RecordLine; pub(crate) use sampler::SampleTicker; -#[cfg(test)] +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] pub(crate) use sampler::test_hooks as sampler_test_hooks; #[cfg(test)] pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index cfecec717..52737acc4 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -6354,6 +6354,7 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[cfg(any(target_os = "linux", target_os = "macos"))] fn sample_lines(record: &Path) -> usize { fs::read_to_string(record) .expect("read record") From 24407bb942fb71d65e07542d0dd194b84752ab00 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:59:43 -0700 Subject: [PATCH 039/102] docs: name NeMo Fabric in the resource snapshot docs The RuntimeResourceSnapshot overview, the started_at and observed_at descriptions, the LocalHostProcess::new doc comment and the Python docstring used a standalone "Fabric" for the product. The repository's naming rule asks for "NeMo Fabric" after first use. The two snapshot schemas are regenerated with just schemas; the TypeScript contract does not change. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 10 +++++----- schemas/sdk/run-result.schema.json | 6 +++--- schemas/sdk/runtime-resource-snapshot.schema.json | 6 +++--- .../nemo-fabric-runtime/src/nemo_fabric/types.py | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 61755580b..7e07a1c99 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -39,16 +39,16 @@ const NANOS_PER_SECOND: f64 = 1_000_000_000.0; /// /// Memory, process count, and CPU time cover the adapter host process and its /// live descendants. For a runtime started on a service, the shared service -/// process is not included. Values Fabric cannot read are returned as an error, -/// never as zero. +/// process is not included. Values NeMo Fabric cannot read are returned as an +/// error, never as zero. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct RuntimeResourceSnapshot { /// Runtime this snapshot describes. pub runtime_id: String, - /// When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. + /// When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. #[schemars(extend("format" = "date-time"))] pub started_at: String, - /// When Fabric read these values, as an RFC 3339 UTC timestamp. + /// When NeMo Fabric read these values, as an RFC 3339 UTC timestamp. #[schemars(extend("format" = "date-time"))] pub observed_at: String, /// Sum of the resident set sizes of the live processes, in bytes. Pages @@ -75,7 +75,7 @@ pub(crate) struct LocalHostProcess { } impl LocalHostProcess { - /// Record the host `pid`, started by Fabric at `started_at`. + /// Record the host `pid`, started by NeMo Fabric at `started_at`. pub(crate) fn new(pid: u32, started_at: SystemTime) -> Self { Self { pid, diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 504d6b129..7d5e1ef80 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -280,7 +280,7 @@ "type": "object" }, "RuntimeResourceSnapshot": { - "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values NeMo Fabric cannot read are returned as an\nerror, never as zero.", "properties": { "cpu_time_seconds": { "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", @@ -289,7 +289,7 @@ "type": "number" }, "observed_at": { - "description": "When Fabric read these values, as an RFC 3339 UTC timestamp.", + "description": "When NeMo Fabric read these values, as an RFC 3339 UTC timestamp.", "format": "date-time", "type": "string" }, @@ -310,7 +310,7 @@ "type": "string" }, "started_at": { - "description": "When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "description": "When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", "format": "date-time", "type": "string" } diff --git a/schemas/sdk/runtime-resource-snapshot.schema.json b/schemas/sdk/runtime-resource-snapshot.schema.json index 0b3ac2916..3b3a8747b 100644 --- a/schemas/sdk/runtime-resource-snapshot.schema.json +++ b/schemas/sdk/runtime-resource-snapshot.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values NeMo Fabric cannot read are returned as an\nerror, never as zero.", "properties": { "cpu_time_seconds": { "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", @@ -9,7 +9,7 @@ "type": "number" }, "observed_at": { - "description": "When Fabric read these values, as an RFC 3339 UTC timestamp.", + "description": "When NeMo Fabric read these values, as an RFC 3339 UTC timestamp.", "format": "date-time", "type": "string" }, @@ -30,7 +30,7 @@ "type": "string" }, "started_at": { - "description": "When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "description": "When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", "format": "date-time", "type": "string" } diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index c6a3cb682..001e060a9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1843,8 +1843,8 @@ class RuntimeResourceSnapshot(FabricMapping): Attributes: runtime_id: Runtime this snapshot describes. - started_at: When Fabric started the adapter host, as RFC 3339 UTC. - observed_at: When Fabric read these values, as RFC 3339 UTC. + started_at: When NeMo Fabric started the adapter host, as RFC 3339 UTC. + observed_at: When NeMo Fabric read these values, as RFC 3339 UTC. resident_memory_bytes: Sum of the live processes' resident set sizes. Pages shared between processes count once per process. process_count: Live processes in the tree, including the adapter host. From 778d8d8b50e0d9337161aaab828cb18cdace9ac3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:59:44 -0700 Subject: [PATCH 040/102] docs: regenerate the references for the snapshot product name Regenerated the Rust reference with Rust 1.94.0 and the Python reference with lazydocs; only the RuntimeResourceSnapshot page and the RuntimeResourceSnapshot attributes in the types page change. Signed-off-by: mnajafian-nv --- .../api/python-library-reference/nemo_fabric.types.md | 4 ++-- .../resource-snapshot/struct-runtimeresourcesnapshot.mdx | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 59c7e7c06..3d0201fac 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1255,8 +1255,8 @@ Memory, process count, and CPU time cover the adapter host process and its live **Attributes:** - `runtime_id`: Runtime this snapshot describes. - - `started_at`: When Fabric started the adapter host, as RFC 3339 UTC. - - `observed_at`: When Fabric read these values, as RFC 3339 UTC. + - `started_at`: When NeMo Fabric started the adapter host, as RFC 3339 UTC. + - `observed_at`: When NeMo Fabric read these values, as RFC 3339 UTC. - `resident_memory_bytes`: Sum of the live processes' resident set sizes. Pages shared between processes count once per process. - `process_count`: Live processes in the tree, including the adapter host. - `cpu_time_seconds`: CPU time used since the adapter host started, including exited descendants. It can drop when a descendant leaves the tree. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx index aff244954..9c5acd32d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx @@ -14,7 +14,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Timestamped resource usage of one runtime's local adapter host process tree. -Memory, process count, and CPU time cover the adapter host process and its live descendants. For a runtime started on a service, the shared service process is not included. Values Fabric cannot read are returned as an error, never as zero. +Memory, process count, and CPU time cover the adapter host process and its live descendants. For a runtime started on a service, the shared service process is not included. Values NeMo Fabric cannot read are returned as an error, never as zero. ## Fields @@ -24,11 +24,11 @@ Runtime this snapshot describes. ### `started_at: String` -When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. +When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. ### `observed_at: String` -When Fabric read these values, as an RFC 3339 UTC timestamp. +When NeMo Fabric read these values, as an RFC 3339 UTC timestamp. ### `resident_memory_bytes: u64` From 395efa67e5c3a22217269a0180602cdf85f19d28 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 08:16:33 -0700 Subject: [PATCH 041/102] test: say the exited-child threshold is two ticks only on Linux The comment called the 20 ms threshold two 10 ms clock ticks, but the test also runs on macOS, where proc_pid_rusage reports CPU time in nanoseconds. It now says 20 ms is two ticks on Linux. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 7e07a1c99..5029cdf1a 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -834,9 +834,9 @@ mod tests { assert_eq!(running.process_count, 2, "child never seen: {running:?}"); assert_eq!(exited.process_count, 1, "child still running: {exited:?}"); - // The host only sleeps, so two 10 ms clock ticks can come only from - // the exited child. A fast shell can finish the loop in well under - // 100 ms. + // The host only sleeps, so 20 ms of CPU can come only from the exited + // child. That is two clock ticks on Linux (10 ms each), and a fast + // shell can finish the loop in well under 100 ms. assert!( exited.cpu_time_seconds >= 0.02, "exited child's CPU (at least two clock ticks) missing: {exited:?}" From f18b0edc559b61fdf8e5a199e679d32715c1211b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 08:28:45 -0700 Subject: [PATCH 042/102] chore: regenerate the run record line schema for the snapshot wording The merged snapshot docs now say NeMo Fabric, and the run record line schema embeds those descriptions. Signed-off-by: mnajafian-nv --- schemas/sdk/run-record-line.schema.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index f14f319a7..37ea73f27 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -411,7 +411,7 @@ ] }, "RuntimeResourceSnapshot": { - "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values NeMo Fabric cannot read are returned as an\nerror, never as zero.", "properties": { "cpu_time_seconds": { "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", @@ -420,7 +420,7 @@ "type": "number" }, "observed_at": { - "description": "When Fabric read these values, as an RFC 3339 UTC timestamp.", + "description": "When NeMo Fabric read these values, as an RFC 3339 UTC timestamp.", "format": "date-time", "type": "string" }, @@ -441,7 +441,7 @@ "type": "string" }, "started_at": { - "description": "When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "description": "When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", "format": "date-time", "type": "string" } From ddff2da28d8eed519366454a27dee5a42ce92ce2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 08:50:34 -0700 Subject: [PATCH 043/102] docs: call end minus start CPU time an estimate The run record docs said end minus start CPU time is a partial count that only leaves out descendants. It can also overcount: a start snapshot can read a parent before it reaps a child, and the end snapshot then counts that child's earlier CPU through the parent's waited-children total. The Python SDK guide and the integration skill now call the difference an estimate that can overcount or undercount, and say to treat a negative difference as unknown. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 7 +++---- .../references/results-and-errors.md | 10 +++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 11aa774a0..3adafc440 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1070,10 +1070,9 @@ can be read, `resources` is `{"error": "resource_observation_unavailable"}` or `{"error": "resource_observation_unsupported"}`. CPU time is cumulative since the adapter host started. When both the start and end lines hold a snapshot, -the end line's `cpu_time_seconds` minus the start line's is a partial observed -CPU delta for the invocation: it leaves out descendants that left the process -tree before the end snapshot. Read a decrease as unknown, never as zero or a -negative CPU time. +the end line's `cpu_time_seconds` minus the start line's is an estimate of the +invocation's CPU time. Reaping and changes to the process tree can make it +overcount or undercount. Treat a negative difference as unknown. One NeMo Fabric thread writes the sample lines for every recorded invocation in the process, one record after another. Sampling does not take the host diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index a64699e3d..7409c0038 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -63,11 +63,11 @@ gives NeMo Fabric's failure cause (`fabric_failed`, `sandbox_failed`, for a timeout, and the adapter host's exit status. Only a `failed` result reads `agent_failed`; an error NeMo Fabric raises never does. Each line stores the runtime's resource snapshot as `resources`. When both the start and -end lines hold one, end minus start is a partial observed CPU delta that -leaves out descendants that left the process tree; read a decrease as -unknown, never as zero or negative. It holds no prompts, output, or error -messages. A recent sample shows that NeMo Fabric observed the record -recently, not that the agent is making progress. After three sample intervals +end lines hold one, end minus start CPU time is an estimate. Reaping and +changes to the process tree can make it overcount or undercount the +invocation's CPU time. Treat a negative difference as unknown. The file holds +no prompts, output, or error messages. A recent sample shows that NeMo Fabric +observed the record recently, not that the agent is making progress. After three sample intervals without an update, the record is stale and the invocation's outcome is unknown until an end line exists; never count a stale record as failed. One slow snapshot read delays samples for every record in the process, and a From e1dd296ce286fed769f53eb85ad0ff89137b330f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 08:52:00 -0700 Subject: [PATCH 044/102] test: drop a sample read that finishes after the exchange ends The existing in-flight test finishes the record, so the end line closes the writer and a late sample has no file to write to. It passes even without the sampling window check. Core drops the sampling registration earlier, when the exchange ends, while the file stays open for the end line. The new test blocks a sample read, drops the registration without finishing the record, releases the read, and checks that the file still holds only the start line. With the sampling_closed check in RecordFile::append_sample removed, the late sample is appended and the test fails. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 51 +++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index e693ac6bf..bf76679ff 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1223,6 +1223,57 @@ mod tests { let _ = std::fs::remove_dir_all(sentinel_folder); } + #[test] + fn a_sample_read_in_flight_when_the_exchange_ends_writes_nothing_to_the_open_record() { + use std::sync::Barrier; + use std::sync::atomic::{AtomicUsize, Ordering}; + let ticker = SampleTicker::new(Duration::from_millis(10)); + let (entered, release) = (Arc::new(Barrier::new(2)), Arc::new(Barrier::new(2))); + let calls = Arc::new(AtomicUsize::new(0)); + let read: ReadResources = { + let (entered, release) = (Arc::clone(&entered), Arc::clone(&release)); + Arc::new(move || { + // Call 0 is the start line's read; call 1 is the first sample + // read, which waits until the test has closed the window. + if calls.fetch_add(1, Ordering::SeqCst) == 1 { + entered.wait(); + release.wait(); + } + Ok(snapshot()) + }) + }; + let (folder, mut recorder) = start_recorder("window-closed", &ticker, read); + // A second record on the same ticker, sampled after the first in + // each tick, shows when the blocked tick has finished. + let (sentinel_folder, sentinel) = start_recorder( + "window-closed-sentinel", + &ticker, + Arc::new(|| Ok(snapshot())), + ); + let path = folder.join(RECORD_FILE_NAME); + let sentinel_path = sentinel_folder.join(RECORD_FILE_NAME); + + entered.wait(); + // What core does when the exchange ends: drop the registration while + // the record file stays open for the end line. + drop(recorder.take_sampling()); + let sentinel_samples = samples(&sentinel_path).len(); + release.wait(); + wait_for_samples(&sentinel_path, sentinel_samples + 1); + + let lines = read_lines(&path); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert_eq!(lines[0]["kind"], "start"); + let (_, _, invocation) = record_test_fixture(); + recorder + .finish(Ok(succeeded_result(&invocation))) + .expect("result"); + assert!(samples(&path).is_empty(), "{:?}", read_lines(&path)); + drop(sentinel); + let _ = std::fs::remove_dir_all(folder); + let _ = std::fs::remove_dir_all(sentinel_folder); + } + #[test] fn sample_line_with_a_snapshot_error_keeps_only_the_error_code() { let ticker = SampleTicker::new(Duration::from_millis(20)); From 6bb233f5e4f1cd6647cc1bd3d3e5b32c66df3730 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:03:21 -0700 Subject: [PATCH 045/102] docs: tie the run record format's stability to nemo-fabric report The record line's version doc, the schema catalog, the Python SDK guide and the integration skill said the v1alpha1 format may change until a record reader ships. nemo-fabric watch also reads records and is planned to ship first, so that wording would end the alpha period too early. They now say the format may change until the nemo-fabric report command ships. The run record line schema is regenerated with just schemas; the TypeScript contract does not change. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/lines.rs | 3 ++- docs/sdk/python.mdx | 3 ++- schemas/SCHEMA.md | 4 ++-- schemas/sdk/run-record-line.schema.json | 2 +- skills/nemo-fabric-integrate/references/results-and-errors.md | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index a7f70f5a9..f2a43e577 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -33,7 +33,8 @@ const UNRECOGNIZED_CODE: &str = "unrecognized"; pub(crate) struct RecordLine { /// Always `nemo.fabric.run_record`. pub schema: RecordSchema, - /// Line format version. `v1alpha1` may change until a record reader ships. + /// Line format version. `v1alpha1` may change until the `nemo-fabric report` + /// command ships. pub version: RecordVersion, /// Runtime id. pub runtime_id: String, diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 3adafc440..c190b4936 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1185,7 +1185,8 @@ delay. The file is created with mode `0600` on Linux and macOS. Creation refuses an existing file or symbolic link at `run-record.jsonl`. The line format is published as `schemas/sdk/run-record-line.schema.json` at -version `v1alpha1`, which may change until a reader for run records ships. +version `v1alpha1`, which may change until the `nemo-fabric report` command +ships. ## Events diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 48de38fee..153efb228 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -128,8 +128,8 @@ descriptor schema. - `sdk/error-info`: structured runtime or adapter error metadata. - `sdk/fabric-event`: NeMo Fabric lifecycle or progress event. - `sdk/run-record-line`: one line of the opt-in per-invocation run record, - `run-record.jsonl`. The schema is `v1alpha1` and may change until a reader - for run records ships. + `run-record.jsonl`. The schema is `v1alpha1` and may change until the + `nemo-fabric report` command ships. - `sdk/runtime-resource-snapshot`: timestamped memory, process count, and CPU time of a runtime's local adapter host and its descendants. diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index 37ea73f27..20da40125 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -758,7 +758,7 @@ }, "version": { "$ref": "#/$defs/RecordVersion", - "description": "Line format version. `v1alpha1` may change until a record reader ships." + "description": "Line format version. `v1alpha1` may change until the `nemo-fabric report`\ncommand ships." } }, "required": [ diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 7409c0038..ab84d20b7 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -78,7 +78,7 @@ Recording is synchronous and best-effort: a failed record write adds a `record_write_failed` event and does not replace the result, and slow storage delays the invocation. The format is the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) -and may change until a reader ships. +and may change until the `nemo-fabric report` command ships. ## Runtime Resource Snapshots From baf3566374dee184bf711e446d393ac60394a048 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:24:13 -0700 Subject: [PATCH 046/102] test: bound the sample race test handshakes The two in-flight sample tests paired the test thread and the blocked sample read with Barrier::wait, which has no timeout. A regression that stopped the ticker from reading, or that made the test wait on the blocked read, would hang the test run instead of failing it. Both tests now share one helper whose handshakes are mpsc channels: the test waits up to 10 seconds for the read to start and the read waits up to 10 seconds for release, each with a message that names the missing step. With the ticker's sample loop removed, both tests fail after the timeout with "the ticker never started the first sample read". With the sampling_closed check in RecordFile::append_sample removed, the exchange-end test still fails. With that check removed and append_end keeping the writer open, the record-end test still fails. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 86 ++++++++++++--------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index bf76679ff..102cbd45c 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -518,6 +518,8 @@ fn end_artifacts(manifest: &ArtifactManifest) -> Vec { #[cfg(test)] mod tests { + use std::sync::mpsc::{self, Receiver, Sender}; + use super::*; use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; use crate::record::lines::{RecordLine, RecordSchema, RecordVersion}; @@ -1137,6 +1139,42 @@ mod tests { ); } + /// How long a race test's handshake waits before it fails. + const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); + + /// A read that blocks its first sample read. Call 0 is the start line's + /// read. Call 1 signals the returned receiver, then waits until the test + /// sends on the returned sender. Both waits time out, so a regression + /// fails the test instead of hanging it. + fn read_blocked_at_first_sample() -> (ReadResources, Receiver<()>, Sender<()>) { + use std::sync::atomic::{AtomicUsize, Ordering}; + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel::<()>(); + let release_rx = Mutex::new(release_rx); + let calls = AtomicUsize::new(0); + let read: ReadResources = Arc::new(move || { + if calls.fetch_add(1, Ordering::SeqCst) == 1 { + entered_tx + .send(()) + .expect("the test waits for the sample read"); + release_rx + .lock() + .expect("release receiver lock") + .recv_timeout(HANDSHAKE_TIMEOUT) + .expect("the test never released the blocked sample read"); + } + Ok(snapshot()) + }); + (read, entered_rx, release_tx) + } + + /// Wait until the blocked sample read has started. + fn wait_until_entered(entered: &Receiver<()>) { + entered + .recv_timeout(HANDSHAKE_TIMEOUT) + .expect("the ticker never started the first sample read"); + } + #[test] fn two_records_share_one_ticker_and_samples_stop_at_the_end_line() { let ticker = SampleTicker::new(Duration::from_millis(20)); @@ -1181,23 +1219,9 @@ mod tests { #[test] fn a_sample_read_in_flight_when_the_record_ends_writes_nothing_after_the_end_line() { - use std::sync::Barrier; - use std::sync::atomic::{AtomicUsize, Ordering}; let ticker = SampleTicker::new(Duration::from_millis(10)); - let (entered, release) = (Arc::new(Barrier::new(2)), Arc::new(Barrier::new(2))); - let calls = Arc::new(AtomicUsize::new(0)); - let read: ReadResources = { - let (entered, release) = (Arc::clone(&entered), Arc::clone(&release)); - Arc::new(move || { - // Call 0 is the start line's read; call 1 is the first sample - // read, which waits until the test has finished the record. - if calls.fetch_add(1, Ordering::SeqCst) == 1 { - entered.wait(); - release.wait(); - } - Ok(snapshot()) - }) - }; + // The first sample read waits until the test has finished the record. + let (read, entered, release) = read_blocked_at_first_sample(); let (folder, recorder) = start_recorder("in-flight", &ticker, read); // A second record on the same ticker, sampled after the first in // each tick, shows when the blocked tick has finished. @@ -1206,13 +1230,15 @@ mod tests { let path = folder.join(RECORD_FILE_NAME); let sentinel_path = sentinel_folder.join(RECORD_FILE_NAME); - entered.wait(); + wait_until_entered(&entered); let (_, _, invocation) = record_test_fixture(); recorder .finish(Ok(succeeded_result(&invocation))) .expect("result"); let sentinel_samples = samples(&sentinel_path).len(); - release.wait(); + release + .send(()) + .expect("the blocked sample read waits for release"); wait_for_samples(&sentinel_path, sentinel_samples + 1); let lines = read_lines(&path); @@ -1225,23 +1251,9 @@ mod tests { #[test] fn a_sample_read_in_flight_when_the_exchange_ends_writes_nothing_to_the_open_record() { - use std::sync::Barrier; - use std::sync::atomic::{AtomicUsize, Ordering}; let ticker = SampleTicker::new(Duration::from_millis(10)); - let (entered, release) = (Arc::new(Barrier::new(2)), Arc::new(Barrier::new(2))); - let calls = Arc::new(AtomicUsize::new(0)); - let read: ReadResources = { - let (entered, release) = (Arc::clone(&entered), Arc::clone(&release)); - Arc::new(move || { - // Call 0 is the start line's read; call 1 is the first sample - // read, which waits until the test has closed the window. - if calls.fetch_add(1, Ordering::SeqCst) == 1 { - entered.wait(); - release.wait(); - } - Ok(snapshot()) - }) - }; + // The first sample read waits until the test has closed the window. + let (read, entered, release) = read_blocked_at_first_sample(); let (folder, mut recorder) = start_recorder("window-closed", &ticker, read); // A second record on the same ticker, sampled after the first in // each tick, shows when the blocked tick has finished. @@ -1253,12 +1265,14 @@ mod tests { let path = folder.join(RECORD_FILE_NAME); let sentinel_path = sentinel_folder.join(RECORD_FILE_NAME); - entered.wait(); + wait_until_entered(&entered); // What core does when the exchange ends: drop the registration while // the record file stays open for the end line. drop(recorder.take_sampling()); let sentinel_samples = samples(&sentinel_path).len(); - release.wait(); + release + .send(()) + .expect("the blocked sample read waits for release"); wait_for_samples(&sentinel_path, sentinel_samples + 1); let lines = read_lines(&path); From c723c68b6ee3ea6c9019ae22a657275ba29f66d8 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:39:57 -0700 Subject: [PATCH 047/102] fix: require RFC 3339 UTC resource snapshot timestamps RuntimeResourceSnapshot documents started_at and observed_at as RFC 3339 UTC, but from_mapping accepted any non-empty string, so values such as "yesterday" or a +02:00 offset passed validation. Parse both fields and reject values without the date-T-time form or with a nonzero UTC offset. The original string is kept unchanged. Signed-off-by: mnajafian-nv --- .../src/nemo_fabric/types.py | 23 +++++++++++++++++-- .../python/test_runtime_resource_snapshot.py | 18 +++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 001e060a9..ecad2667d 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -10,6 +10,8 @@ from collections.abc import Mapping from collections.abc import Sequence from copy import deepcopy +from datetime import datetime +from datetime import timedelta from pathlib import Path from types import MappingProxyType from typing import Any @@ -68,6 +70,22 @@ def _required_text(value: Any, name: str) -> str: return value +def _utc_timestamp(value: Any, name: str) -> str: + text = _required_text(value, name) + message = f"{name} must be an RFC 3339 UTC timestamp" + # fromisoformat also accepts a space separator, date-only, and basic ISO + # 8601 forms, so require the RFC 3339 "YYYY-MM-DDT" prefix first. + if text[10:11] != "T": + raise FabricConfigError(message) + try: + parsed = datetime.fromisoformat(text) + except ValueError: + raise FabricConfigError(message) from None + if parsed.utcoffset() != timedelta(0): + raise FabricConfigError(message) + return text + + def _boolean(value: Any, name: str) -> bool: if not isinstance(value, bool): raise FabricConfigError(f"{name} must be a boolean") @@ -1872,8 +1890,9 @@ class RuntimeResourceSnapshot(FabricMapping): @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: - for field in ("runtime_id", "started_at", "observed_at"): - data[field] = _required_text(data.get(field), field.replace("_", " ")) + data["runtime_id"] = _required_text(data.get("runtime_id"), "runtime id") + for field in ("started_at", "observed_at"): + data[field] = _utc_timestamp(data.get(field), field.replace("_", " ")) for field in ("resident_memory_bytes", "process_count"): value = data.get(field) if ( diff --git a/tests/python/test_runtime_resource_snapshot.py b/tests/python/test_runtime_resource_snapshot.py index d14f4354a..7afe7cae6 100644 --- a/tests/python/test_runtime_resource_snapshot.py +++ b/tests/python/test_runtime_resource_snapshot.py @@ -106,6 +106,24 @@ def test_runtime_resource_snapshot_round_trips_and_rejects_invalid_fields(): RuntimeResourceSnapshot.from_mapping({**_SNAPSHOT, field: value}) +@pytest.mark.parametrize("field", ["started_at", "observed_at"]) +def test_runtime_resource_snapshot_timestamps_accept_core_utc_format(field): + snapshot = RuntimeResourceSnapshot.from_mapping( + {**_SNAPSHOT, field: "2026-10-14T18:01:10.205Z"} + ) + + assert getattr(snapshot, field) == "2026-10-14T18:01:10.205Z" + + +@pytest.mark.parametrize("field", ["started_at", "observed_at"]) +@pytest.mark.parametrize( + "value", ["yesterday", "2026-10-14", "2026-10-14T18:01:10+02:00"] +) +def test_runtime_resource_snapshot_timestamps_reject_non_rfc3339_utc(field, value): + with pytest.raises(FabricConfigError, match="RFC 3339 UTC timestamp"): + RuntimeResourceSnapshot.from_mapping({**_SNAPSHOT, field: value}) + + def test_run_result_resource_snapshot_is_typed_or_none(): base = { "agent_name": "demo", From 6b10b68da2aadb575dec30db086283f459d714ed Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:45:40 -0700 Subject: [PATCH 048/102] docs: limit the record write failure event to returned results When the invocation raises, the run record cannot add an event to a result, and the raised error passes through unchanged. Signed-off-by: mnajafian-nv --- .../nemo-fabric-integrate/references/results-and-errors.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 1c06367a3..abe962f88 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -58,8 +58,10 @@ result lists the file as the artifact with kind `run_record`. The file has a elapsed time, error code, numeric usage, and artifact names. It holds no prompts, output, or error messages. A record without an end line has an unknown outcome; do not count it as failed. Recording is synchronous and -best-effort: a failed record write adds a `record_write_failed` event and does -not replace the result, and slow storage delays the invocation. The format is +best-effort: when the invocation returns a result, a failed record write adds +a `record_write_failed` event to it and changes nothing else; when the +invocation raises, the error stays the same error. Slow storage delays the +invocation. The format is the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) and may change until a reader ships. From 5b0033b92aa74a0877a179dca240e7c0dfc7eee5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:52:31 -0700 Subject: [PATCH 049/102] fix: require the full RFC 3339 shape for snapshot timestamps datetime.fromisoformat also accepts basic ISO 8601 forms such as 2026-10-14T180110Z, so check the RFC 3339 date-time shape before parsing. Signed-off-by: mnajafian-nv --- .../nemo-fabric-runtime/src/nemo_fabric/types.py | 12 +++++++++--- tests/python/test_runtime_resource_snapshot.py | 8 +++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index ecad2667d..3987fe121 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -6,6 +6,7 @@ from __future__ import annotations import math +import re from collections.abc import Iterator from collections.abc import Mapping from collections.abc import Sequence @@ -70,12 +71,17 @@ def _required_text(value: Any, name: str) -> str: return value +# RFC 3339 date-time shape. fromisoformat also accepts a space separator, +# date-only values and basic ISO 8601 forms, so check the shape first. +_RFC3339_DATE_TIME = re.compile( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})" +) + + def _utc_timestamp(value: Any, name: str) -> str: text = _required_text(value, name) message = f"{name} must be an RFC 3339 UTC timestamp" - # fromisoformat also accepts a space separator, date-only, and basic ISO - # 8601 forms, so require the RFC 3339 "YYYY-MM-DDT" prefix first. - if text[10:11] != "T": + if _RFC3339_DATE_TIME.fullmatch(text) is None: raise FabricConfigError(message) try: parsed = datetime.fromisoformat(text) diff --git a/tests/python/test_runtime_resource_snapshot.py b/tests/python/test_runtime_resource_snapshot.py index 7afe7cae6..c0dbeaf8b 100644 --- a/tests/python/test_runtime_resource_snapshot.py +++ b/tests/python/test_runtime_resource_snapshot.py @@ -117,7 +117,13 @@ def test_runtime_resource_snapshot_timestamps_accept_core_utc_format(field): @pytest.mark.parametrize("field", ["started_at", "observed_at"]) @pytest.mark.parametrize( - "value", ["yesterday", "2026-10-14", "2026-10-14T18:01:10+02:00"] + "value", + [ + "yesterday", + "2026-10-14", + "2026-10-14T18:01:10+02:00", + "2026-10-14T180110Z", + ], ) def test_runtime_resource_snapshot_timestamps_reject_non_rfc3339_utc(field, value): with pytest.raises(FabricConfigError, match="RFC 3339 UTC timestamp"): From ac9f02c1272b9882114741f98787737600f03d36 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:52:42 -0700 Subject: [PATCH 050/102] docs: name the result fields a record write failure leaves unchanged A returned result can still gain the run_record artifact, so list the fields that stay the same instead of saying nothing else changes. Signed-off-by: mnajafian-nv --- .../nemo-fabric-integrate/references/results-and-errors.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index abe962f88..0c2b8add4 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -59,10 +59,9 @@ elapsed time, error code, numeric usage, and artifact names. It holds no prompts, output, or error messages. A record without an end line has an unknown outcome; do not count it as failed. Recording is synchronous and best-effort: when the invocation returns a result, a failed record write adds -a `record_write_failed` event to it and changes nothing else; when the -invocation raises, the error stays the same error. Slow storage delays the -invocation. The format is -the `v1alpha1` +a `record_write_failed` event to it, and its status, output, error, and usage +stay the same; when the invocation raises, the error stays the same error. +Slow storage delays the invocation. The format is the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) and may change until a reader ships. From f83521b602a41e28a8aa2bc8ab7de1bbfcbf5a83 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:54:34 -0700 Subject: [PATCH 051/102] fix: record a schema-invalid result extension as invalid_agent_run_result The result extension check raises InvalidAdapterExtension for a value that breaks the declared schema; the end line recorded it as fabric_error, so one fault had two codes. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 34 +++++++++++++++++++-- docs/sdk/python.mdx | 3 +- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 45451bd2c..2f41f53f6 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -241,9 +241,13 @@ fn result_end_line(result: &RunResult, elapsed_ms: u64) -> EndLine { fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { let (code, stage) = match error { FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), - // After the record starts, only the check of the adapter's result - // extensions raises this, the same fault as a rejected result. - FabricError::AdapterCompatibility { .. } => (INVALID_AGENT_RUN_RESULT, ErrorStage::Invoke), + // After the record starts, the check of the adapter's result + // extensions raises these: an extension the descriptor does not + // declare, or one that breaks its declared schema. Both are the same + // fault as a rejected result. + FabricError::AdapterCompatibility { .. } | FabricError::InvalidAdapterExtension { .. } => { + (INVALID_AGENT_RUN_RESULT, ErrorStage::Invoke) + } FabricError::Write { .. } => (ARTIFACT_WRITE_FAILED, ErrorStage::Artifact), _ => (FABRIC_ERROR, ErrorStage::Invoke), }; @@ -302,6 +306,30 @@ mod tests { use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; use crate::record::test_hooks::{self, Fault}; + #[test] + fn rejected_result_extensions_read_invalid_agent_run_result() { + let undeclared = FabricError::AdapterCompatibility { + adapter_id: "demo".to_string(), + field: "result.extensions".to_string(), + reason: "not declared".to_string(), + }; + let schema_invalid = FabricError::InvalidAdapterExtension { + adapter_id: "demo".to_string(), + descriptor_source: crate::config::DescriptorSource::Bundled, + descriptor_path: std::path::PathBuf::from("adapter.json"), + extension_path: "result.extensions.demo".to_string(), + reason: "breaks the declared schema".to_string(), + }; + for error in [undeclared, schema_invalid] { + let line = error_end_line(&error, 1); + assert_eq!( + line.error.expect("end line error").code, + INVALID_AGENT_RUN_RESULT, + "{error}" + ); + } + } + fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { let config: FabricConfig = serde_json::from_value(serde_json::json!({ "schema_version": "fabric.agent/v1alpha1", diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 3fa5363a9..a7dce1079 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -987,7 +987,8 @@ The file holds a start line and an end line. Every line carries `schema` raises instead of returning a result, the end line reads `failed`. An adapter host error keeps its code, such as `host_timeout`, `host_crashed`, `host_io`, or `invalid_agent_run_result`. A result extension that the - adapter's descriptor does not declare also reads `invalid_agent_run_result`. + adapter's descriptor does not declare, or that breaks its declared schema, + also reads `invalid_agent_run_result`. A failed write of `adapter-invocation.json` or a result artifact reads `artifact_write_failed`, and any other error reads `fabric_error`. From fd56894dec8ba98754b1c03e0cac6e8225dd11fc Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 10:05:39 -0700 Subject: [PATCH 052/102] docs: mention sample lines in the RecordConfig description The record also gets sample lines while the adapter host works on the request; the description feeds the agent and run plan schemas. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 6 ++++-- .../nemo-fabric-core/config/struct-recordconfig.mdx | 2 +- schemas/sdk/agent.schema.json | 2 +- schemas/sdk/run-plan.schema.json | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 5cf456ae5..26adbd1a2 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -1074,8 +1074,10 @@ impl RuntimeConfig { /// Per-invocation run record settings. /// -/// When enabled, core appends a start line and an end line for each local -/// invocation to `/.fabric///run-record.jsonl`. +/// When enabled, core appends a start line, sample lines about every 10 +/// seconds while the adapter host works on the request, and an end line for +/// each local invocation to +/// `/.fabric///run-record.jsonl`. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RecordConfig { diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx index b72fc30ee..97ff3f08d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx @@ -13,7 +13,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Per-invocation run record settings. -When enabled, core appends a start line and an end line for each local invocation to `/.fabric///run-record.jsonl`. +When enabled, core appends a start line, sample lines about every 10 seconds while the adapter host works on the request, and an end line for each local invocation to `/.fabric///run-record.jsonl`. ## Fields diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index 6ba4ad3ee..54648362e 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -560,7 +560,7 @@ }, "RecordConfig": { "additionalProperties": false, - "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line, sample lines about every 10\nseconds while the adapter host works on the request, and an end line for\neach local invocation to\n`/.fabric///run-record.jsonl`.", "properties": { "enabled": { "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index 03499b493..504966e9a 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -1928,7 +1928,7 @@ }, "RecordConfig": { "additionalProperties": false, - "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line, sample lines about every 10\nseconds while the adapter host works on the request, and an end line for\neach local invocation to\n`/.fabric///run-record.jsonl`.", "properties": { "enabled": { "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", From bc75c3982c4d4b38fd4af86cba16765b8ad2e010 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 14:22:11 -0700 Subject: [PATCH 053/102] docs: regenerate references after merging main Regenerate the Rust library reference with Rust 1.94.0 so it lists main's environment lifecycle items next to the resource snapshot items, with updated page positions. Signed-off-by: mnajafian-nv --- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 70 ++++++++++++++++++- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../resource-snapshot/index.mdx | 2 +- .../runtime/enum-errorstage.mdx | 2 +- .../enum-openaichatcompletionchunkobject.mdx | 2 +- .../runtime/enum-openaistreamhost.mdx | 2 +- .../runtime/enum-openaistreamprofile.mdx | 2 +- .../enum-openaistreamprotocolversion.mdx | 2 +- .../runtime/enum-openaistreamrecord.mdx | 2 +- .../runtime/enum-runstatus.mdx | 2 +- .../runtime/enum-serviceownership.mdx | 2 +- .../runtime/fn-attach-service.mdx | 2 +- .../runtime/fn-invoke-openai-stream.mdx | 2 +- .../runtime/fn-invoke-runtime.mdx | 2 +- .../runtime/fn-prepare-service.mdx | 2 +- .../runtime/fn-release-service.mdx | 2 +- .../nemo-fabric-core/runtime/fn-run-plan.mdx | 2 +- .../runtime/fn-runtime-resource-snapshot.mdx | 2 +- .../runtime/fn-start-runtime-in.mdx | 2 +- .../runtime/fn-start-runtime-with-service.mdx | 2 +- .../runtime/fn-start-runtime.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 3 +- .../runtime/struct-environmenthandle.mdx | 4 +- .../runtime/struct-errorinfo.mdx | 2 +- .../runtime/struct-fabricevent.mdx | 2 +- .../runtime/struct-invocationhandle.mdx | 2 +- .../struct-openaichatcompletionchunk.mdx | 2 +- ...struct-openaichatcompletionchunkchoice.mdx | 2 +- .../struct-openaichatcompletionchunkdelta.mdx | 2 +- .../runtime/struct-openaistreaminvocation.mdx | 2 +- .../runtime/struct-openaistreamsink.mdx | 2 +- .../runtime/struct-openaistreamtransport.mdx | 2 +- .../runtime/struct-runrequest.mdx | 2 +- .../runtime/struct-runresult.mdx | 2 +- .../runtime/struct-runtimecontext.mdx | 2 +- .../runtime/struct-runtimehandle.mdx | 2 +- .../struct-runtimetelemetrycontext.mdx | 2 +- .../runtime/struct-runusage.mdx | 2 +- .../runtime/struct-servicehandle.mdx | 2 +- .../runtime/struct-servicereference.mdx | 2 +- .../runtime/struct-telemetryref.mdx | 2 +- .../schema/enum-schemaname.mdx | 2 +- .../nemo-fabric-core/schema/index.mdx | 2 +- 50 files changed, 120 insertions(+), 51 deletions(-) diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 6c276ae6b..71771291a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 127 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index fa45b1f21..a6a947127 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 128 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 218c1dde6..630b1a01b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 129 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index 7a918b620..936364ce9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 130 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index f0f77cf94..916915ab1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 131 +position: 135 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 75900423a..fe01749e7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -497,6 +497,42 @@ Environment provider. Adapter kind. +### `UnsupportedEnvironmentRelease` + +
+ +An environment provider does not implement release. + +#### Fields + +##### `provider: String` + +Environment provider. + +### `EnvironmentProviderOperation` + +
+ +An environment provider process could not complete an operation. + +#### Fields + +##### `provider: String` + +Environment provider. + +##### `operation: String` + +Lifecycle operation. + +##### `code: String` + +Stable provider or host error code. + +##### `message: String` + +Human-readable message that excludes credentials and raw diagnostics. + ### `EnvironmentHandleRequired`
@@ -533,6 +569,38 @@ Actual value from the environment handle. Environment handle id. +### `ResourceObservationUnavailable` + +
+ +A runtime's resources cannot be read now: the runtime stopped, is stopping, its adapter host process exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read. + +#### Fields + +##### `runtime_id: String` + +Runtime handle id. + +##### `reason: String` + +Why the runtime cannot be observed. + +### `ResourceObservationUnsupported` + +
+ +The runtime's adapter kind, environment provider, or operating system has no resource reader. + +#### Fields + +##### `runtime_id: String` + +Runtime handle id. + +##### `reason: String` + +What has no resource reader. + ### `InvalidProcessSettings`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index 0bf1ac774..ecc3b0451 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 134 +position: 136 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 6b00f4d18..da30f61c4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 136 +position: 140 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index affbe0d0f..a0348db94 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -3,7 +3,7 @@ title: "Module resource_snapshot" sidebar-title: "resource_snapshot" slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" description: "Resource snapshots of a runtime's local adapter host and its descendants." -position: 133 +position: 137 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx index eb802320b..cdaccd776 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx @@ -2,7 +2,7 @@ title: "Enum Error Stage" sidebar-title: "ErrorStage" description: "NeMo Fabric lifecycle stage associated with an error." -position: 23 +position: 24 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx index cca22a793..83505d3b2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Chat Completion Chunk Object" sidebar-title: "OpenAiChatCompletionChunkObject" description: "Exact OpenAI object discriminator accepted by the v1 chunk profile." -position: 24 +position: 25 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx index f2d130b85..5e337a0e0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Host" sidebar-title: "OpenAiStreamHost" description: "Supported native-streaming listener host." -position: 25 +position: 26 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx index 1c3f839ae..eb3f8fe51 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Profile" sidebar-title: "OpenAiStreamProfile" description: "Supported OpenAI-compatible chunk profile." -position: 26 +position: 27 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx index 86fddd94c..ee69f77c3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Protocol Version" sidebar-title: "OpenAiStreamProtocolVersion" description: "Supported southbound native-streaming protocol version." -position: 27 +position: 28 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx index a3a933cdf..d02adc8bf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Record" sidebar-title: "OpenAiStreamRecord" description: "One correlated NDJSON record on the adapter-native stream channel." -position: 28 +position: 29 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx index 97954ce13..f7acd827c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx @@ -2,7 +2,7 @@ title: "Enum RunStatus" sidebar-title: "RunStatus" description: "Runtime completion status." -position: 29 +position: 30 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx index c5050a301..310cf8983 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx @@ -2,7 +2,7 @@ title: "Enum Service Ownership" sidebar-title: "ServiceOwnership" description: "Ownership of a service exposed through NeMo Fabric." -position: 30 +position: 31 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx index d8ff25825..494089eb6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx @@ -2,7 +2,7 @@ title: "Function attach_service" sidebar-title: "attach_service" description: "Attach to a caller-owned long-lived service for a resolved run plan." -position: 31 +position: 33 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx index b81268897..1fa28e020 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx @@ -2,7 +2,7 @@ title: "Function invoke_openai_stream" sidebar-title: "invoke_openai_stream" description: "Invoke a started harness runtime and pass through native OpenAI chat-completion chunks." -position: 32 +position: 34 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx index e1471de73..28b005c5d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx @@ -2,7 +2,7 @@ title: "Function invoke_runtime" sidebar-title: "invoke_runtime" description: "Invoke a started harness runtime." -position: 33 +position: 35 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx index a59f0067c..49e08a5bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx @@ -2,7 +2,7 @@ title: "Function prepare_service" sidebar-title: "prepare_service" description: "Prepare a Fabric-owned long-lived service for a resolved run plan." -position: 35 +position: 37 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx index b356d2db0..72ca1b83e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx @@ -2,7 +2,7 @@ title: "Function release_service" sidebar-title: "release_service" description: "Release a Fabric-owned service or detach from a caller-owned service." -position: 36 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 55a5a47cb..331c08b3c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -2,7 +2,7 @@ title: "Function run_plan" sidebar-title: "run_plan" description: "Invoke a NeMo Fabric run plan." -position: 37 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx index 4275ce2f6..87bb90002 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx @@ -2,7 +2,7 @@ title: "Function runtime_resource_snapshot" sidebar-title: "runtime_resource_snapshot" description: "Read a timestamped resource snapshot of a started runtime." -position: 38 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-in.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-in.mdx index d31455b12..ccec560e4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-in.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-in.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_in" sidebar-title: "start_runtime_in" description: "Start a runtime in an explicitly prepared environment." -position: 42 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx index c61b166c3..96b3675dc 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_with_service" sidebar-title: "start_runtime_with_service" description: "Start a runtime connected to an active service." -position: 40 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx index 5b10ad19a..0d7562938 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime" sidebar-title: "start_runtime" description: "Start or connect to a harness runtime in its configured local environment." -position: 41 +position: 42 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index a55e5bd91..77db99a02 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 41 +position: 45 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 5fa3f62d9..acfe954c9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 135 +position: 138 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -59,6 +59,7 @@ Runtime invocation helpers. - [release_environment](fn-release-environment.mdx): Release or detach an environment through its provider. - [release_service](fn-release-service.mdx): Release a Fabric-owned service or detach from a caller-owned service. - [run_plan](fn-run-plan.mdx): Invoke a NeMo Fabric run plan. +- [runtime_resource_snapshot](fn-runtime-resource-snapshot.mdx): Read a timestamped resource snapshot of a started runtime. - [start_runtime](fn-start-runtime.mdx): Start or connect to a harness runtime in its configured local environment. - [start_runtime_in](fn-start-runtime-in.mdx): Start a runtime in an explicitly prepared environment. - [start_runtime_with_service](fn-start-runtime-with-service.mdx): Start a runtime connected to an active service. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx index 55d47378d..67102fa82 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx @@ -29,11 +29,11 @@ Where NeMo Fabric control code runs. ### `workspace: Option` -Workspace visible to the harness runtime. +Workspace visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `artifacts: Option` -Artifact root visible to the harness runtime. +Artifact root visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `env: BTreeMap` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx index 9468739a0..376449e6a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx @@ -2,7 +2,7 @@ title: "Struct Error Info" sidebar-title: "ErrorInfo" description: "Normalized error metadata." -position: 5 +position: 6 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx index 5330e632a..5ec70a8ad 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx @@ -2,7 +2,7 @@ title: "Struct Fabric Event" sidebar-title: "FabricEvent" description: "NeMo Fabric lifecycle or progress event." -position: 6 +position: 7 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx index 3f3ac0d42..1df9629e6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx @@ -2,7 +2,7 @@ title: "Struct Invocation Handle" sidebar-title: "InvocationHandle" description: "One request sent to a runtime." -position: 7 +position: 8 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx index 026b2de72..96e965402 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk" sidebar-title: "OpenAiChatCompletionChunk" description: "OpenAI Chat Completions chunk accepted by the native streaming profile." -position: 8 +position: 9 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx index 94b2e70ee..f3dbf5424 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Choice" sidebar-title: "OpenAiChatCompletionChunkChoice" description: "One choice within an OpenAI Chat Completions streaming chunk." -position: 9 +position: 10 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx index 845afed45..53b0a6972 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Delta" sidebar-title: "OpenAiChatCompletionChunkDelta" description: "Incremental assistant message fields carried by one OpenAI choice." -position: 10 +position: 11 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx index 49b4c9c49..b5a3b7fcb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Invocation" sidebar-title: "OpenAiStreamInvocation" description: "One adapter-native OpenAI streaming invocation." -position: 11 +position: 12 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx index 8ad3827d5..a178c8113 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Sink" sidebar-title: "OpenAiStreamSink" description: "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric." -position: 12 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx index 2ed1e5c62..4a7f2e2f9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Transport" sidebar-title: "OpenAiStreamTransport" description: "SDK-owned loopback transport for one native OpenAI streaming invocation." -position: 13 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index 8dcc6950f..d0200756d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -2,7 +2,7 @@ title: "Struct RunRequest" sidebar-title: "RunRequest" description: "A request passed to a NeMo Fabric-managed harness runtime." -position: 14 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 3e05964a5..acde0de61 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -2,7 +2,7 @@ title: "Struct RunResult" sidebar-title: "RunResult" description: "Result from a NeMo Fabric-managed harness invocation." -position: 15 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 28af565b2..625527702 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Context" sidebar-title: "RuntimeContext" description: "Context generated for one invocation of a started runtime." -position: 17 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx index b0392902d..0671b565e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Handle" sidebar-title: "RuntimeHandle" description: "Active or resumable harness runtime." -position: 18 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx index 1525ed89d..890abc95b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Telemetry Context" sidebar-title: "RuntimeTelemetryContext" description: "Runtime telemetry config passed to adapters." -position: 19 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx index 6f16e387b..864be9b27 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx @@ -2,7 +2,7 @@ title: "Struct RunUsage" sidebar-title: "RunUsage" description: "Normalized invocation usage exposed to NeMo Fabric consumers." -position: 16 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx index 8c36ff5de..dead906b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Service Handle" sidebar-title: "ServiceHandle" description: "Public identity and binding for one prepared or attached service." -position: 20 +position: 21 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx index e28440353..4a249074a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx @@ -2,7 +2,7 @@ title: "Struct Service Reference" sidebar-title: "ServiceReference" description: "Caller-supplied reference to an already-running service." -position: 21 +position: 22 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx index 2d2bc6240..15d28ef49 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Ref" sidebar-title: "TelemetryRef" description: "Reference to telemetry emitted by Relay or another configured telemetry path." -position: 22 +position: 23 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index e45e2090f..38c553dc6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -187,7 +187,7 @@ Runtime resource snapshot schema. #### `ALL` -
22]"}} />
+
23]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 45c3ca38e..ccd80095b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 136 +position: 139 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} From c233cf43dfac52833dfe7e225cc6879e365062b5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 14:25:39 -0700 Subject: [PATCH 054/102] test: reject the run record on the OpenShell provider Main now ships OpenShell as a real environment provider whose task paths pass config validation. Check that an OpenShell environment with valid task paths still fails planning when the run record is on, next to the existing docker case. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 111ede01d..49abb07b2 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -7479,18 +7479,25 @@ mod tests { #[test] fn record_enabled_with_a_non_local_provider_fails_validation() { - let mut config = typed_config("nvidia.fabric.hermes"); - config.runtime.artifacts = Some(PathBuf::from("runs")); - config.runtime.record = Some(RecordConfig { enabled: true }); - config.environment.as_mut().expect("environment").provider = "docker".to_string(); + for provider in ["docker", "openshell"] { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + let environment = config.environment.as_mut().expect("environment"); + environment.provider = provider.to_string(); + // Valid OpenShell task paths, so only the record check can fail. + environment.workspace = Some(PathBuf::from("/sandbox/workspace")); + environment.artifacts = Some(PathBuf::from("/sandbox/artifacts")); - assert_eq!( - record_error(&config), - Some(( - "runtime.record.enabled".to_string(), - "supported only with the local environment provider".to_string() - )) - ); + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "supported only with the local environment provider".to_string() + )), + "{provider}" + ); + } } #[test] From caa72f5fdb408d5534882aa259b1fdd67b5f6d04 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 14:26:37 -0700 Subject: [PATCH 055/102] docs: regenerate references after merging main Regenerate the Rust library reference with Rust 1.94.0 so it lists main's environment lifecycle items next to the run record items, with updated page positions. Signed-off-by: mnajafian-nv --- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 38 ++++++++++++++++++- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../runtime/enum-errorstage.mdx | 2 +- .../enum-openaichatcompletionchunkobject.mdx | 2 +- .../runtime/enum-openaistreamhost.mdx | 2 +- .../runtime/enum-openaistreamprofile.mdx | 2 +- .../enum-openaistreamprotocolversion.mdx | 2 +- .../runtime/enum-openaistreamrecord.mdx | 2 +- .../runtime/enum-runstatus.mdx | 2 +- .../runtime/enum-serviceownership.mdx | 2 +- .../runtime/fn-attach-service.mdx | 2 +- .../runtime/fn-invoke-openai-stream.mdx | 2 +- .../runtime/fn-invoke-runtime.mdx | 2 +- .../runtime/fn-prepare-service.mdx | 2 +- .../runtime/fn-release-service.mdx | 2 +- .../nemo-fabric-core/runtime/fn-run-plan.mdx | 2 +- .../runtime/fn-start-runtime-with-service.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 2 +- .../runtime/struct-environmenthandle.mdx | 4 +- .../runtime/struct-errorinfo.mdx | 2 +- .../runtime/struct-fabricevent.mdx | 2 +- .../runtime/struct-invocationhandle.mdx | 2 +- .../struct-openaichatcompletionchunk.mdx | 2 +- ...struct-openaichatcompletionchunkchoice.mdx | 2 +- .../struct-openaichatcompletionchunkdelta.mdx | 2 +- .../runtime/struct-openaistreaminvocation.mdx | 2 +- .../runtime/struct-openaistreamsink.mdx | 2 +- .../runtime/struct-openaistreamtransport.mdx | 2 +- .../runtime/struct-runrequest.mdx | 2 +- .../runtime/struct-runresult.mdx | 2 +- .../runtime/struct-runtimecontext.mdx | 2 +- .../runtime/struct-runtimehandle.mdx | 2 +- .../struct-runtimetelemetrycontext.mdx | 2 +- .../runtime/struct-runusage.mdx | 2 +- .../runtime/struct-servicehandle.mdx | 2 +- .../runtime/struct-servicereference.mdx | 2 +- .../runtime/struct-telemetryref.mdx | 2 +- .../schema/enum-schemaname.mdx | 2 +- .../nemo-fabric-core/schema/index.mdx | 2 +- 46 files changed, 83 insertions(+), 47 deletions(-) diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index feabb8df6..b0aba3459 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 126 +position: 130 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 4c4e21b44..e62476ee1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 127 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index de4682271..74aa25a6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 128 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index 7f45f6f6e..359d8fe9e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 129 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index 4f6f26b00..548c11e07 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 130 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index d504f49ef..6462f4179 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -497,6 +497,42 @@ Environment provider. Adapter kind. +### `UnsupportedEnvironmentRelease` + +
+ +An environment provider does not implement release. + +#### Fields + +##### `provider: String` + +Environment provider. + +### `EnvironmentProviderOperation` + +
+ +An environment provider process could not complete an operation. + +#### Fields + +##### `provider: String` + +Environment provider. + +##### `operation: String` + +Lifecycle operation. + +##### `code: String` + +Stable provider or host error code. + +##### `message: String` + +Human-readable message that excludes credentials and raw diagnostics. + ### `EnvironmentHandleRequired`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index 0bf1ac774..239052d7c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 134 +position: 135 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index f9c667fa9..3a81be390 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 134 +position: 138 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx index eb802320b..cdaccd776 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx @@ -2,7 +2,7 @@ title: "Enum Error Stage" sidebar-title: "ErrorStage" description: "NeMo Fabric lifecycle stage associated with an error." -position: 23 +position: 24 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx index cca22a793..83505d3b2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Chat Completion Chunk Object" sidebar-title: "OpenAiChatCompletionChunkObject" description: "Exact OpenAI object discriminator accepted by the v1 chunk profile." -position: 24 +position: 25 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx index f2d130b85..5e337a0e0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Host" sidebar-title: "OpenAiStreamHost" description: "Supported native-streaming listener host." -position: 25 +position: 26 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx index 1c3f839ae..eb3f8fe51 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Profile" sidebar-title: "OpenAiStreamProfile" description: "Supported OpenAI-compatible chunk profile." -position: 26 +position: 27 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx index 86fddd94c..ee69f77c3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Protocol Version" sidebar-title: "OpenAiStreamProtocolVersion" description: "Supported southbound native-streaming protocol version." -position: 27 +position: 28 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx index a3a933cdf..d02adc8bf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Record" sidebar-title: "OpenAiStreamRecord" description: "One correlated NDJSON record on the adapter-native stream channel." -position: 28 +position: 29 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx index 97954ce13..f7acd827c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx @@ -2,7 +2,7 @@ title: "Enum RunStatus" sidebar-title: "RunStatus" description: "Runtime completion status." -position: 29 +position: 30 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx index c5050a301..310cf8983 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx @@ -2,7 +2,7 @@ title: "Enum Service Ownership" sidebar-title: "ServiceOwnership" description: "Ownership of a service exposed through NeMo Fabric." -position: 30 +position: 31 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx index d8ff25825..494089eb6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx @@ -2,7 +2,7 @@ title: "Function attach_service" sidebar-title: "attach_service" description: "Attach to a caller-owned long-lived service for a resolved run plan." -position: 31 +position: 33 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx index b81268897..1fa28e020 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx @@ -2,7 +2,7 @@ title: "Function invoke_openai_stream" sidebar-title: "invoke_openai_stream" description: "Invoke a started harness runtime and pass through native OpenAI chat-completion chunks." -position: 32 +position: 34 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx index e1471de73..28b005c5d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx @@ -2,7 +2,7 @@ title: "Function invoke_runtime" sidebar-title: "invoke_runtime" description: "Invoke a started harness runtime." -position: 33 +position: 35 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx index a59f0067c..49e08a5bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx @@ -2,7 +2,7 @@ title: "Function prepare_service" sidebar-title: "prepare_service" description: "Prepare a Fabric-owned long-lived service for a resolved run plan." -position: 35 +position: 37 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx index b356d2db0..72ca1b83e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx @@ -2,7 +2,7 @@ title: "Function release_service" sidebar-title: "release_service" description: "Release a Fabric-owned service or detach from a caller-owned service." -position: 36 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 55a5a47cb..331c08b3c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -2,7 +2,7 @@ title: "Function run_plan" sidebar-title: "run_plan" description: "Invoke a NeMo Fabric run plan." -position: 37 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx index 547b47760..5acac20bd 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_with_service" sidebar-title: "start_runtime_with_service" description: "Start a runtime connected to an active service." -position: 39 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index 71febb037..ce8835391 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 40 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 5fa3f62d9..2cc36bc2f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 135 +position: 136 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx index 55d47378d..67102fa82 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx @@ -29,11 +29,11 @@ Where NeMo Fabric control code runs. ### `workspace: Option` -Workspace visible to the harness runtime. +Workspace visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `artifacts: Option` -Artifact root visible to the harness runtime. +Artifact root visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `env: BTreeMap` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx index 9468739a0..376449e6a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx @@ -2,7 +2,7 @@ title: "Struct Error Info" sidebar-title: "ErrorInfo" description: "Normalized error metadata." -position: 5 +position: 6 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx index 5330e632a..5ec70a8ad 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx @@ -2,7 +2,7 @@ title: "Struct Fabric Event" sidebar-title: "FabricEvent" description: "NeMo Fabric lifecycle or progress event." -position: 6 +position: 7 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx index 3f3ac0d42..1df9629e6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx @@ -2,7 +2,7 @@ title: "Struct Invocation Handle" sidebar-title: "InvocationHandle" description: "One request sent to a runtime." -position: 7 +position: 8 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx index 026b2de72..96e965402 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk" sidebar-title: "OpenAiChatCompletionChunk" description: "OpenAI Chat Completions chunk accepted by the native streaming profile." -position: 8 +position: 9 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx index 94b2e70ee..f3dbf5424 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Choice" sidebar-title: "OpenAiChatCompletionChunkChoice" description: "One choice within an OpenAI Chat Completions streaming chunk." -position: 9 +position: 10 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx index 845afed45..53b0a6972 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Delta" sidebar-title: "OpenAiChatCompletionChunkDelta" description: "Incremental assistant message fields carried by one OpenAI choice." -position: 10 +position: 11 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx index 49b4c9c49..b5a3b7fcb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Invocation" sidebar-title: "OpenAiStreamInvocation" description: "One adapter-native OpenAI streaming invocation." -position: 11 +position: 12 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx index 8ad3827d5..a178c8113 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Sink" sidebar-title: "OpenAiStreamSink" description: "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric." -position: 12 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx index 2ed1e5c62..4a7f2e2f9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Transport" sidebar-title: "OpenAiStreamTransport" description: "SDK-owned loopback transport for one native OpenAI streaming invocation." -position: 13 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index 8dcc6950f..d0200756d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -2,7 +2,7 @@ title: "Struct RunRequest" sidebar-title: "RunRequest" description: "A request passed to a NeMo Fabric-managed harness runtime." -position: 14 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 1cb5bd052..f74b804b8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -2,7 +2,7 @@ title: "Struct RunResult" sidebar-title: "RunResult" description: "Result from a NeMo Fabric-managed harness invocation." -position: 15 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 28af565b2..625527702 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Context" sidebar-title: "RuntimeContext" description: "Context generated for one invocation of a started runtime." -position: 17 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx index b0392902d..0671b565e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Handle" sidebar-title: "RuntimeHandle" description: "Active or resumable harness runtime." -position: 18 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx index 1525ed89d..890abc95b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Telemetry Context" sidebar-title: "RuntimeTelemetryContext" description: "Runtime telemetry config passed to adapters." -position: 19 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx index 6f16e387b..864be9b27 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx @@ -2,7 +2,7 @@ title: "Struct RunUsage" sidebar-title: "RunUsage" description: "Normalized invocation usage exposed to NeMo Fabric consumers." -position: 16 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx index 8c36ff5de..dead906b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Service Handle" sidebar-title: "ServiceHandle" description: "Public identity and binding for one prepared or attached service." -position: 20 +position: 21 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx index e28440353..4a249074a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx @@ -2,7 +2,7 @@ title: "Struct Service Reference" sidebar-title: "ServiceReference" description: "Caller-supplied reference to an already-running service." -position: 21 +position: 22 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx index 2d2bc6240..15d28ef49 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Ref" sidebar-title: "TelemetryRef" description: "Reference to telemetry emitted by Relay or another configured telemetry path." -position: 22 +position: 23 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index a0442791d..282ef841d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -187,7 +187,7 @@ One line of an opt-in per-invocation run record, version `v1alpha1`. #### `ALL` -
22]"}} />
+
23]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 45c3ca38e..52a9671f8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 136 +position: 137 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} From 874731d52141eeed91f9723a971e999e8431f722 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 14:36:00 -0700 Subject: [PATCH 056/102] docs: regenerate references after merging main Regenerate the Rust library reference with Rust 1.94.0 after the merges: the config index lists RecordConfig, SchemaName::ALL shows 24 entries, and page positions follow main's environment lifecycle items. Signed-off-by: mnajafian-nv --- .../nemo-fabric-core/adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../nemo-fabric-core/agent-execution/index.mdx | 2 +- .../rust-library-reference/nemo-fabric-core/config/index.mdx | 3 ++- .../rust-library-reference/nemo-fabric-core/doctor/index.mdx | 2 +- .../nemo-fabric-core/error/enum-fabricerror.mdx | 2 +- .../rust-library-reference/nemo-fabric-core/error/index.mdx | 2 +- .../api/rust-library-reference/nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/resource-snapshot/index.mdx | 2 +- .../rust-library-reference/nemo-fabric-core/runtime/index.mdx | 2 +- .../nemo-fabric-core/schema/enum-schemaname.mdx | 2 +- .../rust-library-reference/nemo-fabric-core/schema/index.mdx | 2 +- 12 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 71771291a..3ba18310c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 131 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index a6a947127..a7c60bd2d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 132 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 630b1a01b..782999cf5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 133 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index 936364ce9..5159ab6bd 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 134 +position: 135 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -53,6 +53,7 @@ NeMo Fabric config models and loading helpers. - [McpServerPlan](struct-mcpserverplan.mdx): Resolved MCP server exposure. - [MetadataConfig](struct-metadataconfig.mdx): Human-readable metadata. - [ModelConfig](struct-modelconfig.mdx): Model configuration. +- [RecordConfig](struct-recordconfig.mdx): Per-invocation run record settings. - [RelayAtifConfig](struct-relayatifconfig.mdx): Relay ATIF export configuration. - [RelayAtofConfig](struct-relayatofconfig.mdx): Relay ATOF export configuration. - [RelayComponentConfig](struct-relaycomponentconfig.mdx): Generic NeMo Relay plugin component configuration. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index 916915ab1..84cffbff2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 135 +position: 136 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index fe01749e7..f1e2fe312 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -2,7 +2,7 @@ title: "Enum Fabric Error" sidebar-title: "FabricError" description: "Errors raised by NeMo Fabric config loading and validation." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index ecc3b0451..cf7a42463 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 136 +position: 137 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index da30f61c4..c450d80cb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 140 +position: 141 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index a0348db94..48167fea9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -3,7 +3,7 @@ title: "Module resource_snapshot" sidebar-title: "resource_snapshot" slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" description: "Resource snapshots of a runtime's local adapter host and its descendants." -position: 137 +position: 138 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index acfe954c9..2b8e9d342 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 138 +position: 139 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index d4789a3c6..4c3932563 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -194,7 +194,7 @@ Runtime resource snapshot schema. #### `ALL` -
23]"}} />
+
24]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index ccd80095b..de3b12928 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 139 +position: 140 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} From 1619b1d379c6e3057ab66d9c5fd003d4c73366a3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sat, 17 Oct 2026 21:49:08 -0700 Subject: [PATCH 057/102] test: build race-test fixtures before the sampling window opens The two in-flight sample tests resolved the run plan, created folders and built the result while the blocked sample read waited 10 s for its release. On a slow file system that work outran the handshake and the tests failed with a timeout. Both tests now prepare the fixture, folders and result before the first record starts, and keep the same race and assertions. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 59 ++++++++++++++------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 48e60beee..01a946912 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1128,13 +1128,23 @@ mod tests { ticker: &Arc, read: ReadResources, ) -> (PathBuf, InvocationRecorder) { - let (plan, runtime, invocation) = record_test_fixture(); - let folder = temp_folder(name); + start_recorder_in(temp_folder(name), &record_test_fixture(), ticker, read) + } + + /// Start a record in `folder` from a fixture the test already built, so + /// a race test does no plan resolution or folder setup while a blocked + /// sample read waits for it. + fn start_recorder_in( + folder: PathBuf, + (plan, runtime, invocation): &(RunPlan, RuntimeHandle, InvocationHandle), + ticker: &Arc, + read: ReadResources, + ) -> (PathBuf, InvocationRecorder) { let request = RunRequest::text("input"); let recorder = InvocationRecorder::start(RecordStart { - plan: &plan, - runtime: &runtime, - invocation: &invocation, + plan, + runtime, + invocation, request: &request, fabric_home: &folder, timeout: Duration::from_secs(5), @@ -1254,21 +1264,28 @@ mod tests { #[test] fn a_sample_read_in_flight_when_the_record_ends_writes_nothing_after_the_end_line() { let ticker = SampleTicker::new(Duration::from_millis(10)); + // Build everything before the first record starts: once it does, the + // blocked sample read waits only HANDSHAKE_TIMEOUT for the release. + let fixture = record_test_fixture(); + let result = succeeded_result(&fixture.2); + let folder = temp_folder("in-flight"); + let sentinel_folder = temp_folder("in-flight-sentinel"); // The first sample read waits until the test has finished the record. let (read, entered, release) = read_blocked_at_first_sample(); - let (folder, recorder) = start_recorder("in-flight", &ticker, read); + let (folder, recorder) = start_recorder_in(folder, &fixture, &ticker, read); // A second record on the same ticker, sampled after the first in // each tick, shows when the blocked tick has finished. - let (sentinel_folder, sentinel) = - start_recorder("in-flight-sentinel", &ticker, Arc::new(|| Ok(snapshot()))); + let (sentinel_folder, sentinel) = start_recorder_in( + sentinel_folder, + &fixture, + &ticker, + Arc::new(|| Ok(snapshot())), + ); let path = folder.join(RECORD_FILE_NAME); let sentinel_path = sentinel_folder.join(RECORD_FILE_NAME); wait_until_entered(&entered); - let (_, _, invocation) = record_test_fixture(); - recorder - .finish(Ok(succeeded_result(&invocation))) - .expect("result"); + recorder.finish(Ok(result)).expect("result"); let sentinel_samples = samples(&sentinel_path).len(); release .send(()) @@ -1286,13 +1303,20 @@ mod tests { #[test] fn a_sample_read_in_flight_when_the_exchange_ends_writes_nothing_to_the_open_record() { let ticker = SampleTicker::new(Duration::from_millis(10)); + // Build everything before the first record starts: once it does, the + // blocked sample read waits only HANDSHAKE_TIMEOUT for the release. + let fixture = record_test_fixture(); + let result = succeeded_result(&fixture.2); + let folder = temp_folder("window-closed"); + let sentinel_folder = temp_folder("window-closed-sentinel"); // The first sample read waits until the test has closed the window. let (read, entered, release) = read_blocked_at_first_sample(); - let (folder, mut recorder) = start_recorder("window-closed", &ticker, read); + let (folder, mut recorder) = start_recorder_in(folder, &fixture, &ticker, read); // A second record on the same ticker, sampled after the first in // each tick, shows when the blocked tick has finished. - let (sentinel_folder, sentinel) = start_recorder( - "window-closed-sentinel", + let (sentinel_folder, sentinel) = start_recorder_in( + sentinel_folder, + &fixture, &ticker, Arc::new(|| Ok(snapshot())), ); @@ -1312,10 +1336,7 @@ mod tests { let lines = read_lines(&path); assert_eq!(lines.len(), 1, "{lines:?}"); assert_eq!(lines[0]["kind"], "start"); - let (_, _, invocation) = record_test_fixture(); - recorder - .finish(Ok(succeeded_result(&invocation))) - .expect("result"); + recorder.finish(Ok(result)).expect("result"); assert!(samples(&path).is_empty(), "{:?}", read_lines(&path)); drop(sentinel); let _ = std::fs::remove_dir_all(folder); From 9d6bc218bf4debd918ea49ad44eb8e7789fba6e4 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 15:28:16 -0700 Subject: [PATCH 058/102] fix: count every Deep Agents model call in token usage The Deep Agents adapter summed usage_metadata from the AI messages that reached graph state through the updates stream. Three cases came out wrong: - Deep Agents adds SummarizationMiddleware to every agent and subagent. Its summary call returns only text, so the summary model's message and usage never enter graph state, and an invocation that summarized under-reported input, output and total tokens. - When the invocation raised, including GraphRecursionError, the adapter returned no messages, so usage was None even though model calls had finished. - A count was summed over the calls that reported it and returned as known. ChatOpenAI also turns an omitted Chat Completions count into 0 and derives a missing total, so a partial provider report came out as measured zeros. The adapter now records usage from LangChain's chat-model callbacks, which fire for every chat-model call that runs in the adapter process: the main agent, declarative subagents and summarization. Agent Protocol subagents run remotely and are not included. A new recorder is created for each invocation, so earlier turns are never counted again. It outlives an exception, so a failed invocation still reports the fields that every counted call reported. Each token count and the cost are known only when every counted call in the invocation reported them, and an explicit zero stays a known zero. A request the provider rejects with a 4xx status (other than 408) contributes no usage. A 5xx response, timeout, cancellation or unfinished call leaves the invocation's counts and cost unknown, because the provider may have done work without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the recorder, and only the final response's reported usage counts. When a response carries OpenAI-shaped raw usage, the raw counts decide what was reported: an omitted count stays unknown, the provider's total is used when present, and a derived total counts only when input and output were both reported. The graph tests run the real Deep Agents graph and ChatOpenAI against a scripted Chat Completions endpoint for summarization, parallel task subagents, 400, 408, 503 and timeout failures, the recursion limit, partial reports and explicit zeros. Separate ChatOpenAI.with_retry tests cover 429 then success and 503 then success. The existing usage tests now produce their messages through a LangChain chat-model call so the callbacks fire. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 28 +- .../deepagents/adapter.py | 254 +++++---- tests/adapters/test_deepagents.py | 491 +++++++++++++++--- tests/e2e/test_deepagents.py | 1 - tests/integrations/test_relay_scope_leak.py | 4 +- 5 files changed, 618 insertions(+), 160 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 2090da4cd..78e94ce51 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -227,10 +227,30 @@ service. The normalized result includes the final response, buffered messages and per-step events, LangGraph thread id, token usage (and cost when the provider -reports it), and errors. Usage aggregates the current turn across the main agent -and any delegated subagents (streamed with `subgraphs=True`). Configuration and -preflight failures (a missing credential, an absent `deepagents` package, or an -invalid MCP server) fail runtime start before an invocation is accepted. +reports it), and errors. Configuration and preflight failures (a missing +credential, an absent `deepagents` package, or an invalid MCP server) fail +runtime start before an invocation is accepted. + +Usage sums every chat-model call that runs in the adapter process for the +invocation: the main agent, declarative subagents, and Deep Agents +summarization. Agent Protocol subagents (entries with `graph_id`) run in their +remote service, and their tokens are not included. A token count or cost is +reported only when every counted call reported it; otherwise it stays unknown. +This also holds for a failed invocation. A request the provider rejects with a +4xx status (other than 408) contributes no usage. A 5xx response, timeout, +cancellation or unfinished call leaves the invocation's counts and cost +unknown, because the provider may have done work without reporting usage. +OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries +to success is not visible to the adapter, and only the final response's +reported usage counts. Usage replayed from a configured LangChain model cache +would be counted as reported; Fabric configures no such cache. + +When a response carries OpenAI-shaped usage in +`response_metadata.token_usage`, as `ChatOpenAI` responses without streaming +do, the raw counts decide what was reported: an omitted count stays unknown, +and the provider's total is used when present. Other providers and streamed +responses use LangChain's `usage_metadata`, which some integrations fill with 0 +for an omitted count. ## Runtime Lifecycle diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 618957f1f..37771f419 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -21,9 +21,12 @@ from pathlib import Path from typing import Any from typing import NamedTuple +from uuid import UUID from langchain.agents.middleware import AgentMiddleware +from langchain_core.callbacks import BaseCallbackHandler from langchain_core.messages import ToolMessage +from langchain_core.outputs import LLMResult from langgraph.errors import GraphRecursionError from nemo_fabric_adapter_contract.models import AgentConfig from nemo_fabric_adapter_contract.models import AgentMcpServerConfig @@ -750,7 +753,7 @@ async def invoke( resumed=resumed, result_state=outcome.result_state, events=outcome.events or [], - turn_messages=outcome.turn_messages or [], + model_calls=outcome.model_calls or [], error=outcome.error, telemetry_runtime=telemetry_runtime, relay_artifacts=relay_artifacts, @@ -874,24 +877,26 @@ async def _invoke_agent( ) -> TurnOutcome: """Run one agent turn, normalizing an invocation failure into an error string.""" + model_usage = ModelCallUsage() try: - result_state, events, turn_messages = await invoke_compiled_agent( + result_state, events = await invoke_compiled_agent( self._agent, user_message, self._thread_id, - callbacks=callbacks, + callbacks=[*(callbacks or []), model_usage], ) except GraphRecursionError: return TurnOutcome( + model_calls=model_usage.calls(), error="Deep Agents reached its graph recursion limit.", error_code="deepagents_recursion_limit_reached", ) except Exception as exc: # normalized adapter failure - return TurnOutcome(error=_error_text(exc)) + return TurnOutcome(model_calls=model_usage.calls(), error=_error_text(exc)) return TurnOutcome( result_state=result_state, events=events, - turn_messages=turn_messages, + model_calls=model_usage.calls(), ) def _telemetry_output( @@ -966,18 +971,65 @@ def _apply_callbacks( return config +class ModelCallUsage(BaseCallbackHandler): + """Record what each chat-model call of one invocation reports. + + LangChain fires these callbacks for every chat-model call in the run, + including calls whose response never enters graph state, such as the Deep + Agents summarization call. The records live on the handler, so calls that + finished before an invocation raised are still reported. + + A call is recorded as unreported when it starts, and its usage replaces that + record when it ends. A request the provider rejects with a 4xx status (other + than 408) contributes no usage and is dropped. A 5xx response, timeout, + cancellation or unfinished call stays unreported and makes the invocation's + counts and cost unknown, because the provider may have done work without + reporting usage. + """ + + run_inline = True + + def __init__(self) -> None: + super().__init__() + self._calls: dict[UUID, dict[str, Any]] = {} + + def on_chat_model_start( + self, serialized: Any, messages: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self._calls[run_id] = {} + + def on_llm_start( + self, serialized: Any, prompts: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self._calls[run_id] = {} + + def on_llm_end(self, response: LLMResult, *, run_id: UUID, **kwargs: Any) -> None: + generations = response.generations[0] if response.generations else [] + message = getattr(generations[0], "message", None) if generations else None + self._calls[run_id] = {} if message is None else _message_to_dict(message) + + def on_llm_error( + self, error: BaseException, *, run_id: UUID, **kwargs: Any + ) -> None: + # Provider SDKs (OpenAI, Anthropic) raise errors carrying the HTTP status + # code when the provider answered with an error response. A 4xx other + # than 408 (request timeout) means the provider rejected the request. + status_code = getattr(error, "status_code", None) + if isinstance(status_code, int) and 400 <= status_code < 500: + if status_code != 408: + self._calls.pop(run_id, None) + + def calls(self) -> list[dict[str, Any]]: + return list(self._calls.values()) + + async def invoke_compiled_agent( agent: Any, user_message: str, thread_id: str | None, callbacks: list[Any] | None = None, -) -> tuple[Any, list[dict[str, Any]], list[dict[str, Any]]]: - """Run a compiled agent and return state, events, and current-turn messages. - - On a later runtime invocation the final ``values`` state also replays - prior-turn messages, so usage/cost must be aggregated from the messages - emitted *this* turn — the ``updates`` deltas — rather than the full final - state. +) -> tuple[Any, list[dict[str, Any]]]: + """Run a compiled agent and return its final state and per-step events. ``callbacks`` (e.g. the NeMo Relay callback handler) are appended to the LangGraph run config; any callbacks already present are preserved rather than @@ -992,12 +1044,10 @@ async def invoke_compiled_agent( config = config or None events: list[dict[str, Any]] = [] - turn_messages: list[Any] = [] final: Any = None # subgraphs=True surfaces subagent (delegated ``task``) steps as 3-tuples whose - # namespace identifies the subgraph. Folding their message deltas into this turn - # keeps usage/cost accurate for delegating runs; the final ``values`` state is - # taken from the main graph only (empty namespace). + # namespace identifies the subgraph, so they appear in ``events``; the final + # ``values`` state is taken from the main graph only (empty namespace). stream = ( agent.astream(inputs, config, stream_mode=["updates", "values"], subgraphs=True) if config @@ -1009,16 +1059,9 @@ async def invoke_compiled_agent( if namespace: event["subgraph"] = "/".join(str(part) for part in namespace) events.append(event) - for node_output in chunk.values(): - if isinstance(node_output, dict): - new = node_output.get("messages") - if isinstance(new, list): - turn_messages.extend(new) - elif new is not None: - turn_messages.append(new) elif mode == "values" and not namespace: final = chunk - return final, events, _dedup_messages(_messages_to_dicts(turn_messages)) + return final, events class Observability(NamedTuple): @@ -1034,7 +1077,7 @@ class TurnOutcome(NamedTuple): result_state: Any = None events: list[dict[str, Any]] | None = None - turn_messages: list[dict[str, Any]] | None = None + model_calls: list[dict[str, Any]] | None = None error: str | None = None error_code: str | None = None telemetry_error: str | None = None @@ -1151,7 +1194,7 @@ def normalize_output( resumed: bool, result_state: Any, events: list[dict[str, Any]], - turn_messages: list[dict[str, Any]], + model_calls: list[dict[str, Any]], error: str | None, telemetry_runtime: dict[str, Any] | None, relay_artifacts: list[dict[str, str]] | None, @@ -1160,9 +1203,9 @@ def normalize_output( ) -> dict[str, Any]: messages = _extract_messages(result_state) response = _final_response(messages) - # Aggregate usage/cost from this turn's messages only; the replayed final state - # replays prior-turn messages that must not be re-counted. - usage = _aggregate_usage(turn_messages) + # Usage comes from this invocation's model calls, not from the final state, + # which also replays earlier turns and omits internal calls like summarization. + usage = _aggregate_usage(model_calls) output: dict[str, Any] = { "harness": HARNESS, @@ -1210,26 +1253,6 @@ def _messages_to_dicts(raw: Any) -> list[dict[str, Any]]: return [item if isinstance(item, dict) else _message_to_dict(item) for item in raw] -def _dedup_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Drop duplicate messages by id, preserving first-seen order. - - With ``subgraphs=True`` the same message can surface in both a subgraph's - updates and the parent stream; de-duplicating by id keeps usage aggregation - from counting a message twice. Messages without an id are always kept. - """ - - seen: set[str] = set() - unique: list[dict[str, Any]] = [] - for message in messages: - message_id = message.get("id") - if isinstance(message_id, str): - if message_id in seen: - continue - seen.add(message_id) - unique.append(message) - return unique - - def _message_to_dict(message: Any) -> dict[str, Any]: role = ( getattr(message, "type", None) or getattr(message, "role", None) or "assistant" @@ -1259,56 +1282,103 @@ def _final_response(messages: list[dict[str, Any]]) -> Any: return messages[-1].get("content") if messages else None -def _aggregate_usage(messages: list[dict[str, Any]]) -> dict[str, Any] | None: +def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: + """Sum model-call usage; a count is known only when every call reported it. + + One call without a count makes the invocation's sum for that count a + partial value, so it stays unknown rather than under-reported. + """ + + if not calls: + return None totals: dict[str, Any] = {} - cost = 0.0 - has_cost = False - for message in messages: - usage = message.get("usage") if isinstance(message.get("usage"), dict) else {} - for source, target in ( - ("input_tokens", "prompt_tokens"), - ("output_tokens", "completion_tokens"), - ("total_tokens", "total_tokens"), - ): - value = usage.get(source) - if ( - isinstance(value, int) - and not isinstance(value, bool) - and 0 <= value <= (1 << 64) - 1 - ): - totals[target] = int(totals.get(target, 0)) + value - # Cost is not part of LangChain's UsageMetadata; surface it only when a - # model/provider reports it on the usage or response metadata. - candidate = ( - usage.get("total_cost") or usage.get("cost") or _metadata_cost(message) - ) - if ( - isinstance(candidate, (int, float)) - and not isinstance(candidate, bool) - and math.isfinite(candidate) - and candidate >= 0 - ): - cost += float(candidate) - has_cost = True - if has_cost: - totals["cost"] = cost + for source, target in ( + ("input_tokens", "prompt_tokens"), + ("output_tokens", "completion_tokens"), + ("total_tokens", "total_tokens"), + ): + values = [_reported_tokens(call, source) for call in calls] + if all(value is not None for value in values): + totals[target] = sum(values) + costs = [_reported_cost(call) for call in calls] + if all(cost is not None for cost in costs): + totals["cost"] = math.fsum(costs) return totals or None -def _metadata_cost(message: dict[str, Any]) -> float | None: - metadata = message.get("response_metadata") - if isinstance(metadata, dict): - value = metadata.get("cost") - if ( - isinstance(value, (int, float)) - and not isinstance(value, bool) - and math.isfinite(value) - and value >= 0 - ): - return float(value) +# Raw OpenAI Chat Completions usage keys behind each LangChain usage counter. +_OPENAI_USAGE_KEYS = { + "input_tokens": "prompt_tokens", + "output_tokens": "completion_tokens", + "total_tokens": "total_tokens", +} + + +def _reported_tokens(call: dict[str, Any], name: str) -> int | None: + """Return one call's token count, or ``None`` when the provider omitted it. + + ``ChatOpenAI`` turns an omitted Chat Completions count into 0 and derives a + missing or zero total from input plus output. When a response carries + OpenAI-shaped raw usage in ``response_metadata.token_usage``, the raw counts + are used: an omitted count is unknown, and a derived total counts only when + both of its parts were reported. Other responses use ``usage_metadata``. + """ + + usage = call.get("usage") + value = usage.get(name) if isinstance(usage, dict) else None + if not _is_token_count(value): + return None + metadata = call.get("response_metadata") + raw = metadata.get("token_usage") if isinstance(metadata, dict) else None + if not isinstance(raw, dict): + return value + reported = raw.get(_OPENAI_USAGE_KEYS[name]) + if _is_token_count(reported): + return reported + if name == "total_tokens" and reported is None: + parts = (raw.get("prompt_tokens"), raw.get("completion_tokens")) + if all(_is_token_count(part) for part in parts): + return value return None +def _is_token_count(value: Any) -> bool: + return ( + isinstance(value, int) + and not isinstance(value, bool) + and 0 <= value <= (1 << 64) - 1 + ) + + +def _reported_cost(call: dict[str, Any]) -> float | None: + """Return a provider-reported cost for one call, or ``None`` when absent. + + Cost is not part of LangChain's UsageMetadata; some providers report it on the + usage or response metadata. + """ + + usage = call.get("usage") if isinstance(call.get("usage"), dict) else {} + metadata = call.get("response_metadata") + candidates = ( + usage.get("total_cost"), + usage.get("cost"), + metadata.get("cost") if isinstance(metadata, dict) else None, + ) + for value in candidates: + if value is not None: + return float(value) if _is_cost(value) else None + return None + + +def _is_cost(value: Any) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and value >= 0 + ) + + def _supported_kwargs(callable_obj: Any, kwargs: dict[str, Any]) -> dict[str, Any]: try: signature = inspect.signature(callable_obj) diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 23304be4d..c916cb2a3 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -25,6 +25,9 @@ from unittest.mock import MagicMock import pytest +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.language_models.fake_chat_models import GenericFakeChatModel +from langchain_core.messages import AIMessage from langchain_openai import ChatOpenAI as InstalledChatOpenAI from langgraph.errors import GraphRecursionError from nemo_fabric_adapter_contract.codec import ContractValidationError @@ -34,6 +37,7 @@ from nemo_fabric_adapter_contract.models import AgentRunRequest from nemo_fabric_adapter_contract.models import AgentRunResult from nemo_fabric_adapter_contract.models import AgentRunStatus +from nemo_fabric_adapter_contract.models import AgentUsage from nemo_fabric_adapter_contract.models import McpOAuth2Config from nemo_fabric_adapter_contract.models import McpServiceAccountConfig from nemo_fabric_adapter_contract.models import RuntimeContext @@ -49,6 +53,23 @@ def _activation(*diagnostics: dict[str, object]) -> types.SimpleNamespace: ROOT = Path(__file__).resolve().parents[2] +async def call_model(config: dict[str, Any] | None, message: AIMessage) -> AIMessage: + """Return ``message`` from a real LangChain chat-model call under ``config``. + + The call fires the run's LangChain callbacks the way a Deep Agents model call + does, which is where the adapter reads usage. Test stand-ins for the Relay + handler are not LangChain handlers, so they are left out. + """ + + callbacks = [ + callback + for callback in (config or {}).get("callbacks") or [] + if isinstance(callback, BaseCallbackHandler) + ] + model = GenericFakeChatModel(messages=iter([message])) + return await model.ainvoke("prompt", config={"callbacks": callbacks}) + + def test_descriptor_declares_supported_normalized_config(): descriptor = json.loads( (ROOT / "adapters/python/deepagents/deepagents.fabric-adapter.json").read_text( @@ -192,11 +213,17 @@ async def stream( recorder["subgraphs"] = subgraphs recorder["checkpointer"] = kwargs.get("checkpointer") user = inputs["messages"][-1]["content"] - ai = { - "role": "ai", - "content": f"reply to {user}", - "usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12}, - } + ai = await call_model( + config, + AIMessage( + content=f"reply to {user}", + usage_metadata={ + "input_tokens": 5, + "output_tokens": 7, + "total_tokens": 12, + }, + ), + ) # subgraphs=True yields 3-tuples ``(namespace, mode, chunk)``; the main # graph has an empty namespace. ``updates`` carries the message produced # this turn; ``values`` is the full replayed state. @@ -1445,7 +1472,7 @@ def test_native_telemetry_requires_mapping(tmp_path, make_payload): ) -async def test_relay_disabled_adds_no_scope_or_callbacks( +async def test_relay_disabled_adds_no_scope_or_relay_callbacks( tmp_path, make_payload, fake_sdks ): # With telemetry disabled the invocation runs without a Relay scope, callback @@ -1454,7 +1481,9 @@ async def test_relay_disabled_adds_no_scope_or_callbacks( assert output["completed"] is True assert "telemetry" not in output - assert (fake_sdks["config"] or {}).get("callbacks") is None + # The only run callback is the adapter's own model-usage recorder. + callbacks = fake_sdks["config"]["callbacks"] + assert [type(callback) for callback in callbacks] == [adapter.ModelCallUsage] assert "relay-mw" not in (fake_sdks["create_kwargs"].get("middleware") or []) @@ -2070,14 +2099,19 @@ async def test_cost_is_extracted_from_response_metadata( ): import deepagents - message = { - "role": "ai", - "content": "done", - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, - "response_metadata": {"cost": 0.0025}, - } - async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): + message = await call_model( + config, + AIMessage( + content="done", + usage_metadata={ + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + }, + response_metadata={"cost": 0.0025}, + ), + ) yield ((), "updates", {"agent": {"messages": [message]}}) yield ((), "values", {"messages": [message]}) @@ -2108,6 +2142,23 @@ def test_aggregate_usage_discards_tokens_larger_than_uint64(): assert adapter._aggregate_usage([{"usage": {"input_tokens": 1 << 64}}]) is None +@pytest.mark.parametrize( + ("costs", "expected"), + [ + pytest.param([0.0, 0.0], 0.0, id="explicit-zero"), + pytest.param([0.5, 0.25], 0.75, id="every-call"), + pytest.param([0.5, None], None, id="one-call-without-cost"), + ], +) +def test_aggregate_usage_reports_cost_only_when_every_call_did(costs, expected): + calls = [ + {"usage": {"input_tokens": 1}, "response_metadata": {"cost": cost}} + for cost in costs + ] + + assert adapter._aggregate_usage(calls).get("cost") == expected + + async def test_replayed_state_usage_counts_current_turn_only( tmp_path, make_payload, monkeypatch ): @@ -2115,20 +2166,25 @@ async def test_replayed_state_usage_counts_current_turn_only( # cost must reflect only the message emitted this turn, not the replayed one. import deepagents - prior = { - "role": "ai", - "content": "prior", - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, - "response_metadata": {"cost": 0.001}, - } - current = { - "role": "ai", - "content": "now", - "usage": {"input_tokens": 2, "output_tokens": 2, "total_tokens": 4}, - "response_metadata": {"cost": 0.002}, - } + prior = AIMessage( + content="prior", + usage_metadata={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + response_metadata={"cost": 0.001}, + ) async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): + current = await call_model( + config, + AIMessage( + content="now", + usage_metadata={ + "input_tokens": 2, + "output_tokens": 2, + "total_tokens": 4, + }, + response_metadata={"cost": 0.002}, + ), + ) # Only the current turn's message is emitted as an update... yield ((), "updates", {"agent": {"messages": [current]}}) # ...but the final state also replays the prior turn. @@ -2427,47 +2483,360 @@ def test_deepagents_passthrough_rejects_fabric_owned_option(): adapter._validated_deepagents_settings({"model": "untrusted-model"}) -async def test_subagent_usage_folded_from_subgraph(tmp_path, make_payload, monkeypatch): - # Usage from a delegated subagent is emitted under a subgraph namespace; folding - # it into this turn keeps usage/cost accurate. Duplicate ids are counted once. - import deepagents - - sub_ai = { - "role": "ai", - "content": "subagent work", - "id": "sub-1", - "usage": {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, - } - main_ai = { - "role": "ai", - "content": "final", - "id": "main-1", - "usage": {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5}, +def chat_completion( + usage: dict[str, Any] | None, + *, + tool_calls: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Return an OpenAI Chat Completions response body with the given usage.""" + + message: dict[str, Any] = {"role": "assistant", "content": "done"} + if tool_calls: + message["content"] = None + message["tool_calls"] = tool_calls + body: dict[str, Any] = { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 0, + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": "tool_calls" if tool_calls else "stop", + } + ], } + if usage is not None: + body["usage"] = usage + return body - async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): - assert subgraphs is True - # subagent step under a subgraph namespace, emitted twice (dedup by id) - yield (("task:researcher",), "updates", {"agent": {"messages": [sub_ai]}}) - yield (("task:researcher",), "updates", {"agent": {"messages": [sub_ai]}}) - # main graph step + replayed final state - yield ((), "updates", {"agent": {"messages": [main_ai]}}) - yield ((), "values", {"messages": [main_ai]}) - agent = MagicMock() - agent.astream = astream - monkeypatch.setattr(deepagents, "create_deep_agent", MagicMock(return_value=agent)) +LS_TOOL_CALL = { + "id": "call-1", + "type": "function", + "function": {"name": "ls", "arguments": '{"path": "/"}'}, +} - output = await invoke_once(make_payload(tmp_path)) - # subagent (10/20/30) + main (2/3/5), the duplicate subagent message counted once - assert output["usage"] == { - "prompt_tokens": 12, - "completion_tokens": 23, - "total_tokens": 35, +@pytest.fixture(name="openai_endpoint") +def openai_endpoint_fixture(fake_sdks, monkeypatch) -> dict[str, Any]: + """Run the real Deep Agents graph and ChatOpenAI against a scripted endpoint. + + Tests append response bodies, ``httpx.Response`` objects, or exceptions to + raise to ``replies``; each model request takes the next one, and its JSON + body is kept in ``requests``. ``profile`` sets the chat model profile, which Deep Agents uses + to choose its summarization trigger. + """ + + import httpx + + for name in ( + "deepagents", + "deepagents.backends", + "deepagents.middleware", + "deepagents.middleware.subagents", + "langchain.chat_models", + "langgraph", + "langgraph.checkpoint", + "langgraph.checkpoint.sqlite", + "langgraph.checkpoint.sqlite.aio", + ): + monkeypatch.delitem(sys.modules, name, raising=False) + endpoint: dict[str, Any] = {"replies": [], "requests": [], "profile": None} + + def respond(request: httpx.Request) -> httpx.Response: + endpoint["requests"].append(json.loads(request.content)) + reply = endpoint["replies"].pop(0) + if isinstance(reply, Exception): + raise reply + if isinstance(reply, httpx.Response): + return reply + return httpx.Response(200, json=reply) + + def build_chat_openai(**kwargs: Any) -> InstalledChatOpenAI: + return InstalledChatOpenAI( + **kwargs, + max_retries=0, + profile=endpoint["profile"], + http_async_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), + ) + + fake_sdks["chat_openai"].side_effect = build_chat_openai + return endpoint + + +async def run_invocations(payload: dict[str, Any], count: int) -> list[AgentRunResult]: + runtime = adapter.DeepAgentsRuntime() + await runtime.start(lifecycle_start_payload(payload)) + try: + return [ + await runtime.invoke(*lifecycle_invocation(payload)) for _ in range(count) + ] + finally: + await runtime.stop() + + +async def test_summarization_model_call_usage_is_counted( + tmp_path, make_payload, openai_endpoint +): + # Deep Agents summarizes older history with a separate model call whose response + # never enters graph state. Its usage belongs to the invocation that made it. + openai_endpoint["profile"] = {"max_input_tokens": 1000} + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 1500, "completion_tokens": 10, "total_tokens": 1510} + ), + # Second invocation: the summary call, then the agent's model call. + chat_completion( + {"prompt_tokens": 1400, "completion_tokens": 60, "total_tokens": 1460} + ), + chat_completion( + {"prompt_tokens": 300, "completion_tokens": 5, "total_tokens": 305} + ), + ] + payload = make_payload(tmp_path) + payload["request"]["input"] = "word " * 6000 + + first, second = await run_invocations(payload, 2) + + summary_request, agent_request = openai_endpoint["requests"][1:] + assert "tools" not in summary_request + assert "tools" in agent_request + assert first.usage == AgentUsage( + input_tokens=1500, output_tokens=10, total_tokens=1510 + ) + assert second.status is AgentRunStatus.SUCCEEDED + assert second.usage == AgentUsage( + input_tokens=1700, output_tokens=65, total_tokens=1765 + ) + + +async def test_subagent_model_calls_are_counted( + tmp_path, make_payload, openai_endpoint +): + # Two parallel ``task`` calls run the general-purpose subagent twice; each + # subagent's model call belongs to this invocation. + task_calls = [ + { + "id": f"task-{index}", + "type": "function", + "function": { + "name": "task", + "arguments": json.dumps( + { + "description": f"Echo the word {index}.", + "subagent_type": "general-purpose", + } + ), + }, + } + for index in (1, 2) + ] + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + tool_calls=task_calls, + ), + chat_completion( + {"prompt_tokens": 10, "completion_tokens": 10, "total_tokens": 20} + ), + chat_completion( + {"prompt_tokens": 100, "completion_tokens": 100, "total_tokens": 200} + ), + chat_completion( + {"prompt_tokens": 1000, "completion_tokens": 1000, "total_tokens": 2000} + ), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert len(openai_endpoint["requests"]) == 4 + subgraphs = { + event["subgraph"] for event in result.output["events"] if "subgraph" in event } - # the subgraph step is recorded with its namespace label - assert any(evt.get("subgraph") == "task:researcher" for evt in output["events"]) + assert len(subgraphs) == 2 + assert result.usage == AgentUsage( + input_tokens=1111, output_tokens=1111, total_tokens=2222 + ) + + +async def test_recursion_limit_failure_keeps_usage_of_completed_model_calls( + tmp_path, make_payload, openai_endpoint +): + tool_call_reply = chat_completion( + {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, + tool_calls=[LS_TOOL_CALL], + ) + openai_endpoint["replies"] += [tool_call_reply] * 20 + payload = make_payload(tmp_path) + payload["config"]["runtime"] = {"max_turns": 6} + + (result,) = await run_invocations(payload, 1) + + calls = len(openai_endpoint["requests"]) + assert calls > 1 + assert result.status is AgentRunStatus.FAILED + assert result.error.code == "deepagents_recursion_limit_reached" + assert result.usage == AgentUsage( + input_tokens=10 * calls, output_tokens=calls, total_tokens=11 * calls + ) + + +def provider_failure(kind: str) -> Any: + """Return a scripted endpoint reply that fails a model call.""" + + import httpx + + if kind == "timeout": + return httpx.ReadTimeout("provider did not answer in time") + return httpx.Response(int(kind), json={"error": {"message": "request failed"}}) + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + # A 4xx rejection other than 408 spends nothing, so the finished call's + # usage stays known. + pytest.param( + "400", + AgentUsage(input_tokens=120, output_tokens=8, total_tokens=128), + id="400-rejection", + ), + # The provider may have done work without reporting usage. + pytest.param("408", None, id="408-request-timeout"), + pytest.param("503", None, id="503-server-error"), + pytest.param("timeout", None, id="read-timeout"), + ], +) +async def test_failed_model_call_usage_follows_provider_response( + tmp_path, make_payload, openai_endpoint, failure, expected +): + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 120, "completion_tokens": 8, "total_tokens": 128}, + tool_calls=[LS_TOOL_CALL], + ), + provider_failure(failure), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage == expected + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + pytest.param( + "429", + {"prompt_tokens": 30, "completion_tokens": 2, "total_tokens": 32}, + id="429-then-success", + ), + pytest.param("503", None, id="503-then-success"), + ], +) +async def test_retried_model_call_usage_follows_failed_attempt( + fake_sdks, openai_endpoint, failure, expected +): + # LangChain's with_retry runs each attempt as its own chat-model call, as the + # Deep Agents summarization model does. + openai_endpoint["replies"] += [ + provider_failure(failure), + chat_completion( + {"prompt_tokens": 30, "completion_tokens": 2, "total_tokens": 32} + ), + ] + model = fake_sdks["chat_openai"]( + model="nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + api_key="test123", + base_url="https://integrate.api.nvidia.com/v1", + ) + model_usage = adapter.ModelCallUsage() + + await model.with_retry(stop_after_attempt=2, wait_exponential_jitter=False).ainvoke( + "summarize", config={"callbacks": [model_usage]} + ) + + assert adapter._aggregate_usage(model_usage.calls()) == expected + + +async def test_recorder_keeps_a_cancelled_call_unreported(): + # Recorder unit test: a cancelled call ends without on_llm_end and, when + # LangChain gathers it, without on_llm_error; its start record keeps the + # invocation's counts unknown. + model_usage = adapter.ModelCallUsage() + await call_model( + {"callbacks": [model_usage]}, + AIMessage( + content="done", + usage_metadata={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ), + ) + model_usage.on_chat_model_start({}, [], run_id=uuid.uuid4()) + + assert adapter._aggregate_usage(model_usage.calls()) is None + + +@pytest.mark.parametrize( + ("usages", "expected"), + [ + pytest.param( + [{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}], + AgentUsage(input_tokens=0, output_tokens=0, total_tokens=0), + id="explicit-zero", + ), + pytest.param( + [{"prompt_tokens": 100}], + AgentUsage(input_tokens=100), + id="input-only", + ), + pytest.param( + [{"completion_tokens": 5}], + AgentUsage(output_tokens=5), + id="output-only", + ), + pytest.param( + [{"prompt_tokens": 100, "completion_tokens": 5}], + AgentUsage(input_tokens=100, output_tokens=5, total_tokens=105), + id="total-omitted", + ), + pytest.param( + [{"completion_tokens": 5, "total_tokens": 105}], + AgentUsage(output_tokens=5, total_tokens=105), + id="provider-total-with-input-omitted", + ), + pytest.param( + [{"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 0}], + AgentUsage(input_tokens=7, output_tokens=5, total_tokens=0), + id="provider-total-zero-kept", + ), + pytest.param( + [ + {"prompt_tokens": 100, "completion_tokens": 5, "total_tokens": 105}, + None, + ], + None, + id="one-call-without-usage", + ), + ], +) +async def test_partial_usage_reports_keep_missing_counts_unknown( + tmp_path, make_payload, openai_endpoint, usages, expected +): + # ChatOpenAI turns an omitted count into 0; a count is known only when every + # model call in the invocation reported it. + openai_endpoint["replies"] += [ + *(chat_completion(usage, tool_calls=[LS_TOOL_CALL]) for usage in usages[:-1]), + chat_completion(usages[-1]), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage == expected async def test_bad_mcp_transport_fails_runtime_start(tmp_path, make_payload): diff --git a/tests/e2e/test_deepagents.py b/tests/e2e/test_deepagents.py index 30b07e9ef..08155598c 100644 --- a/tests/e2e/test_deepagents.py +++ b/tests/e2e/test_deepagents.py @@ -486,7 +486,6 @@ async def test_deepagents_builtin_subagent_delegation(): call["args"]["subagent_type"] == "general-purpose" for call in task_calls ), result.to_mapping() assert any("subgraph" in event for event in output["events"]), result.to_mapping() - # delegated steps are folded into this turn's usage aggregation assert output["resumed"] is False, result.to_mapping() diff --git a/tests/integrations/test_relay_scope_leak.py b/tests/integrations/test_relay_scope_leak.py index 9dabfdcd5..5d5dbc58a 100644 --- a/tests/integrations/test_relay_scope_leak.py +++ b/tests/integrations/test_relay_scope_leak.py @@ -168,7 +168,7 @@ async def test_uuid_request_id_seeds_real_relay_parent( monkeypatch, make_runtime_context ): async def fake_invoke(agent, user_message, thread_id, callbacks=None): - return {"messages": []}, [], [] + return {"messages": []}, [] monkeypatch.setattr(adapter, "invoke_compiled_agent", fake_invoke) @@ -210,7 +210,7 @@ async def test_overlapping_turns_stay_telemetry_clean( async def fake_invoke(agent, user_message, thread_id, callbacks=None): if callbacks: await _overlapping_chain_runs(callbacks[0]) - return {"messages": []}, [], [] + return {"messages": []}, [] monkeypatch.setattr(adapter, "invoke_compiled_agent", fake_invoke) recording_scope = _RecordingScope() From cdc0636a647d562a02a72110f9921b8943d3bd04 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 17:00:16 -0700 Subject: [PATCH 059/102] docs: name NeMo Fabric in the Deep Agents usage limits Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 78e94ce51..66a44eece 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -243,7 +243,7 @@ unknown, because the provider may have done work without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the adapter, and only the final response's reported usage counts. Usage replayed from a configured LangChain model cache -would be counted as reported; Fabric configures no such cache. +would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming From 566003925848ac3b00393af70e26bdf49155b613 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 17:22:03 -0700 Subject: [PATCH 060/102] docs: clarify when run records get sample lines Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 069f9b30d..faf9a193d 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1088,11 +1088,11 @@ result lists it as the `record` artifact with kind `run_record`: record = next(a for a in result.artifacts.artifacts if a.kind == "run_record") ``` -The file holds a start line, a sample line about every 10 seconds until the -adapter host answers or the timeout fires when the start line has -`sample_interval_ms`, and an end line. Every line carries `schema` -(`nemo.fabric.run_record`), `version` (`v1alpha1`), `kind`, `runtime_id`, -`invocation_id`, `request_id`, and `time`. +The file holds a start line, sample lines, and an end line. Sample lines are +written only when the start line has `sample_interval_ms`; then one is written +about every 10 seconds until the adapter host answers or the timeout fires. +Every line carries `schema` (`nemo.fabric.run_record`), `version` (`v1alpha1`), +`kind`, `runtime_id`, `invocation_id`, `request_id`, and `time`. - The `start` line is written before the request is sent. It holds the agent name, harness, adapter ID, each configured model's provider and model name, From 6980d82bcf8f100fe227277ab4057194436a4baa Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 07:53:51 -0700 Subject: [PATCH 061/102] test: check warm-runtime invocation CPU is end minus start A warm runtime's snapshot CPU time is cumulative since the adapter host started, and readers estimate each invocation's CPU time as the end line's value minus the start line's. Nothing checked that the start line reads the snapshot when its own invocation starts, so a record whose start value fell back to the host's start would report the running total as the second invocation's CPU time. The test runs two records against one fake cumulative CPU counter and asserts that end minus start is 2.0 s and 0.5 s, not 3.0 s and 3.5 s. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 38 +++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 01a946912..b5f68eb7d 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1407,4 +1407,42 @@ mod tests { ); let _ = std::fs::remove_dir_all(folder); } + + #[test] + fn warm_runtime_end_minus_start_cpu_is_each_invocation_own_cpu_time() { + use std::sync::atomic::{AtomicU64, Ordering}; + // The snapshot's CPU time is cumulative since the host started, so a + // warm runtime's second invocation starts with the first one's CPU + // time already counted. Whole milliseconds keep the subtraction exact. + let host_cpu_ms = Arc::new(AtomicU64::new(1_000)); + let read: ReadResources = { + let host_cpu_ms = Arc::clone(&host_cpu_ms); + Arc::new(move || { + let cpu_ms = host_cpu_ms.load(Ordering::SeqCst); + Ok(RuntimeResourceSnapshot { + cpu_time_seconds: Duration::from_millis(cpu_ms).as_secs_f64(), + ..snapshot() + }) + }) + }; + let ticker = idle_ticker(); + let (_, _, invocation) = record_test_fixture(); + let cpu = |line: &Value| line["resources"]["cpu_time_seconds"].as_f64().expect("cpu"); + + let mut own_cpu = Vec::new(); + for (name, burn_ms) in [("warm-cpu-1", 2_000), ("warm-cpu-2", 500)] { + let (folder, mut recorder) = start_recorder(name, &ticker, Arc::clone(&read)); + host_cpu_ms.fetch_add(burn_ms, Ordering::SeqCst); + recorder.exchange_ended(recorder.read_exchange_end(Instant::now(), &read())); + recorder + .finish(Ok(succeeded_result(&invocation))) + .expect("result"); + let lines = read_lines(&folder.join(RECORD_FILE_NAME)); + let _ = std::fs::remove_dir_all(folder); + own_cpu.push(cpu(lines.last().expect("end line")) - cpu(&lines[0])); + } + + // Not 3.0 and 3.5, the host's totals at each end. + assert_eq!(own_cpu, [2.0, 0.5]); + } } From a0902a1706f5b2593870a3bed6e890d63d485801 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:08:49 -0700 Subject: [PATCH 062/102] test: check the end line keeps the exchange-end CPU time The warm-runtime CPU test left the fake counter unchanged between the exchange end and finish, so an end line that read its snapshot at finish would still have passed. The host can keep using CPU after the exchange ends, and that time belongs to neither invocation's end line. The test now adds 0.25 s after each exchange end. It asserts that the end lines keep the exchange-end totals of 3.0 s and 3.75 s, and that end minus start stays 2.0 s and 0.5 s although the second start line includes the first invocation's extra 0.25 s. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index b5f68eb7d..fc19855a7 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1409,11 +1409,13 @@ mod tests { } #[test] - fn warm_runtime_end_minus_start_cpu_is_each_invocation_own_cpu_time() { + fn warm_runtime_end_minus_start_cpu_is_each_exchange_own_cpu_time() { use std::sync::atomic::{AtomicU64, Ordering}; // The snapshot's CPU time is cumulative since the host started, so a // warm runtime's second invocation starts with the first one's CPU - // time already counted. Whole milliseconds keep the subtraction exact. + // time already counted. CPU time the host uses after the exchange + // ends and before the result is finished belongs to neither end + // line. Whole and quarter seconds keep the subtraction exact. let host_cpu_ms = Arc::new(AtomicU64::new(1_000)); let read: ReadResources = { let host_cpu_ms = Arc::clone(&host_cpu_ms); @@ -1429,20 +1431,29 @@ mod tests { let (_, _, invocation) = record_test_fixture(); let cpu = |line: &Value| line["resources"]["cpu_time_seconds"].as_f64().expect("cpu"); - let mut own_cpu = Vec::new(); + let (mut end_cpu, mut own_cpu) = (Vec::new(), Vec::new()); for (name, burn_ms) in [("warm-cpu-1", 2_000), ("warm-cpu-2", 500)] { let (folder, mut recorder) = start_recorder(name, &ticker, Arc::clone(&read)); host_cpu_ms.fetch_add(burn_ms, Ordering::SeqCst); recorder.exchange_ended(recorder.read_exchange_end(Instant::now(), &read())); + // The host keeps running between the exchange end and finish. + host_cpu_ms.fetch_add(250, Ordering::SeqCst); recorder .finish(Ok(succeeded_result(&invocation))) .expect("result"); let lines = read_lines(&folder.join(RECORD_FILE_NAME)); let _ = std::fs::remove_dir_all(folder); - own_cpu.push(cpu(lines.last().expect("end line")) - cpu(&lines[0])); + let end = cpu(lines.last().expect("end line")); + end_cpu.push(end); + own_cpu.push(end - cpu(&lines[0])); } - // Not 3.0 and 3.5, the host's totals at each end. + // The end lines keep the exchange-end totals, not 3.25 and 4.0 read + // at finish. The second start line's 3.25 includes the first + // invocation's 0.25 after its exchange, which neither difference + // counts. + assert_eq!(end_cpu, [3.0, 3.75]); + // Not 3.0 and 3.75, the host's totals at each exchange end. assert_eq!(own_cpu, [2.0, 0.5]); } } From 4585ac7ee03f0d055f71fc8dd46eef91778fbd97 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:51:15 -0700 Subject: [PATCH 063/102] fix: report Deep Agents cached input tokens LangChain carries cache reads per model call in usage_metadata.input_token_details.cache_read, and OpenAI raw usage carries them in prompt_tokens_details.cached_tokens, but the adapter never reported cached_input_tokens. Sum cached input per call with the same rule as the other counts: one call without a cache count leaves the sum unknown, and a reported 0 stays 0. Raw OpenAI usage is read first because ChatOpenAI renames the cache key for priority and flex service tiers. Report input_tokens_include_cache=True whenever input tokens are known. LangChain's UsageMetadata.input_tokens counts every input token type; langchain-anthropic adds cache reads and writes back into it, and OpenAI's prompt_tokens already includes cached tokens. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 7 ++ .../deepagents/adapter.py | 30 +++++ tests/adapters/test_deepagents.py | 113 ++++++++++++++++-- 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 66a44eece..9e41e8f53 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -252,6 +252,13 @@ and the provider's total is used when present. Other providers and streamed responses use LangChain's `usage_metadata`, which some integrations fill with 0 for an omitted count. +Cached input tokens come from `prompt_tokens_details.cached_tokens` in +OpenAI-shaped raw usage and from `usage_metadata.input_token_details.cache_read` +otherwise. They follow the same rule: one call without a cache count leaves the +invocation's cached count unknown, and a reported 0 stays 0. When input tokens +are known, the adapter reports that they include cached tokens, because +LangChain's `UsageMetadata.input_tokens` counts every input token type. + ## Runtime Lifecycle NeMo Fabric starts one local adapter host for every runtime. During runtime start, diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 37771f419..ba304d6b3 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -770,6 +770,7 @@ async def invoke( "input_tokens": usage.get("prompt_tokens"), "output_tokens": usage.get("completion_tokens"), "total_tokens": usage.get("total_tokens"), + "cached_input_tokens": usage.get("cached_tokens"), } tokens = { name: value @@ -778,6 +779,11 @@ async def invoke( and not isinstance(value, bool) and 0 <= value <= (1 << 64) - 1 } + if "input_tokens" in tokens: + # LangChain's UsageMetadata defines input_tokens as the sum of every + # input token type, cache reads included; OpenAI's raw prompt_tokens + # includes cached tokens too. + tokens["input_tokens_include_cache"] = True cost = usage.get("cost") if ( not isinstance(cost, (int, float)) @@ -1300,6 +1306,9 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: values = [_reported_tokens(call, source) for call in calls] if all(value is not None for value in values): totals[target] = sum(values) + cached = [_reported_cached_tokens(call) for call in calls] + if all(value is not None for value in cached): + totals["cached_tokens"] = sum(cached) costs = [_reported_cost(call) for call in calls] if all(cost is not None for cost in costs): totals["cost"] = math.fsum(costs) @@ -1342,6 +1351,27 @@ def _reported_tokens(call: dict[str, Any], name: str) -> int | None: return None +def _reported_cached_tokens(call: dict[str, Any]) -> int | None: + """Return one call's cache-read input tokens, or ``None`` when not reported. + + OpenAI-shaped raw usage reports them as ``prompt_tokens_details.cached_tokens``; + ``ChatOpenAI`` renames that key for priority and flex service tiers, so the raw + count is read when present. Other responses use LangChain's + ``usage_metadata.input_token_details.cache_read``. + """ + + metadata = call.get("response_metadata") + raw = metadata.get("token_usage") if isinstance(metadata, dict) else None + if isinstance(raw, dict): + details, key = raw.get("prompt_tokens_details"), "cached_tokens" + else: + usage = call.get("usage") + details = usage.get("input_token_details") if isinstance(usage, dict) else None + key = "cache_read" + value = details.get(key) if isinstance(details, dict) else None + return value if _is_token_count(value) else None + + def _is_token_count(value: Any) -> bool: return ( isinstance(value, int) diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index c916cb2a3..5268f3537 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2604,11 +2604,17 @@ async def test_summarization_model_call_usage_is_counted( assert "tools" not in summary_request assert "tools" in agent_request assert first.usage == AgentUsage( - input_tokens=1500, output_tokens=10, total_tokens=1510 + input_tokens=1500, + output_tokens=10, + total_tokens=1510, + input_tokens_include_cache=True, ) assert second.status is AgentRunStatus.SUCCEEDED assert second.usage == AgentUsage( - input_tokens=1700, output_tokens=65, total_tokens=1765 + input_tokens=1700, + output_tokens=65, + total_tokens=1765, + input_tokens_include_cache=True, ) @@ -2658,7 +2664,10 @@ async def test_subagent_model_calls_are_counted( } assert len(subgraphs) == 2 assert result.usage == AgentUsage( - input_tokens=1111, output_tokens=1111, total_tokens=2222 + input_tokens=1111, + output_tokens=1111, + total_tokens=2222, + input_tokens_include_cache=True, ) @@ -2680,7 +2689,10 @@ async def test_recursion_limit_failure_keeps_usage_of_completed_model_calls( assert result.status is AgentRunStatus.FAILED assert result.error.code == "deepagents_recursion_limit_reached" assert result.usage == AgentUsage( - input_tokens=10 * calls, output_tokens=calls, total_tokens=11 * calls + input_tokens=10 * calls, + output_tokens=calls, + total_tokens=11 * calls, + input_tokens_include_cache=True, ) @@ -2701,7 +2713,12 @@ def provider_failure(kind: str) -> Any: # usage stays known. pytest.param( "400", - AgentUsage(input_tokens=120, output_tokens=8, total_tokens=128), + AgentUsage( + input_tokens=120, + output_tokens=8, + total_tokens=128, + input_tokens_include_cache=True, + ), id="400-rejection", ), # The provider may have done work without reporting usage. @@ -2785,12 +2802,17 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): [ pytest.param( [{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}], - AgentUsage(input_tokens=0, output_tokens=0, total_tokens=0), + AgentUsage( + input_tokens=0, + output_tokens=0, + total_tokens=0, + input_tokens_include_cache=True, + ), id="explicit-zero", ), pytest.param( [{"prompt_tokens": 100}], - AgentUsage(input_tokens=100), + AgentUsage(input_tokens=100, input_tokens_include_cache=True), id="input-only", ), pytest.param( @@ -2800,7 +2822,12 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): ), pytest.param( [{"prompt_tokens": 100, "completion_tokens": 5}], - AgentUsage(input_tokens=100, output_tokens=5, total_tokens=105), + AgentUsage( + input_tokens=100, + output_tokens=5, + total_tokens=105, + input_tokens_include_cache=True, + ), id="total-omitted", ), pytest.param( @@ -2810,7 +2837,12 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): ), pytest.param( [{"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 0}], - AgentUsage(input_tokens=7, output_tokens=5, total_tokens=0), + AgentUsage( + input_tokens=7, + output_tokens=5, + total_tokens=0, + input_tokens_include_cache=True, + ), id="provider-total-zero-kept", ), pytest.param( @@ -2839,6 +2871,69 @@ async def test_partial_usage_reports_keep_missing_counts_unknown( assert result.usage == expected +@pytest.mark.parametrize( + ("cached", "expected"), + [ + pytest.param([40, 60], 100, id="every-call-reports-cache"), + pytest.param([40, None], None, id="one-call-omits-cache"), + pytest.param([0, 0], 0, id="explicit-zero"), + ], +) +async def test_cached_input_tokens_are_known_only_when_every_call_reported_them( + tmp_path, make_payload, openai_endpoint, cached, expected +): + def usage(cached_tokens: int | None) -> dict[str, Any]: + reported: dict[str, Any] = { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + } + if cached_tokens is not None: + reported["prompt_tokens_details"] = {"cached_tokens": cached_tokens} + return reported + + openai_endpoint["replies"] += [ + chat_completion(usage(cached[0]), tool_calls=[LS_TOOL_CALL]), + chat_completion(usage(cached[1])), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage == AgentUsage( + input_tokens=200, + cached_input_tokens=expected, + input_tokens_include_cache=True, + output_tokens=10, + total_tokens=210, + ) + + +@pytest.mark.parametrize( + ("cache_read", "expected"), + [ + pytest.param(30, 30, id="reported"), + pytest.param(0, 0, id="explicit-zero"), + pytest.param(None, None, id="omitted"), + ], +) +def test_anthropic_cache_reads_are_counted_as_included_input(cache_read, expected): + # langchain-anthropic adds cache reads and writes to input_tokens and keeps an + # omitted cache count out of input_token_details. + from anthropic.types import Usage + from langchain_anthropic.chat_models import _create_usage_metadata + + usage_metadata = _create_usage_metadata( + Usage(input_tokens=10, output_tokens=5, cache_read_input_tokens=cache_read) + ) + totals = adapter._aggregate_usage( + [adapter._message_to_dict(AIMessage("done", usage_metadata=usage_metadata))] + ) + + assert totals.get("cached_tokens") == expected + assert totals["prompt_tokens"] == 10 + (cache_read or 0) + + async def test_bad_mcp_transport_fails_runtime_start(tmp_path, make_payload): # A misconfigured MCP server must fail loudly, not be silently dropped. payload = make_payload(tmp_path) From fee4c9287832d1fd24664bfa1848ac784af66bad Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:30:54 -0700 Subject: [PATCH 064/102] fix: report zero Deep Agents usage when no model call ran The adapter drops a model call that the provider rejects with a 4xx status other than 408, because the provider spent no tokens on it. When every call was rejected, or no call started, the call list was empty and the usage was reported as unknown, although the invocation provably spent no tokens. The adapter now reports 0 for every token count and the cost in that case. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 15 +++++----- .../deepagents/adapter.py | 6 ++-- tests/adapters/test_deepagents.py | 29 +++++++++++++++++++ 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 9e41e8f53..e88b41c04 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -237,13 +237,14 @@ summarization. Agent Protocol subagents (entries with `graph_id`) run in their remote service, and their tokens are not included. A token count or cost is reported only when every counted call reported it; otherwise it stays unknown. This also holds for a failed invocation. A request the provider rejects with a -4xx status (other than 408) contributes no usage. A 5xx response, timeout, -cancellation or unfinished call leaves the invocation's counts and cost -unknown, because the provider may have done work without reporting usage. -OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries -to success is not visible to the adapter, and only the final response's -reported usage counts. Usage replayed from a configured LangChain model cache -would be counted as reported; NeMo Fabric configures no such cache. +4xx status (other than 408) contributes no usage, so an invocation that made no +model call, or whose every call was rejected, reports 0 tokens and 0 cost. A +5xx response, timeout, cancellation or unfinished call leaves the invocation's +counts and cost unknown, because the provider may have done work without +reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx +the SDK retries to success is not visible to the adapter, and only the final +response's reported usage counts. Usage replayed from a configured LangChain +model cache would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index ba304d6b3..e5561f215 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1292,11 +1292,11 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: """Sum model-call usage; a count is known only when every call reported it. One call without a count makes the invocation's sum for that count a - partial value, so it stays unknown rather than under-reported. + partial value, so it stays unknown rather than under-reported. Without a + call, because none started or the provider rejected every one, the + invocation spent nothing, so every count and the cost are 0. """ - if not calls: - return None totals: dict[str, Any] = {} for source, target in ( ("input_tokens", "prompt_tokens"), diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 5268f3537..6cdb0f466 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2744,6 +2744,35 @@ async def test_failed_model_call_usage_follows_provider_response( assert result.usage == expected +async def test_rejected_only_model_call_reports_zero_usage( + tmp_path, make_payload, openai_endpoint +): + # The provider rejected the only model call, so the invocation spent nothing. + openai_endpoint["replies"].append(provider_failure("400")) + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage == AgentUsage( + input_tokens=0, + cached_input_tokens=0, + output_tokens=0, + total_tokens=0, + input_tokens_include_cache=True, + cost_usd=0.0, + ) + + +def test_aggregate_usage_without_model_calls_is_zero(): + assert adapter._aggregate_usage([]) == { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "cached_tokens": 0, + "cost": 0.0, + } + + @pytest.mark.parametrize( ("failure", "expected"), [ From 2ee128d7485f15db1a5b0c66f98cbfdb04b30277 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:31:49 -0700 Subject: [PATCH 065/102] fix: report unknown Deep Agents usage with an Agent Protocol subagent Deep Agents routes a subagent spec with graph_id to its async subagent middleware, which starts and updates tasks on a remote Agent Protocol server. Those model calls never reach the adapter's callbacks, so the reported counts left them out while appearing complete. The adapter blocked such subagents only when a tools policy was configured. When the configured subagents include one with graph_id, the adapter now reports the invocation's token counts and cost as unknown. The local sums stay in output.usage. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 24 ++++++++-------- .../deepagents/adapter.py | 25 ++++++++++++++++- tests/adapters/test_deepagents.py | 28 +++++++++++++++++++ 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index e88b41c04..56f93989d 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -234,17 +234,19 @@ runtime start before an invocation is accepted. Usage sums every chat-model call that runs in the adapter process for the invocation: the main agent, declarative subagents, and Deep Agents summarization. Agent Protocol subagents (entries with `graph_id`) run in their -remote service, and their tokens are not included. A token count or cost is -reported only when every counted call reported it; otherwise it stays unknown. -This also holds for a failed invocation. A request the provider rejects with a -4xx status (other than 408) contributes no usage, so an invocation that made no -model call, or whose every call was rejected, reports 0 tokens and 0 cost. A -5xx response, timeout, cancellation or unfinished call leaves the invocation's -counts and cost unknown, because the provider may have done work without -reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx -the SDK retries to success is not visible to the adapter, and only the final -response's reported usage counts. Usage replayed from a configured LangChain -model cache would be counted as reported; NeMo Fabric configures no such cache. +remote service, where the adapter cannot count their calls, so a runtime +configured with one reports unknown token counts and cost. A token count or +cost is reported only when every counted call reported it; otherwise it stays +unknown. This also holds for a failed invocation. A request the provider +rejects with a 4xx status (other than 408) contributes no usage, so an +invocation that made no model call, or whose every call was rejected, reports 0 +tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call +leaves the invocation's counts and cost unknown, because the provider may have +done work without reporting usage. OpenAI SDK retries happen inside one +LangChain call, so a 5xx the SDK retries to success is not visible to the +adapter, and only the final response's reported usage counts. Usage replayed +from a configured LangChain model cache would be counted as reported; NeMo +Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index e5561f215..38a0eacbf 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -541,6 +541,18 @@ def _validated_deepagents_settings(extra: Any) -> dict[str, Any]: return dict(extra) +def _has_agent_protocol_subagent(subagents: Any) -> bool: + """Whether ``create_deep_agent`` runs a subagent on an Agent Protocol server. + + Deep Agents routes a subagent spec with ``graph_id`` to its async subagent + middleware, which starts and updates tasks on a remote server. + """ + return isinstance(subagents, list) and any( + isinstance(subagent, dict) and "graph_id" in subagent + for subagent in subagents + ) + + def _gate_subagent( subagent: dict[str, Any], enabled: set[str] | None, blocked: set[str] ) -> dict[str, Any]: @@ -612,6 +624,7 @@ def __init__(self) -> None: self._callback_handler_type: Any = None self._telemetry_quarantine: str | None = None self._telemetry_quarantine_cause: str | None = None + self._has_remote_subagent = False async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -662,6 +675,9 @@ async def start(self, payload: dict[str, Any]) -> None: model, settings, ) + self._has_remote_subagent = _has_agent_protocol_subagent( + agent_kwargs.get("subagents") + ) self._checkpointer = await open_checkpointer( checkpointer_path(runtime_context, base_dir, runtime_id) ) @@ -765,7 +781,14 @@ async def invoke( failed = bool(output.pop("failed", False)) reported_error = output.pop("error", None) raw_usage = output.get("usage") - usage = raw_usage if isinstance(raw_usage, dict) else {} + # An Agent Protocol subagent's model calls run on its remote server and + # never reach this process, so the local sums may undercount the + # invocation and its counts and cost stay unknown. + usage = ( + raw_usage + if isinstance(raw_usage, dict) and not self._has_remote_subagent + else {} + ) token_fields = { "input_tokens": usage.get("prompt_tokens"), "output_tokens": usage.get("completion_tokens"), diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 6cdb0f466..873126845 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2763,6 +2763,34 @@ async def test_rejected_only_model_call_reports_zero_usage( ) +async def test_agent_protocol_subagent_makes_usage_unknown( + tmp_path, make_payload, openai_endpoint +): + # An Agent Protocol subagent runs its model calls on a remote server, which + # the adapter cannot count, so local sums would undercount the invocation. + payload = make_payload(tmp_path) + payload["config"]["harness"]["settings"]["deepagents"] = { + "subagents": [ + { + "name": "researcher", + "description": "Researches on a remote server.", + "graph_id": "research_agent", + "url": "https://agents.example.test", + } + ] + } + openai_endpoint["replies"].append( + chat_completion( + {"prompt_tokens": 120, "completion_tokens": 8, "total_tokens": 128} + ) + ) + + (result,) = await run_invocations(payload, 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage is None + + def test_aggregate_usage_without_model_calls_is_zero(): assert adapter._aggregate_usage([]) == { "prompt_tokens": 0, From 06de4c0281439b64a9d05340ebca771278fe5743 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:32:07 -0700 Subject: [PATCH 066/102] docs: say a failed Deep Agents call may have been billed The usage notes said that after a 5xx response, timeout, or cancelled call the provider may have done work. The reason the counts stay unknown is narrower: the provider may have billed the call without reporting usage. The README, the recorder docstring, and a test comment now say that. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 2 +- .../deepagents/src/nemo_fabric_adapters/deepagents/adapter.py | 4 ++-- tests/adapters/test_deepagents.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 56f93989d..c1a5a8c98 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -242,7 +242,7 @@ rejects with a 4xx status (other than 408) contributes no usage, so an invocation that made no model call, or whose every call was rejected, reports 0 tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call leaves the invocation's counts and cost unknown, because the provider may have -done work without reporting usage. OpenAI SDK retries happen inside one +billed the call without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the adapter, and only the final response's reported usage counts. Usage replayed from a configured LangChain model cache would be counted as reported; NeMo diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 38a0eacbf..f886155c9 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1012,8 +1012,8 @@ class ModelCallUsage(BaseCallbackHandler): record when it ends. A request the provider rejects with a 4xx status (other than 408) contributes no usage and is dropped. A 5xx response, timeout, cancellation or unfinished call stays unreported and makes the invocation's - counts and cost unknown, because the provider may have done work without - reporting usage. + counts and cost unknown, because the provider may have billed the call + without reporting usage. """ run_inline = True diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 873126845..eec895d6a 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2721,7 +2721,7 @@ def provider_failure(kind: str) -> Any: ), id="400-rejection", ), - # The provider may have done work without reporting usage. + # The provider may have billed the call without reporting usage. pytest.param("408", None, id="408-request-timeout"), pytest.param("503", None, id="503-server-error"), pytest.param("timeout", None, id="read-timeout"), From e535051318613392a86fcb75cd4a469cedd21b18 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:38:36 -0700 Subject: [PATCH 067/102] style: format the Deep Agents remote subagent check Apply ruff format to the check added in the previous commit. Signed-off-by: mnajafian-nv --- .../deepagents/src/nemo_fabric_adapters/deepagents/adapter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index f886155c9..9a51bd817 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -548,8 +548,7 @@ def _has_agent_protocol_subagent(subagents: Any) -> bool: middleware, which starts and updates tasks on a remote server. """ return isinstance(subagents, list) and any( - isinstance(subagent, dict) and "graph_id" in subagent - for subagent in subagents + isinstance(subagent, dict) and "graph_id" in subagent for subagent in subagents ) From cb42bfe7d37d76beb93d1c0d8aa7f6bf61edda53 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:45:54 -0700 Subject: [PATCH 068/102] test: keep zero observed Deep Agents calls unknown with a remote subagent The adapter reports 0 when its callbacks observed no billed model call. That 0 holds for the invocation only when every model call runs through those callbacks. Deep Agents runs declarative subagents and summarization inside the graph, where the run config's callbacks reach them, but an Agent Protocol subagent (graph_id) runs on a remote server that the callbacks never see. The adapter already reports unknown usage for a runtime with such a subagent; this test pins that for an invocation whose only observed call was rejected, and the docstring and README state the condition. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 18 +++++++------- .../deepagents/adapter.py | 5 +++- tests/adapters/test_deepagents.py | 24 +++++++++++++++++++ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index c1a5a8c98..6340b4ce3 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -238,15 +238,15 @@ remote service, where the adapter cannot count their calls, so a runtime configured with one reports unknown token counts and cost. A token count or cost is reported only when every counted call reported it; otherwise it stays unknown. This also holds for a failed invocation. A request the provider -rejects with a 4xx status (other than 408) contributes no usage, so an -invocation that made no model call, or whose every call was rejected, reports 0 -tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call -leaves the invocation's counts and cost unknown, because the provider may have -billed the call without reporting usage. OpenAI SDK retries happen inside one -LangChain call, so a 5xx the SDK retries to success is not visible to the -adapter, and only the final response's reported usage counts. Usage replayed -from a configured LangChain model cache would be counted as reported; NeMo -Fabric configures no such cache. +rejects with a 4xx status (other than 408) contributes no usage, so without an +Agent Protocol subagent an invocation that made no model call, or whose every +call was rejected, reports 0 tokens and 0 cost. A 5xx response, timeout, +cancellation or unfinished call leaves the invocation's counts and cost +unknown, because the provider may have billed the call without reporting usage. +OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries +to success is not visible to the adapter, and only the final response's +reported usage counts. Usage replayed from a configured LangChain model cache +would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 9a51bd817..bfda396dc 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1316,7 +1316,10 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: One call without a count makes the invocation's sum for that count a partial value, so it stays unknown rather than under-reported. Without a call, because none started or the provider rejected every one, the - invocation spent nothing, so every count and the cost are 0. + observed calls spent nothing, so every count and the cost are 0. That 0 + covers the invocation only when every model call runs through these + callbacks; the caller makes usage unknown when an Agent Protocol subagent + can run models on a remote server. """ totals: dict[str, Any] = {} diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index eec895d6a..816ccb1ec 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2791,6 +2791,30 @@ async def test_agent_protocol_subagent_makes_usage_unknown( assert result.usage is None +async def test_rejected_only_call_with_agent_protocol_subagent_is_unknown( + tmp_path, make_payload, openai_endpoint +): + # Zero observed calls prove nothing when a remote subagent can run models + # that the adapter's callbacks never see. + payload = make_payload(tmp_path) + payload["config"]["harness"]["settings"]["deepagents"] = { + "subagents": [ + { + "name": "researcher", + "description": "Researches on a remote server.", + "graph_id": "research_agent", + "url": "https://agents.example.test", + } + ] + } + openai_endpoint["replies"].append(provider_failure("400")) + + (result,) = await run_invocations(payload, 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage is None + + def test_aggregate_usage_without_model_calls_is_zero(): assert adapter._aggregate_usage([]) == { "prompt_tokens": 0, From ce744150ea1e26be5b6b8a42a26d95b8a533d77f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:04:44 -0700 Subject: [PATCH 069/102] fix: preserve empty OpenShell environment arguments Signed-off-by: mnajafian-nv --- examples/openshell/run-demo.sh | 4 ++- tests/scripts/test_openshell_launcher.py | 44 +++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/examples/openshell/run-demo.sh b/examples/openshell/run-demo.sh index 56d2b534f..4fe36156e 100644 --- a/examples/openshell/run-demo.sh +++ b/examples/openshell/run-demo.sh @@ -272,9 +272,11 @@ run_consumer() { if [[ "${EXAMPLE_MODE}" == caller_owned || "${EXAMPLE_MODE}" == both ]]; then CALLER_SANDBOX_ATTEMPTED=1 + # Guard empty arrays for Bash 3.2's set -u behavior without adding empty args. "${OPENSHELL_CLI}" --gateway-endpoint "http://127.0.0.1:${GATEWAY_PORT}" \ --workspace default sandbox create --name "${SANDBOX_NAME}" --from "${RUNTIME_IMAGE}" \ - --policy "${POLICY_FILE}" "${MODEL_ENV_ARGS[@]}" "${SERVER_ENV_ARGS[@]}" \ + --policy "${POLICY_FILE}" ${MODEL_ENV_ARGS[@]+"${MODEL_ENV_ARGS[@]}"} \ + ${SERVER_ENV_ARGS[@]+"${SERVER_ENV_ARGS[@]}"} \ --detach --no-tty -- fabric-agent-ctl serve SANDBOX_JSON="$("${OPENSHELL_CLI}" --gateway-endpoint "http://127.0.0.1:${GATEWAY_PORT}" \ --workspace default sandbox get "${SANDBOX_NAME}" --output json)" diff --git a/tests/scripts/test_openshell_launcher.py b/tests/scripts/test_openshell_launcher.py index 317b8052d..15ab68caa 100644 --- a/tests/scripts/test_openshell_launcher.py +++ b/tests/scripts/test_openshell_launcher.py @@ -12,6 +12,7 @@ import json import os from pathlib import Path +import shlex import shutil import socket import subprocess @@ -172,7 +173,22 @@ def _run(launcher, workload, env): ) -def test_generic_workload_gets_ownership_and_consumer_args(launcher): +@pytest.mark.parametrize( + ("model_env_args", "server_env_args"), + [ + pytest.param([], None, id="default-server-env"), + pytest.param([], [], id="no-env"), + pytest.param(["--env", "MODEL_HINT=words with spaces"], [], id="no-server-env"), + pytest.param( + ["--env", "MODEL_HINT=words with spaces"], + ["--env", "SERVER_HINT=words with spaces"], + id="quoted-env", + ), + ], +) +def test_generic_workload_gets_ownership_and_consumer_args( + launcher, model_env_args, server_env_args +): _, root, _ = launcher workload = root / "third-workload" workload.mkdir() @@ -185,6 +201,12 @@ def test_generic_workload_gets_ownership_and_consumer_args(launcher): stage_workload_image() { :; } run_workload_consumer() { local mode="$1"; shift; uv run --no-sync consumer "$mode" "$@"; } """ + + f"MODEL_ENV_ARGS=({shlex.join(model_env_args)})\n" + + ( + f"SERVER_ENV_ARGS=({shlex.join(server_env_args)})\n" + if server_env_args is not None + else "" + ) ) completed = _run(launcher, hook, {"OPENSHELL_EXAMPLE_MODE": "both"}) @@ -195,6 +217,26 @@ def test_generic_workload_gets_ownership_and_consumer_args(launcher): assert all(call["args"][-2:] == ["--input", "words with spaces"] for call in calls) assert all(call["provider"].endswith("/fabric-environment-openshell") for call in calls) assert not (root / "sandbox-state.json").exists() + create = next( + operation["args"] + for operation in _operations(root) + if operation["command"] == "openshell" + and "sandbox" in operation["args"] + and "create" in operation["args"] + ) + expected_server_env = ( + server_env_args + if server_env_args is not None + else [ + "--env", + "ADAPTER_PYTHON=/opt/fabric/bin/python", + "--env", + "TMPDIR=/sandbox/tmp", + ] + ) + assert create[create.index("--policy") + 2 : create.index("--detach")] == ( + model_env_args + expected_server_env + ) @pytest.mark.parametrize("mode", ["caller_owned", "fabric_owned"]) From 790f3693aaebf9465bc224bd76da41fb3d81d420 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:10:57 -0700 Subject: [PATCH 070/102] fix: avoid premature Relay gateway startup failures Signed-off-by: mnajafian-nv --- adapters/python/claude/README.md | 2 +- adapters/python/codex/README.md | 4 +- adapters/python/common/README.md | 14 +++++ .../common/relay_gateway.py | 21 +++++-- tests/_utils/utils.py | 22 +++++-- tests/adapters/test_adapaters_common_utils.py | 4 +- .../test_adapters_common_relay_gateway.py | 63 +++++++++++++++++++ 7 files changed, 117 insertions(+), 13 deletions(-) diff --git a/adapters/python/claude/README.md b/adapters/python/claude/README.md index a6b8ac271..bfa13847c 100644 --- a/adapters/python/claude/README.md +++ b/adapters/python/claude/README.md @@ -169,7 +169,7 @@ config.enable_relay( ``` For each Relay-enabled Claude runtime, NeMo Fabric starts one `nemo-relay` gateway, -waits for its health endpoint, and stops it with the runtime. NeMo Fabric passes the +waits up to 30 seconds for its health endpoint, and stops it with the runtime. NeMo Fabric passes the gateway URL to the connected Claude Code process through `ANTHROPIC_BASE_URL` and `NEMO_RELAY_GATEWAY_URL`, and passes the selected explicit model endpoint to the gateway as its Anthropic upstream. It also stages a runtime-scoped Claude diff --git a/adapters/python/codex/README.md b/adapters/python/codex/README.md index 3cf1de779..ba7eab1b7 100644 --- a/adapters/python/codex/README.md +++ b/adapters/python/codex/README.md @@ -154,7 +154,9 @@ Codex adapter does not provide a separate `relay` extra; its `harness` and Enable Relay with `FabricConfig.enable_relay(...)`. The adapter starts the installed `nemo-relay` CLI as a supervised sidecar; do not start the gateway -separately. +separately. Gateway readiness has a 30-second budget covering configuration +validation and required worker initialization. Startup fails on early process exit +or an expired budget, and the gateway log is retained for diagnosis. NeMo Fabric routes the selected Responses-compatible provider through the gateway and passes its explicit `base_url` to Relay as the upstream endpoint. diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index 17e537cd5..fb8400337 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -122,6 +122,20 @@ state during `start`. Adapter stdout is reserved for the protocol; diagnostics are redirected to stderr. A host crash or protocol timeout terminates that runtime. +## Relay Gateway Readiness + +The Claude and Codex adapters wait up to 30 seconds for the supervised Relay +gateway's HTTP health endpoint. This budget includes configuration validation +and required worker initialization. Early process exit fails startup immediately; +a gateway that never becomes healthy fails when the budget expires and is stopped. +The adapters retain its log for diagnosis. Local Python hosts separately enforce +a 90-second adapter startup limit. + +Relay 0.10.3 includes an +[upstream startup-memory fix](https://github.com/NVIDIA/NeMo-Relay/pull/1375). +Use that release to obtain the fix on 0.10 installations. Fabric continues to +accept the supported `>=0.9,<0.11` CLI range. + ## Relay Request Correlation In-process Relay SDK adapters that own their Agent scope can use diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 45cf04faa..959f54fa7 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -20,7 +20,10 @@ from nemo_fabric_adapters.common.relay_version import is_supported_relay_version -RELAY_HEALTH_TIMEOUT_SECONDS = 10.0 +# Readiness includes plugin preflight and worker startup before health is served. +# This operational budget is separate from Relay's inner request timeouts and +# Fabric's outer adapter-start limit. +RELAY_STARTUP_TIMEOUT_SECONDS = 30.0 RELAY_STOP_TIMEOUT_SECONDS = 5.0 RELAY_VERSION_TIMEOUT_SECONDS = 5.0 @@ -110,24 +113,32 @@ def wait_for_relay_gateway( process: subprocess.Popen[Any], health_url: str, *, - timeout: float = RELAY_HEALTH_TIMEOUT_SECONDS, + timeout: float = RELAY_STARTUP_TIMEOUT_SECONDS, ) -> None: """Wait until the Relay health endpoint succeeds or startup fails.""" deadline = time.monotonic() + timeout - while time.monotonic() < deadline: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break returncode = process.poll() if returncode is not None: raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) try: - with urllib.request.urlopen(health_url, timeout=1) as response: + with urllib.request.urlopen( + health_url, timeout=min(1.0, remaining) + ) as response: if 200 <= response.status < 300: return except (OSError, urllib.error.URLError): pass - time.sleep(0.1) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(0.1, remaining)) raise RelayGatewayError(f"NeMo Relay gateway did not become ready at {health_url}") diff --git a/tests/_utils/utils.py b/tests/_utils/utils.py index 9b7ca63be..aa5521ae8 100644 --- a/tests/_utils/utils.py +++ b/tests/_utils/utils.py @@ -22,11 +22,17 @@ def assert_atof_skill_selection( output: Mapping[str, Any], expected_skill: str | None ) -> None: records = atof_records(output) - tool_records = [record for record in records if record["category"] == "tool"] + tool_records = [ + record + for record in records + if record["kind"] == "scope" and record["category"] == "tool" + ] llm_end_records = [ record for record in records - if record["category"] == "llm" and record["scope_category"] == "end" + if record["kind"] == "scope" + and record["category"] == "llm" + and record["scope_category"] == "end" ] serialized_tool_records = json.dumps(tool_records).replace("\\\\", "/") serialized_skill_calls = json.dumps(tool_records + llm_end_records).replace( @@ -55,7 +61,9 @@ def assert_atof_model(output: Mapping[str, Any], expected_model: str) -> None: llm_starts = [ record for record in events - if record["category"] == "llm" and record["scope_category"] == "start" + if record["kind"] == "scope" + and record["category"] == "llm" + and record["scope_category"] == "start" ] assert llm_starts, events assert {record["data"]["content"]["model"] for record in llm_starts} == { @@ -81,7 +89,9 @@ def assert_semantic_relay_artifacts( llm_starts = [ event for event in events - if event.get("category") == "llm" and event.get("scope_category") == "start" + if event["kind"] == "scope" + and event["category"] == "llm" + and event["scope_category"] == "start" ] assert llm_starts, events assert all( @@ -92,7 +102,9 @@ def assert_semantic_relay_artifacts( llm_ends = [ event for event in events - if event.get("category") == "llm" and event.get("scope_category") == "end" + if event["kind"] == "scope" + and event["category"] == "llm" + and event["scope_category"] == "end" ] assert llm_ends, events assert any(_relay_event_total_tokens(event) > 0 for event in llm_ends) diff --git a/tests/adapters/test_adapaters_common_utils.py b/tests/adapters/test_adapaters_common_utils.py index d1805f12b..0be298052 100644 --- a/tests/adapters/test_adapaters_common_utils.py +++ b/tests/adapters/test_adapaters_common_utils.py @@ -1332,8 +1332,10 @@ async def test_relay_initializes_v4_atof_atif_config( } common_utils.validate_relay_observability(plugin_config) assert plugin.validate_exact(plugin_config)["config"]["diagnostics"] == [] + # Activation includes discovered host plugins, so compare the same layers. + expected_diagnostics = plugin.validate(plugin_config)["config"]["diagnostics"] async with plugin.activate(plugin_config) as activation: - assert activation.report["config"]["diagnostics"] == [] + assert activation.report["config"]["diagnostics"] == expected_diagnostics def test_relay_validates_all_v4_otlp_fields(): diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index c307254ae..e478f85de 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -271,3 +271,66 @@ def test_stop_relay_gateway_is_idempotent_for_exited_process(): process.terminate.assert_not_called() process.kill.assert_not_called() + + +@pytest.fixture(name="gateway_clock") +def gateway_clock_fixture(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + mock_clock = MagicMock(spec=relay_gateway.time) + mock_clock.monotonic.return_value = 0.0 + + def advance(seconds: float) -> None: + mock_clock.monotonic.return_value += seconds + + mock_clock.sleep.side_effect = advance + monkeypatch.setattr(relay_gateway, "time", mock_clock) + return mock_clock + + +def test_wait_for_relay_gateway_allows_preflight_before_worker_readiness( + monkeypatch, gateway_clock +): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += timeout + if gateway_clock.monotonic() < 14.0: + raise relay_gateway.urllib.error.URLError("still initializing") + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") + + assert 14.0 <= gateway_clock.monotonic() < 30.0 + + +@pytest.mark.parametrize( + ("probe_elapsed", "expected_sleep"), [(0.25, None), (0.20, 0.05)] +) +def test_wait_for_relay_gateway_limits_each_probe_to_remaining_startup_budget( + monkeypatch, gateway_clock, probe_elapsed, expected_sleep +): + process = MagicMock() + process.poll.return_value = None + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += probe_elapsed + raise relay_gateway.urllib.error.URLError("not ready") + + mock_probe = MagicMock(side_effect=probe_health) + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", mock_probe) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + mock_probe.assert_called_once_with("http://127.0.0.1:43210/healthz", timeout=0.25) + if expected_sleep is None: + gateway_clock.sleep.assert_not_called() + else: + gateway_clock.sleep.assert_called_once_with(pytest.approx(expected_sleep)) + assert gateway_clock.monotonic() == pytest.approx(0.25) From 5db49175c27b37f704350cfbfa6fc51bdba1fd0e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:41:46 -0700 Subject: [PATCH 071/102] docs: use the NeMo Fabric product name Signed-off-by: mnajafian-nv --- adapters/python/common/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index fb8400337..919bdd233 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -133,7 +133,7 @@ a 90-second adapter startup limit. Relay 0.10.3 includes an [upstream startup-memory fix](https://github.com/NVIDIA/NeMo-Relay/pull/1375). -Use that release to obtain the fix on 0.10 installations. Fabric continues to +Use that release to obtain the fix on 0.10 installations. NeMo Fabric continues to accept the supported `>=0.9,<0.11` CLI range. ## Relay Request Correlation From 06369c42bf3451c2179c920ee1dde9ace0dbdc7c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 14:09:20 -0700 Subject: [PATCH 072/102] feat: estimate model cost with versioned pricing Signed-off-by: mnajafian-nv --- Cargo.lock | 1 + Cargo.toml | 2 + adapters/python/deepagents/README.md | 4 +- crates/fabric-cli/src/presets.rs | 1 + crates/fabric-cli/templates/rust/main.rs.tmpl | 1 + crates/fabric-core/Cargo.toml | 1 + crates/fabric-core/data/model-pricing.json | 71 ++ crates/fabric-core/src/config.rs | 48 + crates/fabric-core/src/lib.rs | 10 +- crates/fabric-core/src/model_usage.rs | 390 +----- crates/fabric-core/src/price_catalog.rs | 1128 +++++++++++++++++ crates/fabric-core/src/pricing.rs | 683 +++++++++- crates/fabric-core/src/runtime.rs | 89 +- crates/fabric-core/src/usage_normalization.rs | 6 +- crates/fabric-core/tests/model_usage.rs | 521 ++++++++ docs/adapter-contract/tutorials/results.md | 2 +- .../api/python-library-reference/index.md | 1 + .../nemo_fabric.models.md | 185 +-- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../config/enum-capabilitykind.mdx | 34 +- .../config/enum-capabilitytarget.mdx | 34 +- .../config/enum-relayatifstorageconfig.mdx | 32 +- .../config/enum-relayatofmode.mdx | 38 +- .../config/enum-relayatofsinkconfig.mdx | 32 +- .../enum-relayatofstreamfieldnamepolicy.mdx | 38 +- .../config/enum-relayatofstreamtransport.mdx | 38 +- .../enum-relayopentelemetryendpointtype.mdx | 34 +- .../enum-relayopentelemetrymarkprojection.mdx | 38 +- .../config/enum-relayotlptransport.mdx | 38 +- .../config/enum-relayunsupportedbehavior.mdx | 38 +- .../config/enum-resolutionstrategy.mdx | 34 +- .../config/enum-telemetryprovider.mdx | 62 +- .../config/fn-load-adapter-descriptor.mdx | 4 +- .../fn-load-adapter-target-descriptor.mdx | 4 +- .../fn-resolve-run-plan-from-config.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 3 +- .../config/struct-fabricconfig.mdx | 34 +- .../config/struct-pricingconfig.mdx | 106 ++ .../config/struct-relayatifconfig.mdx | 36 +- .../config/struct-relayatofconfig.mdx | 36 +- .../config/struct-relaycomponentconfig.mdx | 32 +- .../config/struct-relayconfig.mdx | 36 +- .../config/struct-relayconfigpolicy.mdx | 34 +- .../struct-relayobservabilityconfig.mdx | 36 +- .../struct-relayopentelemetryconfig.mdx | 36 +- ...truct-relayopentelemetryendpointconfig.mdx | 32 +- .../config/struct-resolvecontext.mdx | 24 +- .../struct-resolvedadapterdescriptor.mdx | 32 +- ...struct-resolvedadaptertargetdescriptor.mdx | 32 +- .../config/struct-runplan.mdx | 36 +- .../config/struct-runtimecapabilities.mdx | 36 +- .../config/struct-runtimeconfig.mdx | 32 +- .../config/struct-skillconfig.mdx | 36 +- .../config/struct-telemetryconfig.mdx | 32 +- .../config/struct-telemetryplan.mdx | 32 +- .../config/struct-telemetryproviderconfig.mdx | 36 +- .../config/struct-tooldefinitionconfig.mdx | 32 +- .../config/struct-toolsconfig.mdx | 36 +- .../config/struct-toolsplan.mdx | 36 +- .../config/struct-workflowconfig.mdx | 32 +- .../struct-workflowentrypointconfig.mdx | 32 +- .../config/struct-workflowtargetspec.mdx | 32 +- .../doctor/enum-doctorstatus.mdx | 34 +- .../doctor/fn-doctor-plan.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../doctor/struct-doctorcheck.mdx | 32 +- .../doctor/struct-doctorreport.mdx | 32 +- .../error/enum-fabricerror.mdx | 24 +- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/error/type-result.mdx | 4 +- .../nemo-fabric-core/fn-version.mdx | 4 +- .../nemo-fabric-core/index.mdx | 4 +- .../model-usage/fn-normalize-model-usage.mdx | 15 + .../nemo-fabric-core/model-usage/index.mdx | 6 +- .../enum-cachereadaccounting.mdx | 116 ++ .../enum-pricingcatalogerror.mdx | 129 ++ .../price-catalog/enum-pricingunit.mdx | 138 ++ .../enum-ratescheduleapplication.mdx | 117 ++ .../price-catalog/enum-tokenrateschedule.mdx | 107 ++ .../nemo-fabric-core/price-catalog/index.mdx | 34 + .../price-catalog/struct-modelpricing.mdx | 131 ++ .../price-catalog/struct-pricingcatalog.mdx | 113 ++ .../struct-promptcachepricing.mdx | 103 ++ .../struct-tokenpricingrates.mdx | 111 ++ .../price-catalog/struct-tokenratetier.mdx | 107 ++ .../price-catalog/struct-userpricecatalog.mdx | 115 ++ .../nemo-fabric-core/pricing/index.mdx | 6 +- .../resource-snapshot/index.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 2 +- .../nemo-fabric-core/schema/index.mdx | 2 +- .../nemo-fabric-core/system-stats/index.mdx | 2 +- .../usage-accounting/index.mdx | 2 +- docs/sdk/python.mdx | 18 +- schemas/sdk/agent.schema.json | 25 + schemas/sdk/run-plan.schema.json | 322 +++++ .../src/nemo_fabric/__init__.py | 2 + .../src/nemo_fabric/models.py | 14 + tests/python/conftest.py | 13 +- tests/python/test_usage_reducer_parity.py | 36 +- 101 files changed, 5267 insertions(+), 1260 deletions(-) create mode 100644 crates/fabric-core/data/model-pricing.json create mode 100644 crates/fabric-core/src/price_catalog.rs create mode 100644 crates/fabric-core/tests/model_usage.rs create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx diff --git a/Cargo.lock b/Cargo.lock index 9e6be276f..8eaa11a9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1345,6 +1345,7 @@ dependencies = [ "schemars", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "windows-sys 0.61.2", ] diff --git a/Cargo.toml b/Cargo.toml index 76ee54284..f7dff9b70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,8 @@ libc = "0.2.186" schemars = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +# Standard digest for reproducible price-catalog snapshots; already in the lockfile. +sha2 = { version = "0.10", default-features = false } thiserror = "2" windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_System_Diagnostics_ToolHelp", "Win32_System_JobObjects", "Win32_System_Threading"] } pyo3 = "0.28.2" diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index fdfb09f2e..e8ccacf66 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -256,8 +256,8 @@ does not collect provider-reported cost or promote harness cost estimates. Native OpenAI pricing identity requires every observed request for that call to use `https://api.openai.com` on the standard HTTPS port. Custom endpoints, including environment-selected endpoints, keep their token and model evidence -without native-provider pricing identity. Catalog estimation is added separately; -this collector does not compute prices. +without native-provider pricing identity; an applicable user catalog can price +them explicitly. Unmanaged ChatOpenAI model overrides retain available observations with unverified scope. Other provider clients that expose usage only through LangChain diff --git a/crates/fabric-cli/src/presets.rs b/crates/fabric-cli/src/presets.rs index a3ec93272..44a3fbb6c 100644 --- a/crates/fabric-cli/src/presets.rs +++ b/crates/fabric-cli/src/presets.rs @@ -308,6 +308,7 @@ fn config( telemetry: None, system_stats: Some(Default::default()), relay: None, + pricing: None, extensions: BTreeMap::new(), } } diff --git a/crates/fabric-cli/templates/rust/main.rs.tmpl b/crates/fabric-cli/templates/rust/main.rs.tmpl index 611a62b46..c281b0105 100644 --- a/crates/fabric-cli/templates/rust/main.rs.tmpl +++ b/crates/fabric-cli/templates/rust/main.rs.tmpl @@ -62,6 +62,7 @@ fn build_config() -> FabricConfig { telemetry: None, system_stats: Some(Default::default()), relay: None, + pricing: None, extensions: BTreeMap::new(), } } diff --git a/crates/fabric-core/Cargo.toml b/crates/fabric-core/Cargo.toml index 440de80fa..1572f92e3 100644 --- a/crates/fabric-core/Cargo.toml +++ b/crates/fabric-core/Cargo.toml @@ -18,6 +18,7 @@ jsonschema.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true thiserror.workspace = true [target.'cfg(unix)'.dependencies] diff --git a/crates/fabric-core/data/model-pricing.json b/crates/fabric-core/data/model-pricing.json new file mode 100644 index 000000000..376658dda --- /dev/null +++ b/crates/fabric-core/data/model-pricing.json @@ -0,0 +1,71 @@ +{ + "version": 1, + "entries": [ + { + "provider": "anthropic", + "model_id": "claude-opus-4-5", + "aliases": ["claude-opus-4-5-20251101"], + "currency": "USD", + "unit": "per_token", + "rates": { + "input_per_million": 5.0, + "output_per_million": 25.0, + "cache_read_per_million": 0.5, + "cache_write_per_million": 6.25 + }, + "prompt_cache": {"read_accounting": "separate"}, + "pricing_as_of": "2026-10-08", + "pricing_source": "https://platform.claude.com/docs/en/about-claude/pricing" + }, + { + "provider": "anthropic", + "model_id": "claude-sonnet-4-5", + "aliases": ["claude-sonnet-4-5-20250929"], + "currency": "USD", + "unit": "per_token", + "rates": { + "input_per_million": 3.0, + "output_per_million": 15.0, + "cache_read_per_million": 0.3, + "cache_write_per_million": 3.75 + }, + "prompt_cache": {"read_accounting": "separate"}, + "pricing_as_of": "2026-10-08", + "pricing_source": "https://platform.claude.com/docs/en/about-claude/pricing" + }, + { + "provider": "openai", + "model_id": "gpt-5.4", + "aliases": ["gpt-5.4-2026-03-05"], + "currency": "USD", + "unit": "per_token", + "rate_schedule": { + "type": "prompt_token_threshold", + "applies_to": "full_request", + "tiers": [ + { + "max_prompt_tokens": 272000, + "rates": { + "input_per_million": 2.5, + "output_per_million": 15.0, + "cache_read_per_million": 0.25, + "cache_write_per_million": 2.5 + } + }, + { + "min_prompt_tokens": 272001, + "rates": { + "input_per_million": 5.0, + "output_per_million": 22.5, + "cache_read_per_million": 0.5, + "cache_write_per_million": 5.0 + } + } + ] + }, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"}, + "pricing_as_of": "2026-10-08", + "pricing_source": "https://developers.openai.com/api/docs/pricing" + } + ] +} diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index c5a57eb28..19fc846d8 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -23,6 +23,8 @@ pub use crate::agent_config::{ use crate::agent_execution::{AgentRunRequest, AgentRunResult}; use crate::error::{FabricError, Result}; +use crate::price_catalog::UserPriceCatalog; + /// Versioned NVIDIA NeMo Fabric agent config. /// /// NeMo Fabric-owned fields apply uniformly, while adapter-translated fields are @@ -72,11 +74,25 @@ pub struct FabricConfig { /// First-class NeMo Relay integration configuration. #[serde(default, skip_serializing_if = "Option::is_none")] pub relay: Option, + /// Optional catalog override for central token-price estimates. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pricing: Option, /// Additive fields not yet recognized by this core version. #[serde(default, flatten)] pub extensions: BTreeMap, } +/// Central token-price configuration; never changes provider requests. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct PricingConfig { + /// Relay-compatible v1 catalog, resolved relative to the config base directory. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub catalog_path: Option, + /// Additive pricing configuration, preserved for future versions. + #[serde(default, flatten)] + pub extensions: BTreeMap, +} + /// How an instruction value is applied to the selected harness. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] @@ -2597,7 +2613,9 @@ fn resolve_selected_descriptors( descriptor, adapter_target_descriptor.as_ref(), ); + let user_price_catalog = resolve_user_price_catalog(&config, &base_dir)?; Ok(RunPlan { + user_price_catalog, agent_name: config.metadata.name.clone(), base_dir, config, @@ -2612,6 +2630,33 @@ fn resolve_selected_descriptors( }) } +fn resolve_user_price_catalog( + config: &FabricConfig, + base_dir: &Path, +) -> Result> { + let Some(path) = config + .pricing + .as_ref() + .and_then(|p| p.catalog_path.as_ref()) + else { + return Ok(None); + }; + if path.as_os_str().to_string_lossy().trim().is_empty() { + return invalid_config("pricing.catalog_path", "must not be blank"); + } + let path = resolve_path(base_dir, path); + let bytes = std::fs::read(&path).map_err(|error| FabricError::InvalidConfig { + field: "pricing.catalog_path".into(), + reason: format!("cannot read {}: {error}", path.display()), + })?; + UserPriceCatalog::from_bytes(path, &bytes) + .map(Some) + .map_err(|error| FabricError::InvalidConfig { + field: "pricing.catalog_path".into(), + reason: error.to_string(), + }) +} + fn validate_capability_plan_compatibility( capability_plan: &CapabilityPlan, descriptor: Option<&AdapterDescriptor>, @@ -4021,6 +4066,9 @@ fn normalize_path(path: PathBuf) -> PathBuf { /// Resolved NeMo Fabric run plan. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct RunPlan { + /// Catalog bytes are frozen during planning. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_price_catalog: Option, /// Stable agent name. pub agent_name: String, /// Base directory used to resolve relative NeMo Fabric paths. diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index d0bd8d4f6..9122aade4 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -12,6 +12,7 @@ mod environment; pub mod error; mod measurement; pub mod model_usage; +pub mod price_catalog; pub mod pricing; pub mod resource_snapshot; pub mod runtime; @@ -38,10 +39,11 @@ pub use config::{ EnvironmentOwnership, EnvironmentPlan, FabricConfig, HarnessConfig, InstructionConfig, InstructionMode, InstructionsConfig, McpAuthenticationConfig, McpConfig, McpExposure, McpServerPlan, McpTransport, MetadataConfig, ModelConfig, OAuthTokenEndpointAuthMethod, - ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, ResolvedAdapterTargetDescriptor, - RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, TelemetryConfig, TelemetryPlan, - TelemetryProvider, TelemetryProviderConfig, ToolDefinitionConfig, ToolsConfig, WorkflowConfig, - WorkflowEntrypointConfig, WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, + PricingConfig, ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, + ResolvedAdapterTargetDescriptor, RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, + TelemetryConfig, TelemetryPlan, TelemetryProvider, TelemetryProviderConfig, + ToolDefinitionConfig, ToolsConfig, WorkflowConfig, WorkflowEntrypointConfig, + WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, load_adapter_target_descriptor, resolve_diagnostic_plan_from_config, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config, resolve_run_plan_from_config_with_adapter_directories, diff --git a/crates/fabric-core/src/model_usage.rs b/crates/fabric-core/src/model_usage.rs index 9d9aaff80..829a7ebad 100644 --- a/crates/fabric-core/src/model_usage.rs +++ b/crates/fabric-core/src/model_usage.rs @@ -139,13 +139,30 @@ pub struct ModelUsage { /// Normalize supported evidence. Legacy usage remains raw rather than gaining /// a completeness guarantee merely because a newer core read it. -pub(crate) fn normalize_model_usage( +pub fn normalize_model_usage( usage: &AgentUsage, + catalog: Option<&crate::price_catalog::UserPriceCatalog>, + bundled_pricing_allowed: bool, ) -> Result, AccountingError> { - let Some(_) = UsageAccounting::from_usage(usage)? else { + normalize_model_usage_for_routes(usage, catalog, |_, _| bundled_pricing_allowed) +} + +pub(crate) fn normalize_model_usage_for_routes( + usage: &AgentUsage, + catalog: Option<&crate::price_catalog::UserPriceCatalog>, + bundled_pricing_allowed: impl Fn(Option<&str>, &str) -> bool, +) -> Result, AccountingError> { + let Some(evidence) = UsageAccounting::from_usage(usage)? else { return Ok(None); }; - Ok(Some(view(normalized::describe_model_usage(usage)?))) + let mut result = normalized::describe_model_usage(usage)?; + crate::pricing::apply_pricing_with_evidence( + &mut result, + catalog, + bundled_pricing_allowed, + Some(&evidence), + ); + Ok(Some(view(result))) } fn tokens(values: &normalized::ModelMeasurements) -> TokenCounts { @@ -262,370 +279,3 @@ fn view(value: normalized::ModelUsage) -> ModelUsage { cost: cost(&measurements.token_cost, &accounting.observed), } } - -#[cfg(test)] -mod accounting_tests { - use super::normalize_model_usage; - use crate::AgentUsage; - use serde_json::{Value, json}; - - fn normalize(base: Value, accounting: Value) -> Value { - let mut usage: AgentUsage = serde_json::from_value(base).unwrap(); - usage - .extensions - .insert("nemo_fabric.usage_accounting.v1".into(), accounting); - serde_json::to_value(normalize_model_usage(&usage).unwrap().unwrap()).unwrap() - } - - #[test] - fn shared_source_vectors_keep_the_reduced_complete_and_observed_contract() { - let cases: Vec = serde_json::from_str(include_str!( - "../../../tests/fixtures/usage_accounting/conformance.json" - )) - .unwrap(); - let dimensions = [ - "input_tokens", - "cached_input_tokens", - "cache_write_input_tokens", - "output_tokens", - "total_tokens", - ]; - for case in cases { - let result = normalize(case["usage"].clone(), case["accounting"].clone()); - for (field, expected) in case["expected"].as_object().unwrap() { - if dimensions.contains(&field.as_str()) { - assert_eq!( - result[field], expected["complete"], - "{} {field}", - case["name"] - ); - assert_eq!( - result["observed"][field], expected["observed"], - "{} {field}", - case["name"] - ); - } - } - assert!(result.get("accounting").is_none()); - assert!(result.get("reasoning_tokens").is_none()); - assert!(result.get("peak_request_input_tokens").is_none()); - assert!(result["input_tokens"].is_number() || result["input_tokens"].is_null()); - } - } - - #[test] - fn legacy_usage_does_not_gain_a_completeness_guarantee() { - let usage: AgentUsage = serde_json::from_value(json!({"input_tokens": 100})).unwrap(); - assert!(normalize_model_usage(&usage).unwrap().is_none()); - } - - #[test] - fn normalized_total_does_not_replace_the_independent_raw_provider_total() { - let mut usage: AgentUsage = serde_json::from_value(json!({ - "input_tokens": 100, "input_tokens_include_cache": true, - "output_tokens": 5, "total_tokens": 999, - "extensions": {"nemo_fabric.detailed_token_usage.v1": {"output_tokens": 5, "output_tokens_include_reasoning": true}} - })).unwrap(); - usage.extensions.insert( - "nemo_fabric.usage_accounting.v1".into(), - json!({"models_exhaustive": false, "gaps": []}), - ); - let normalized = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(normalized.tokens.total_tokens, Some(105)); - assert_eq!(usage.total_tokens, Some(999)); - } - - #[test] - fn overflow_does_not_erase_independent_valid_counts() { - let result = normalize( - json!({}), - json!({"models_exhaustive": true, "gaps": [], "models": [{ - "model": "x", "gaps": [], "reported": {"input_tokens": u64::MAX, "input_tokens_include_cache": true, - "output_tokens": 1, "output_tokens_include_reasoning": true}}]}), - ); - assert_eq!(result["input_tokens"], json!(u64::MAX)); - assert_eq!(result["output_tokens"], 1); - assert!(result["total_tokens"].is_null()); - } - - fn cost_with_requests( - aggregate: f64, - requests: &[Option], - exhaustive: bool, - ) -> AgentUsage { - let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, - "output_tokens": 5, "output_tokens_include_reasoning": true, "cost_usd": aggregate}); - let mut usage = AgentUsage { - input_tokens: Some(100), - input_tokens_include_cache: Some(true), - output_tokens: Some(5), - cost_usd: Some(aggregate), - ..Default::default() - }; - usage - .extensions - .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); - usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ - "models_exhaustive": true, "gaps": [], "models": [{ - "provider": "test", "model": "test-model", "gaps": [], "reported": counts, - "requests_exhaustive": exhaustive, - "requests": requests.iter().map(|cost| json!({"usage": {"cost_usd": cost}})).collect::>() - }] - })); - usage - } - - #[test] - fn contradictory_request_reported_costs_are_rejected() { - for (requests, exhaustive) in [ - (vec![Some(0.2)], true), - (vec![Some(0.2)], false), - (vec![Some(0.04)], true), - (vec![Some(10_000_000_000.0); 2], true), - ] { - let usage = cost_with_requests(0.1, &requests, exhaustive); - assert!( - normalize_model_usage(&usage).is_err(), - "contradictory cost must not become complete: {requests:?}, exhaustive={exhaustive}" - ); - } - } - - #[test] - fn independently_rounded_request_costs_keep_fractional_nanos_and_large_values() { - for (aggregate, requests) in [ - (1.6e-9, vec![Some(0.4e-9); 4]), - ( - 200_000_000.72337076, - vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], - ), - ] { - let usage = cost_with_requests(aggregate, &requests, true); - let result = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(result.tokens.input_tokens, Some(100)); - assert_eq!(result.tokens.total_tokens, Some(105)); - assert!(result.cost.reported.amount_nano_usd.is_some()); - } - } - - #[test] - fn partial_or_missing_request_costs_preserve_independent_complete_dimensions() { - for (requests, exhaustive) in [ - (vec![Some(0.04)], false), - (vec![Some(0.04), None], true), - (vec![None], true), - (vec![Some(0.0)], false), - ] { - let usage = cost_with_requests(0.1, &requests, exhaustive); - let result = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(result.tokens.input_tokens, Some(100)); - assert_eq!(result.tokens.output_tokens, Some(5)); - assert_eq!(result.tokens.total_tokens, Some(105)); - assert_eq!(result.cost.reported.amount_nano_usd, Some(100_000_000)); - } - } - - fn omit_model_cost(usage: &mut AgentUsage) { - usage - .extensions - .get_mut("nemo_fabric.usage_accounting.v1") - .unwrap()["models"][0]["reported"] - .as_object_mut() - .unwrap() - .remove("cost_usd"); - } - - #[test] - fn request_costs_without_model_cost_cannot_certify_a_contradictory_root() { - let mut usage = cost_with_requests(0.1, &[Some(0.2)], true); - omit_model_cost(&mut usage); - let result = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(result.cost.reported.amount_nano_usd, None); - assert_eq!(result.cost.reported.observed_nano_usd, None); - assert_eq!(result.tokens.total_tokens, Some(105)); - } - - #[test] - fn exhaustive_requests_recover_cost_without_losing_legitimate_root_rounding() { - for (aggregate, requests) in [ - (0.1, vec![Some(0.04), Some(0.06)]), - ( - 200_000_000.72337076, - vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], - ), - ] { - let mut usage = cost_with_requests(aggregate, &requests, true); - omit_model_cost(&mut usage); - let with_root = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!( - with_root.cost.reported.amount_nano_usd, - Some((aggregate * 1e9).round() as u64) - ); - assert_eq!(with_root.tokens.total_tokens, Some(105)); - usage.cost_usd = None; - let recovered = normalize_model_usage(&usage).unwrap().unwrap(); - assert!(recovered.cost.reported.amount_nano_usd.is_some()); - assert_eq!( - recovered.cost.reported.amount_nano_usd, - recovered.models[0].cost.reported.amount_nano_usd - ); - } - } - - #[test] - fn incomplete_requests_recover_only_observed_cost_when_no_complete_cost_exists() { - for (requests, exhaustive) in [(vec![Some(0.04)], false), (vec![Some(0.04), None], true)] { - let mut usage = cost_with_requests(0.1, &requests, exhaustive); - omit_model_cost(&mut usage); - usage.cost_usd = None; - let result = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(result.cost.reported.amount_nano_usd, None); - assert_eq!(result.cost.reported.observed_nano_usd, Some(40_000_000)); - assert_eq!(result.tokens.total_tokens, Some(105)); - } - } - - #[test] - fn broad_request_gap_cannot_certify_complete_model_or_invocation_counts() { - let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, - "output_tokens": 5, "output_tokens_include_reasoning": true}); - for gap in ["unfinished_work", "missing_usage", "scope_unverified"] { - for root_complete in [false, true] { - let mut usage = AgentUsage::default(); - if root_complete { - usage.input_tokens = Some(100); - usage.input_tokens_include_cache = Some(true); - usage.output_tokens = Some(5); - usage - .extensions - .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); - } - usage.extensions.insert( - "nemo_fabric.usage_accounting.v1".into(), - json!({ - "models_exhaustive": true, "gaps": [], "models": [{ - "model": "model", "gaps": [], - "reported": if root_complete { json!({}) } else { counts.clone() }, - "requests_exhaustive": true, - "requests": [{"usage": {}, "gap": gap}] - }] - }), - ); - assert!( - normalize_model_usage(&usage).is_err(), - "broad request gap must conflict with complete counts: {gap}, root={root_complete}" - ); - } - } - } - - #[test] - fn broad_request_gap_keeps_observations_when_no_complete_value_is_claimed() { - let observed = json!({"input_tokens": 100, "input_tokens_include_cache": true, - "output_tokens": 5, "output_tokens_include_reasoning": true}); - let result = normalize( - json!({}), - json!({ - "models_exhaustive": false, "gaps": [], "models": [{ - "model": "model", "gaps": [], "observed": observed, - "observed_tokens_aligned": true, "requests_exhaustive": true, - "requests": [{"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}] - }] - }), - ); - assert!(result["input_tokens"].is_null()); - assert!(result["output_tokens"].is_null()); - assert_eq!(result["observed"]["input_tokens"], 100); - assert_eq!(result["observed"]["output_tokens"], 5); - for scope in [&result, &result["models"][0]] { - for dimension in [ - "input_tokens", - "cached_input_tokens", - "cache_write_input_tokens", - "output_tokens", - "total_tokens", - "reported_cost", - ] { - assert!( - scope["gaps"].as_array().unwrap().iter().any(|gap| { - gap["dimensions"] == json!([dimension]) - && gap["reason"] == "unfinished_work" - }), - "request reason missing for {dimension}: {}", - scope["gaps"] - ); - } - } - - let mixed = normalize( - json!({}), - json!({ - "models_exhaustive": true, "gaps": [], "models": [{ - "model": "pending", "gaps": [{"dimensions": ["output_tokens"], "reason": "missing_usage"}], - "observed": observed, "observed_tokens_aligned": true, - "requests_exhaustive": true, - "requests": [ - {"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}, - {"usage": {}, "gap": "scope_unverified"}, - {"usage": {}, "gap": "unfinished_work"} - ] - }, {"model": "settled", "reported": observed, "gaps": []}] - }), - ); - assert!(mixed["input_tokens"].is_null()); - assert_eq!(mixed["observed"]["input_tokens"], 200); - assert_eq!(mixed["observed"]["output_tokens"], 10); - assert_eq!(mixed["models"][1]["total_tokens"], 105); - assert!( - !mixed["models"][1]["gaps"] - .as_array() - .unwrap() - .iter() - .any(|gap| { - gap["reason"] == "unfinished_work" || gap["reason"] == "scope_unverified" - }) - ); - for scope in [&mixed, &mixed["models"][0]] { - let gaps = scope["gaps"].as_array().unwrap(); - for reason in ["unfinished_work", "scope_unverified"] { - assert_eq!( - gaps.iter() - .filter(|gap| { - gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == reason - }) - .count(), - 1, - "each request reason must survive once: {gaps:?}" - ); - } - } - assert!( - mixed["models"][0]["gaps"] - .as_array() - .unwrap() - .iter() - .any(|gap| { - gap["dimensions"] == json!(["output_tokens"]) - && gap["reason"] == "missing_usage" - }) - ); - } - - #[test] - fn dimension_scoped_request_omission_keeps_independent_complete_output() { - let result = normalize( - json!({}), - json!({ - "models_exhaustive": true, "gaps": [], "models": [{ - "model": "model", "gaps": [{"dimensions": ["input_tokens"], "reason": "missing_usage"}], - "reported": {"output_tokens": 5, "output_tokens_include_reasoning": true}, - "requests_exhaustive": true, - "requests": [{"usage": {"output_tokens": 5, "output_tokens_include_reasoning": true}, "tokens_aligned": true}] - }] - }), - ); - assert!(result["input_tokens"].is_null()); - assert_eq!(result["output_tokens"], 5); - assert!(result["total_tokens"].is_null()); - } -} diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs new file mode 100644 index 000000000..f457829f1 --- /dev/null +++ b/crates/fabric-core/src/price_catalog.rs @@ -0,0 +1,1128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Model price catalogs for estimating token cost. +//! +//! A catalog uses the NeMo Relay model pricing catalog format, version 1, so +//! one file can configure both tools. NeMo Fabric parses the format itself and +//! does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. +//! An explicit provider restricts aliases and model-path suffixes to that provider. +//! Without a provider, model-only lookup uses the first matching entry in listed order. +//! +//! NeMo Fabric ships one catalog, embedded at build time, with the list prices +//! of models that its presets, docs, and examples use. Each entry cites the +//! provider pricing page it was taken from and the date it was checked. A +//! caller can add a catalog of their own, whose entries take precedence. + +use std::collections::HashSet; +use std::fmt::Write as _; +use std::path::PathBuf; +use std::sync::LazyLock; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// The only catalog format version NeMo Fabric reads. +const PRICING_CATALOG_VERSION: u32 = 1; + +/// Rates must be below one million US dollars per million tokens. NeMo Relay +/// sets no upper bound; this one keeps every rate exact in nano-USD per +/// million tokens and every cost sum within 128 bits. +const MAX_RATE_PER_MILLION: f64 = 1_000_000.0; + +/// Nano-USD per US dollar. +const NANO_USD_PER_USD: f64 = 1e9; + +/// The catalog NeMo Fabric ships. +const BUNDLED_CATALOG_JSON: &str = include_str!("../data/model-pricing.json"); + +static BUNDLED_CATALOG: LazyLock = LazyLock::new(|| BundledCatalog { + // A unit test checks that the shipped file is valid, so `None` only + // guards against a build that skipped the tests. + catalog: PricingCatalog::from_json(BUNDLED_CATALOG_JSON.as_bytes()).ok(), + sha256: sha256_fingerprint(BUNDLED_CATALOG_JSON.as_bytes()), +}); + +struct BundledCatalog { + catalog: Option, + sha256: String, +} + +/// A model price catalog in the NeMo Relay model pricing catalog format, +/// version 1. +/// +/// NeMo Fabric reads and writes the format unchanged, with one extra check: +/// every rate must be below one million US dollars per million tokens. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PricingCatalog { + /// Catalog format version. Must be `1`. + pub version: u32, + /// Price entries. Lookup uses the first entry that matches. + pub entries: Vec, +} + +/// Prices of one model, in `currency` per million tokens. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ModelPricing { + /// Provider that sets the price, such as `anthropic` or `openai`. + pub provider: String, + /// Canonical model identifier. + pub model_id: String, + /// Other identifiers of the same model, such as dated snapshots. + #[serde(default)] + pub aliases: Vec, + /// ISO 4217 currency of the rates. NeMo Fabric estimates cost only from + /// `USD` entries. + #[serde(default = "default_currency")] + pub currency: String, + /// What a rate is charged per. NeMo Fabric estimates cost only from + /// `per_token` entries. + #[serde(default)] + pub unit: PricingUnit, + /// Flat rates. Ignored when `rate_schedule` is set. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rates: Option, + /// Rates that depend on the number of prompt tokens in a request. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate_schedule: Option, + /// How the provider counts cached prompt tokens. NeMo Relay uses it to + /// split provider usage; NeMo Fabric prices usage that is already + /// normalized, so it does not read it. + pub prompt_cache: PromptCachePricing, + /// Date the price was taken from `pricing_source`, as `YYYY-MM-DD` in + /// the bundled catalog. + pub pricing_as_of: String, + /// Where the price was taken from, such as the provider's pricing page. + pub pricing_source: String, +} + +/// What a model price is charged per. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum PricingUnit { + /// Per token. + #[default] + PerToken, + /// Per request. + PerRequest, + /// Per second. + PerSecond, + /// Per GPU hour, for self-hosted models. + GpuHour, +} + +/// Token rates in currency units per million tokens. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TokenPricingRates { + /// Rate for input tokens that were neither read from nor written to the + /// prompt cache. + pub input_per_million: f64, + /// Rate for output tokens, including reasoning tokens. + pub output_per_million: f64, + /// Rate for input tokens read from the prompt cache. Without it, cached + /// reads cannot be priced; set `0.0` when they are free. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_per_million: Option, + /// Rate for input tokens written to the prompt cache. Without it, cache + /// writes cannot be priced; set `0.0` when they are free. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_write_per_million: Option, +} + +/// Rates that depend on the size of a request. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, tag = "type", rename_all = "snake_case")] +pub enum TokenRateSchedule { + /// One tier's rates apply to a whole request, chosen by the request's + /// prompt tokens. + PromptTokenThreshold { + /// How the chosen tier's rates apply. + #[serde(default)] + applies_to: RateScheduleApplication, + /// Tiers in the order they are tried. The first tier whose bounds + /// contain a request's prompt tokens applies to it. + tiers: Vec, + }, +} + +/// How a chosen rate tier applies to a request. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RateScheduleApplication { + /// The tier's rates apply to every token of the request. + #[default] + FullRequest, +} + +/// Rates for requests whose prompt tokens fall within inclusive bounds. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TokenRateTier { + /// Smallest number of prompt tokens in the tier, or no lower bound. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min_prompt_tokens: Option, + /// Largest number of prompt tokens in the tier, or no upper bound. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_prompt_tokens: Option, + /// Rates of the tier. + pub rates: TokenPricingRates, +} + +/// How a provider counts cached prompt tokens in its usage reports. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PromptCachePricing { + /// Whether the provider's prompt token count includes cached reads. + pub read_accounting: CacheReadAccounting, +} + +/// Whether a provider's prompt token count includes cached reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheReadAccounting { + /// Prompt tokens include cached reads. + IncludedInPromptTokens, + /// Prompt tokens exclude cached reads. + Separate, +} + +/// A price catalog the caller supplied, read and validated when the run plan +/// is resolved. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct UserPriceCatalog { + /// Absolute path the catalog was read from. + pub path: PathBuf, + /// `sha256:` followed by the lowercase hexadecimal SHA-256 of the file. + pub sha256: String, + /// The validated catalog. + pub catalog: PricingCatalog, +} + +/// Why a catalog is invalid. The messages match NeMo Relay's. +#[derive(Debug, thiserror::Error)] +pub enum PricingCatalogError { + /// The catalog is not JSON of the catalog format. + #[error("invalid model pricing catalog JSON: {0}")] + Json(#[from] serde_json::Error), + /// The catalog format version is not 1. + #[error("unsupported model pricing catalog version {version}")] + UnsupportedVersion { + /// Version in the catalog. + version: u32, + }, + /// Two entries or aliases of one provider name the same model. + #[error("duplicate model pricing alias '{model}'")] + DuplicateModelAlias { + /// Normalized `provider/model` key. + model: String, + }, + /// A required text field is blank. + #[error("model pricing entry {entry_index} has empty {field}")] + EmptyField { + /// Zero-based entry index. + entry_index: usize, + /// Field name. + field: String, + }, + /// A rate or a tier bound is out of range. + #[error("model pricing entry {entry_index} has invalid {field}: {value}")] + InvalidRate { + /// Zero-based entry index. + entry_index: usize, + /// Field name. + field: String, + /// Invalid value. + value: f64, + }, +} + +impl UserPriceCatalog { + /// Validate bytes and retain their exact catalog fingerprint. + pub fn from_bytes(path: PathBuf, bytes: &[u8]) -> Result { + Ok(Self { + path, + sha256: sha256_fingerprint(bytes), + catalog: PricingCatalog::from_json(bytes)?, + }) + } +} + +impl PricingCatalog { + /// Parse and validate a catalog. + pub fn from_json(json: &[u8]) -> Result { + let catalog: Self = serde_json::from_slice(json)?; + catalog.validate()?; + Ok(catalog) + } + + /// Find `model`, keeping an explicit provider on every alias or suffix. + /// Without a provider, try the model path's provider prefix before model-only + /// suffixes. The first matching entry in listed order wins for each key. + pub(crate) fn pricing_for(&self, provider: Option<&str>, model: &str) -> Option<&ModelPricing> { + lookup_keys(provider, model) + .iter() + .find_map(|key| self.entries.iter().find(|entry| entry.matches(key))) + } + + fn validate(&self) -> Result<(), PricingCatalogError> { + if self.version != PRICING_CATALOG_VERSION { + return Err(PricingCatalogError::UnsupportedVersion { + version: self.version, + }); + } + let mut seen = HashSet::new(); + for (entry_index, entry) in self.entries.iter().enumerate() { + entry.validate(entry_index)?; + let provider = normalized_provider(&entry.provider); + for model in entry.model_keys() { + let key = format!("{provider}/{model}"); + if !seen.insert(key.clone()) { + return Err(PricingCatalogError::DuplicateModelAlias { model: key }); + } + } + } + Ok(()) + } +} + +impl ModelPricing { + pub(crate) fn rates_at(&self, prompt_tokens: u64) -> Option { + match &self.rate_schedule { + None => self.rates, + Some(TokenRateSchedule::PromptTokenThreshold { tiers, .. }) => tiers + .iter() + .find(|tier| { + tier.min_prompt_tokens.unwrap_or(0) <= prompt_tokens + && tier + .max_prompt_tokens + .is_none_or(|max| prompt_tokens <= max) + }) + .map(|tier| tier.rates), + } + } + /// Rates that apply to every request of a model whose largest request + /// had `peak_prompt_tokens` prompt tokens. `None` when the entry has a + /// rate schedule and the peak is unknown or no single tier covers every + /// request up to it. + pub(crate) fn rates_up_to(&self, peak_prompt_tokens: Option) -> Option { + match &self.rate_schedule { + Some(schedule) => schedule.rates_up_to(peak_prompt_tokens?), + None => self.rates, + } + } + + fn matches(&self, key: &LookupKey) -> bool { + key.provider + .as_deref() + .is_none_or(|provider| normalized_provider(&self.provider) == provider) + && self.model_keys().any(|model| model == key.model) + } + + /// The normalized model identifier and aliases, without blank ones. + fn model_keys(&self) -> impl Iterator + '_ { + std::iter::once(&self.model_id) + .chain(&self.aliases) + .map(|model| normalized_model(model)) + .filter(|model| !model.is_empty()) + } + + fn validate(&self, entry_index: usize) -> Result<(), PricingCatalogError> { + for (field, value) in [ + ("provider", &self.provider), + ("model_id", &self.model_id), + ("currency", &self.currency), + ("pricing_as_of", &self.pricing_as_of), + ("pricing_source", &self.pricing_source), + ] { + if value.trim().is_empty() { + return Err(empty_field(entry_index, field)); + } + } + if self.unit == PricingUnit::PerToken + && self.rates.is_none() + && self.rate_schedule.is_none() + { + return Err(empty_field(entry_index, "rates or rate_schedule")); + } + if let Some(rates) = &self.rates { + rates.validate(entry_index, "rates")?; + } + if let Some(schedule) = &self.rate_schedule { + schedule.validate(entry_index)?; + } + Ok(()) + } +} + +impl TokenPricingRates { + fn validate(&self, entry_index: usize, prefix: &str) -> Result<(), PricingCatalogError> { + for (field, value) in [ + ("input_per_million", Some(self.input_per_million)), + ("output_per_million", Some(self.output_per_million)), + ("cache_read_per_million", self.cache_read_per_million), + ("cache_write_per_million", self.cache_write_per_million), + ] { + if let Some(value) = value + && nano_usd_per_million_tokens(value).is_none() + { + return Err(PricingCatalogError::InvalidRate { + entry_index, + field: format!("{prefix}.{field}"), + value, + }); + } + } + Ok(()) + } +} + +impl TokenRateSchedule { + fn rates_up_to(&self, peak_prompt_tokens: u64) -> Option { + match self { + // NeMo Relay prices each request with the first tier whose bounds + // contain its prompt tokens. Every request had between zero and + // the peak prompt tokens, so one tier's rates apply to all of them + // exactly when the first tier that overlaps that range contains + // all of it. + Self::PromptTokenThreshold { tiers, .. } => { + let tier = tiers + .iter() + .find(|tier| tier.overlaps_up_to(peak_prompt_tokens))?; + tier.contains_up_to(peak_prompt_tokens) + .then_some(tier.rates) + } + } + } + + fn validate(&self, entry_index: usize) -> Result<(), PricingCatalogError> { + let Self::PromptTokenThreshold { tiers, .. } = self; + if tiers.is_empty() { + return Err(empty_field(entry_index, "rate_schedule.tiers")); + } + for (tier_index, tier) in tiers.iter().enumerate() { + if let (Some(min), Some(max)) = (tier.min_prompt_tokens, tier.max_prompt_tokens) + && min > max + { + return Err(PricingCatalogError::InvalidRate { + entry_index, + field: "rate_schedule.tiers.prompt_tokens".to_string(), + // NeMo Relay reports the lower bound in the same way. + value: min as f64, + }); + } + tier.rates.validate( + entry_index, + &format!("rate_schedule.tiers[{tier_index}].rates"), + )?; + } + Ok(()) + } +} + +impl TokenRateTier { + /// Whether the tier contains any prompt token count from zero to `peak`. + fn overlaps_up_to(&self, peak: u64) -> bool { + self.min_prompt_tokens.unwrap_or(0) <= peak + } + + /// Whether the tier contains every prompt token count from zero to `peak`. + fn contains_up_to(&self, peak: u64) -> bool { + self.min_prompt_tokens.unwrap_or(0) == 0 + && self.max_prompt_tokens.is_none_or(|max| peak <= max) + } +} + +fn empty_field(entry_index: usize, field: &str) -> PricingCatalogError { + PricingCatalogError::EmptyField { + entry_index, + field: field.to_string(), + } +} + +fn normalized_model(model: &str) -> String { + model.trim().to_ascii_lowercase() +} + +fn normalized_provider(provider: &str) -> String { + provider.trim().trim_matches('/').to_ascii_lowercase() +} + +/// A lookup key: a normalized model, optionally qualified by a normalized +/// provider. +#[derive(Debug, PartialEq, Eq)] +struct LookupKey { + provider: Option, + model: String, +} + +/// Prefer exact model keys, preserving an explicitly established provider. +fn lookup_keys(provider: Option<&str>, model: &str) -> Vec { + let model = normalized_model(model); + if model.is_empty() { + return Vec::new(); + } + let parts: Vec<&str> = model + .split('/') + .map(str::trim) + .filter(|part| !part.is_empty()) + .collect(); + let last = parts.last().copied().unwrap_or(model.as_str()); + let mut keys = Vec::new(); + let mut push = |provider: Option, model: String| { + let key = LookupKey { provider, model }; + if !key.model.is_empty() && !keys.contains(&key) { + keys.push(key); + } + }; + let explicit_provider = provider + .map(normalized_provider) + .filter(|provider| !provider.is_empty()); + match &explicit_provider { + Some(provider) => { + push(Some(provider.clone()), model.clone()); + push(Some(provider.clone()), last.to_string()); + } + None if parts.len() > 1 => { + push(Some(parts[..parts.len() - 1].join("/")), last.to_string()); + } + None => {} + } + for start in 0..parts.len() { + push(explicit_provider.clone(), parts[start..].join("/")); + } + keys +} + +/// Where a price came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PriceOrigin { + /// The catalog NeMo Fabric ships. + Bundled, + /// The catalog the caller supplied. + User, +} + +/// The price catalogs an estimate consults, the caller's first. +pub(crate) struct PriceCatalogs<'a> { + bundled: Option<&'a PricingCatalog>, + bundled_sha256: &'a str, + user: Option<&'a UserPriceCatalog>, +} + +impl<'a> PriceCatalogs<'a> { + /// The shipped catalog, preceded by the caller's catalog when given. + pub(crate) fn with_user_catalog(user: Option<&'a UserPriceCatalog>) -> Self { + Self::from_parts( + BUNDLED_CATALOG.catalog.as_ref(), + &BUNDLED_CATALOG.sha256, + user, + ) + } + + pub(crate) fn from_parts( + bundled: Option<&'a PricingCatalog>, + bundled_sha256: &'a str, + user: Option<&'a UserPriceCatalog>, + ) -> Self { + Self { + bundled, + bundled_sha256, + user, + } + } + + /// Fingerprint of the shipped catalog. + pub(crate) fn bundled_sha256(&self) -> &'a str { + self.bundled_sha256 + } + + /// Fingerprint of the caller's catalog, when one is configured. + pub(crate) fn user_sha256(&self) -> Option<&'a str> { + self.user.map(|user| user.sha256.as_str()) + } + + /// Whether the shipped catalog parsed. + pub(crate) fn bundled_is_valid(&self) -> bool { + self.bundled.is_some() + } + + /// The entry for a model, from the caller's catalog first. With + /// `user_only`, the shipped catalog is not consulted. + pub(crate) fn find( + &self, + provider: Option<&str>, + model: &str, + user_only: bool, + ) -> Option<(&'a ModelPricing, PriceOrigin)> { + let user = self + .user + .and_then(|user| user.catalog.pricing_for(provider, model)) + .map(|entry| (entry, PriceOrigin::User)); + if user.is_some() || user_only { + return user; + } + self.bundled + .and_then(|bundled| bundled.pricing_for(provider, model)) + .map(|entry| (entry, PriceOrigin::Bundled)) + } +} + +/// Nano-USD per million tokens of a rate in US dollars per million tokens, +/// rounded to the nearest whole number. `None` for a rate no valid catalog +/// holds: negative, not finite, or not below one million. +pub(crate) fn nano_usd_per_million_tokens(rate: f64) -> Option { + (rate.is_finite() && (0.0..MAX_RATE_PER_MILLION).contains(&rate)) + // In range, so the rounded value is a whole number below 10^15 and + // the conversion is exact. + .then(|| (rate * NANO_USD_PER_USD).round() as u128) +} + +/// `sha256:` followed by the lowercase hexadecimal SHA-256 of `bytes`. +pub(crate) fn sha256_fingerprint(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut text = String::with_capacity("sha256:".len() + digest.len() * 2); + text.push_str("sha256:"); + for byte in digest.iter() { + // Writing to a String cannot fail. + let _ = write!(text, "{byte:02x}"); + } + text +} + +fn default_currency() -> String { + "USD".to_string() +} + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + + use super::*; + + fn catalog(entries: Value) -> PricingCatalog { + PricingCatalog::from_json( + json!({"version": 1, "entries": entries}) + .to_string() + .as_bytes(), + ) + .expect("valid catalog") + } + + fn catalog_error(entries: Value) -> String { + PricingCatalog::from_json( + json!({"version": 1, "entries": entries}) + .to_string() + .as_bytes(), + ) + .expect_err("invalid catalog") + .to_string() + } + + fn flat_entry(provider: &str, model_id: &str, input: f64, output: f64) -> Value { + json!({ + "provider": provider, + "model_id": model_id, + "pricing_as_of": "2026-06-04", + "pricing_source": format!("https://example.test/{provider}"), + "rates": {"input_per_million": input, "output_per_million": output}, + "prompt_cache": {"read_accounting": "separate"} + }) + } + + fn threshold_entry(tiers: Value) -> Value { + json!({ + "provider": "threshold-ai", + "model_id": "threshold-model", + "pricing_as_of": "2026-06-05", + "pricing_source": "https://example.test/pricing", + "rate_schedule": {"type": "prompt_token_threshold", "tiers": tiers}, + "prompt_cache": {"read_accounting": "separate"} + }) + } + + fn rates(input: f64) -> Value { + json!({"input_per_million": input, "output_per_million": input * 2.0}) + } + + fn found<'a>( + catalog: &'a PricingCatalog, + provider: Option<&str>, + model: &str, + ) -> (&'a str, &'a str) { + let entry = catalog + .pricing_for(provider, model) + .unwrap_or_else(|| panic!("no entry for {provider:?} {model}")); + (entry.provider.as_str(), entry.model_id.as_str()) + } + + // The shipped catalog. + + #[test] + fn bundled_catalog_is_valid_and_cites_a_dated_provider_page_for_every_entry() { + let catalog = PricingCatalog::from_json(BUNDLED_CATALOG_JSON.as_bytes()) + .expect("the shipped catalog is valid"); + + assert!(!catalog.entries.is_empty()); + for entry in &catalog.entries { + let name = format!("{}/{}", entry.provider, entry.model_id); + assert_eq!(entry.currency, "USD", "{name}"); + assert_eq!(entry.unit, PricingUnit::PerToken, "{name}"); + assert!( + entry.pricing_source.starts_with("https://"), + "{name}: {}", + entry.pricing_source + ); + assert!( + is_iso_date(&entry.pricing_as_of), + "{name}: {}", + entry.pricing_as_of + ); + } + assert!(BUNDLED_CATALOG.catalog.is_some()); + } + + /// `YYYY-MM-DD` with a plausible month and day. The test never compares + /// the date with today's, so an old date cannot fail CI. + fn is_iso_date(text: &str) -> bool { + let parts: Vec<&str> = text.split('-').collect(); + let [year, month, day] = parts.as_slice() else { + return false; + }; + let number = |part: &str, digits: usize| { + (part.len() == digits && part.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| part.parse::().ok()) + .flatten() + }; + matches!( + (number(year, 4), number(month, 2), number(day, 2)), + (Some(_), Some(1..=12), Some(1..=31)) + ) + } + + #[test] + fn bundled_catalog_keeps_native_provider_boundaries_for_routed_models() { + let catalog = BUNDLED_CATALOG.catalog.as_ref().expect("bundled catalog"); + assert!( + catalog + .pricing_for(Some("nvidia"), "aws/anthropic/claude-opus-4-5") + .is_none() + ); + assert!( + catalog + .pricing_for(Some("nvidia"), "azure/openai/gpt-5.4") + .is_none() + ); + assert_eq!( + found(catalog, Some("anthropic"), "aws/anthropic/claude-opus-4-5"), + ("anthropic", "claude-opus-4-5") + ); + assert_eq!( + found(catalog, Some("openai"), "azure/openai/gpt-5.4"), + ("openai", "gpt-5.4") + ); + assert_eq!( + found(catalog, Some("anthropic"), "claude-sonnet-4-5-20250929"), + ("anthropic", "claude-sonnet-4-5") + ); + } + + #[test] + fn catalog_fingerprint_is_the_sha256_of_the_file_bytes() { + assert_eq!( + sha256_fingerprint(b"abc"), + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + let fingerprint = PriceCatalogs::with_user_catalog(None).bundled_sha256(); + assert_eq!( + fingerprint, + sha256_fingerprint(BUNDLED_CATALOG_JSON.as_bytes()) + ); + assert_eq!(fingerprint.len(), "sha256:".len() + 64); + } + + // Lookup, mirroring NeMo Relay's model pricing tests. + + #[test] + fn lookup_matches_aliases_case_insensitively() { + let catalog = catalog(json!([{ + "provider": "configured", + "model_id": "configured-model", + "aliases": ["configured-model-2026-06-04"], + "pricing_as_of": "2026-06-04", + "pricing_source": "file:///tmp/pricing.json", + "rates": {"input_per_million": 0.15, "output_per_million": 0.60}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"} + }])); + + let entry = catalog + .pricing_for(None, "CONFIGURED-MODEL-2026-06-04") + .expect("alias"); + + assert_eq!(entry.model_id, "configured-model"); + assert_eq!(entry.currency, "USD"); + assert_eq!(entry.unit, PricingUnit::PerToken); + } + + #[test] + fn lookup_matches_routed_model_names_by_suffix() { + let catalog = catalog(json!([flat_entry("openai", "gpt-4o-mini", 0.15, 0.6)])); + + assert_eq!( + found(&catalog, None, "azure/openai/gpt-4o-mini"), + ("openai", "gpt-4o-mini") + ); + assert_eq!( + found(&catalog, None, "openai/openai/gpt-4o-mini"), + ("openai", "gpt-4o-mini") + ); + } + + #[test] + fn lookup_prefers_the_exact_routed_model_before_a_suffix() { + let catalog = catalog(json!([ + flat_entry("openai", "gpt-4o-mini", 0.15, 0.6), + flat_entry("azure-openai", "azure/openai/gpt-4o-mini", 1.0, 2.0) + ])); + + assert_eq!( + found(&catalog, None, "azure/openai/gpt-4o-mini"), + ("azure-openai", "azure/openai/gpt-4o-mini") + ); + } + + #[test] + fn lookup_tells_apart_one_model_id_of_two_providers() { + let catalog = catalog(json!([ + flat_entry("openai", "gpt-4o-mini", 0.15, 0.6), + flat_entry("azure/openai", "gpt-4o-mini", 1.0, 2.0) + ])); + + assert_eq!( + found(&catalog, Some("openai"), "gpt-4o-mini"), + ("openai", "gpt-4o-mini") + ); + assert_eq!( + found(&catalog, Some("azure/openai"), "gpt-4o-mini"), + ("azure/openai", "gpt-4o-mini") + ); + } + + #[test] + fn lookup_normalizes_providers_and_ignores_blank_or_unknown_models() { + let catalog = catalog(json!([{ + "provider": "provider-a", + "model_id": "priced-model", + "aliases": ["alias-model"], + "pricing_as_of": "2026-06-04", + "pricing_source": "test", + "rates": {"input_per_million": 1.0, "output_per_million": 2.0}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"} + }])); + + assert_eq!( + found(&catalog, Some("/provider-a/"), "provider-a/priced-model"), + ("provider-a", "priced-model") + ); + assert_eq!( + found(&catalog, None, " ALIAS-MODEL "), + ("provider-a", "priced-model") + ); + assert!(catalog.pricing_for(Some("other"), "priced-model").is_none()); + assert_eq!( + found(&catalog, Some("provider-a"), "path/alias-model"), + ("provider-a", "priced-model") + ); + assert!(catalog.pricing_for(None, " ").is_none()); + assert!(catalog.pricing_for(None, "missing-model").is_none()); + } + + #[test] + fn caller_catalog_takes_precedence_and_alone_prices_self_hosted_models() { + let bundled = catalog(json!([ + flat_entry("openai", "gpt-4o-mini", 0.15, 0.6), + flat_entry("openai", "gpt-4o", 2.5, 10.0) + ])); + let user = UserPriceCatalog { + path: PathBuf::from("/prices.json"), + sha256: "sha256:user".to_string(), + catalog: catalog(json!([flat_entry("openai", "gpt-4o-mini", 1.0, 2.0)])), + }; + let prices = PriceCatalogs::from_parts(Some(&bundled), "sha256:bundled", Some(&user)); + + let origin = |model: &str, user_only: bool| { + prices + .find(Some("openai"), model, user_only) + .map(|(entry, origin)| (entry.provider.as_str(), origin)) + }; + + assert_eq!( + origin("gpt-4o-mini", false), + Some(("openai", PriceOrigin::User)) + ); + assert_eq!( + origin("gpt-4o", false), + Some(("openai", PriceOrigin::Bundled)) + ); + assert_eq!(origin("gpt-4o", true), None); + assert_eq!( + origin("gpt-4o-mini", true), + Some(("openai", PriceOrigin::User)) + ); + assert_eq!(prices.bundled_sha256(), "sha256:bundled"); + assert_eq!(prices.user_sha256(), Some("sha256:user")); + } + + #[test] + fn unrelated_user_provider_cannot_shadow_matching_bundled_price() { + let bundled = catalog(json!([flat_entry("openai", "model", 1.0, 2.0)])); + let user = UserPriceCatalog { + path: PathBuf::from("/prices.json"), + sha256: "sha256:user".to_string(), + catalog: catalog(json!([flat_entry("custom-route", "model", 3.0, 4.0)])), + }; + let prices = PriceCatalogs::from_parts(Some(&bundled), "sha256:bundled", Some(&user)); + let (entry, origin) = prices.find(Some("openai"), "model", false).unwrap(); + assert_eq!(entry.provider, "openai"); + assert_eq!(origin, PriceOrigin::Bundled); + let (entry, origin) = prices.find(Some("custom-route"), "model", true).unwrap(); + assert_eq!(entry.provider, "custom-route"); + assert_eq!(origin, PriceOrigin::User); + } + + // Validation, mirroring NeMo Relay's catalog validation tests. + + #[test] + fn validation_rejects_an_unsupported_version() { + let error = PricingCatalog::from_json(br#"{"version": 2, "entries": []}"#) + .expect_err("version 2") + .to_string(); + + assert!( + error.contains("unsupported model pricing catalog version 2"), + "{error}" + ); + } + + #[test] + fn validation_rejects_duplicate_model_aliases_of_one_provider() { + let mut other = flat_entry("a", "other-model", 1.0, 1.0); + other["aliases"] = json!(["SAME-MODEL"]); + + let error = catalog_error(json!([flat_entry("a", "same-model", 1.0, 1.0), other])); + + assert!( + error.contains("duplicate model pricing alias 'a/same-model'"), + "{error}" + ); + } + + #[test] + fn validation_rejects_blank_required_fields() { + for field in [ + "provider", + "model_id", + "currency", + "pricing_as_of", + "pricing_source", + ] { + let mut entry = flat_entry("configured", "configured-model", 1.0, 2.0); + entry[field] = json!(" "); + + let error = catalog_error(json!([entry])); + + assert!( + error.contains(&format!("empty {field}")), + "{field}: {error}" + ); + } + } + + #[test] + fn validation_rejects_negative_or_too_large_rates() { + for (field, value) in [ + ("input_per_million", -0.1), + ("output_per_million", -0.1), + ("cache_read_per_million", -0.1), + ("cache_write_per_million", -0.1), + ("input_per_million", 1_000_000.0), + ] { + let mut entry = flat_entry("configured", "configured-model", 1.0, 2.0); + entry["rates"][field] = json!(value); + + let error = catalog_error(json!([entry])); + + assert!( + error.contains(&format!("rates.{field}")), + "{field}: {error}" + ); + } + } + + #[test] + fn validation_requires_rates_for_per_token_entries() { + let mut entry = flat_entry("broken", "missing-rates", 1.0, 2.0); + entry.as_object_mut().expect("entry").remove("rates"); + + let error = catalog_error(json!([entry])); + + assert!(error.contains("empty rates or rate_schedule"), "{error}"); + } + + #[test] + fn non_token_units_are_valid_without_rates() { + let mut entry = flat_entry("self-hosted", "nemotron-owned", 1.0, 2.0); + entry.as_object_mut().expect("entry").remove("rates"); + entry["unit"] = json!("gpu_hour"); + + let catalog = catalog(json!([entry])); + + assert_eq!(catalog.entries[0].unit, PricingUnit::GpuHour); + } + + #[test] + fn validation_rejects_invalid_rate_schedules() { + let empty = catalog_error(json!([threshold_entry(json!([]))])); + let reversed = catalog_error(json!([threshold_entry(json!([ + {"min_prompt_tokens": 20, "max_prompt_tokens": 10, "rates": rates(1.0)} + ]))])); + let negative = catalog_error(json!([threshold_entry(json!([{"rates": rates(-1.0)}]))])); + + assert!(empty.contains("rate_schedule.tiers"), "{empty}"); + assert!( + reversed.contains("rate_schedule.tiers.prompt_tokens"), + "{reversed}" + ); + assert!( + negative.contains("rate_schedule.tiers[0].rates.input_per_million"), + "{negative}" + ); + } + + #[test] + fn validation_rejects_unknown_fields_at_every_level() { + let entry = flat_entry("configured", "configured-model", 1.0, 2.0); + let with = |pointer: &str, field: &str| { + let mut catalog = json!({"version": 1, "entries": [entry.clone()]}); + catalog + .pointer_mut(pointer) + .and_then(Value::as_object_mut) + .expect("object") + .insert(field.to_string(), json!(true)); + catalog + }; + let mut tiers = json!({"version": 1, "entries": [threshold_entry(json!([ + {"min_prompt_tokens": 0, "rates": rates(1.0)} + ]))]}); + tiers["entries"][0]["rate_schedule"]["tierz"] = json!([]); + let tier = json!({"version": 1, "entries": [threshold_entry(json!([ + {"min_prompt_toknes": 0, "rates": rates(1.0)} + ]))]}); + + for (catalog, field) in [ + (with("", "unexpected_catalog"), "unexpected_catalog"), + (with("/entries/0", "unexpected_entry"), "unexpected_entry"), + ( + with("/entries/0/rates", "cache_writ_per_million"), + "cache_writ_per_million", + ), + ( + with("/entries/0/prompt_cache", "read_accountng"), + "read_accountng", + ), + (tiers, "tierz"), + (tier, "min_prompt_toknes"), + ] { + let error = PricingCatalog::from_json(catalog.to_string().as_bytes()) + .expect_err("unknown field") + .to_string(); + + assert!( + error.contains(&format!("unknown field `{field}`")), + "{field}: {error}" + ); + } + } + + // Rate tiers. + + #[test] + fn a_rate_schedule_resolves_only_when_one_tier_covers_every_request() { + let catalog = catalog(json!([threshold_entry(json!([ + {"max_prompt_tokens": 200_000, "rates": rates(1.0)}, + {"min_prompt_tokens": 200_001, "rates": rates(10.0)} + ]))])); + let entry = &catalog.entries[0]; + let input = + |peak: Option| entry.rates_up_to(peak).map(|rates| rates.input_per_million); + + assert_eq!(input(Some(0)), Some(1.0)); + assert_eq!(input(Some(200_000)), Some(1.0)); + assert_eq!(input(Some(200_001)), None); + assert_eq!(input(None), None); + } + + #[test] + fn a_rate_schedule_follows_tier_order_like_nemo_relay() { + let catalog = catalog(json!([threshold_entry(json!([ + {"min_prompt_tokens": 100, "rates": rates(10.0)}, + {"max_prompt_tokens": 99, "rates": rates(1.0)} + ]))])); + let gap = catalog_with_first_tier_from(1); + let entry = &catalog.entries[0]; + + assert_eq!( + entry + .rates_up_to(Some(99)) + .map(|rates| rates.input_per_million), + Some(1.0) + ); + assert_eq!(entry.rates_up_to(Some(100)), None); + assert_eq!(gap.entries[0].rates_up_to(Some(10)), None); + } + + fn catalog_with_first_tier_from(min_prompt_tokens: u64) -> PricingCatalog { + catalog(json!([threshold_entry(json!([ + {"min_prompt_tokens": min_prompt_tokens, "rates": rates(1.0)} + ]))])) + } + + #[test] + fn flat_rates_apply_whatever_the_peak() { + let catalog = catalog(json!([flat_entry("openai", "gpt-4o-mini", 0.15, 0.6)])); + + for peak in [None, Some(0), Some(u64::MAX)] { + assert_eq!( + catalog.entries[0] + .rates_up_to(peak) + .map(|rates| rates.input_per_million), + Some(0.15) + ); + } + } + + // Rate conversion. + + #[test] + fn rates_convert_to_whole_nano_usd_per_million_tokens() { + for (rate, expected) in [ + (0.0, Some(0)), + (0.075, Some(75_000_000)), + (0.3, Some(300_000_000)), + (2.5, Some(2_500_000_000)), + (3.75, Some(3_750_000_000)), + (1e-10, Some(0)), + (6e-10, Some(1)), + (999_999.999_999_999, Some(999_999_999_999_999)), + (1_000_000.0, None), + (-0.000_001, None), + (f64::NAN, None), + (f64::INFINITY, None), + ] { + assert_eq!(nano_usd_per_million_tokens(rate), expected, "rate {rate}"); + } + } +} diff --git a/crates/fabric-core/src/pricing.rs b/crates/fabric-core/src/pricing.rs index d8c37bcf9..191c1f965 100644 --- a/crates/fabric-core/src/pricing.rs +++ b/crates/fabric-core/src/pricing.rs @@ -1,11 +1,108 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Persisted pricing-basis types for the stable model-usage result contract. +//! Catalog prices are estimates, separate from provider-reported amounts. use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use crate::measurement::{ + Attribution, Availability, Measurement, MeasurementSource, MeasurementUnit, +}; +use crate::price_catalog::{ + PriceCatalogs, PriceOrigin, PricingUnit, TokenPricingRates, UserPriceCatalog, + nano_usd_per_million_tokens, +}; +use crate::usage_accounting::GapReason; +use crate::usage_normalization::{AccountingDimension, AccountingGap, ModelUsage, ObservedUsage}; + +pub(crate) const OPENAI_ENDPOINT_ENV_KEYS: [&str; 2] = ["OPENAI_BASE_URL", "OPENAI_API_BASE"]; +pub(crate) const ANTHROPIC_ENDPOINT_ENV_KEYS: [&str; 1] = ["ANTHROPIC_BASE_URL"]; + +/// Configured and inherited endpoint overrides must still identify a native +/// endpoint before provider list prices apply. An adapter can inherit URL +/// settings even when models..base_url is absent. +pub(crate) fn native_catalog_route( + model: &crate::ModelConfig, + forwarded: &std::collections::BTreeMap, + inherited: impl Fn(&str) -> Option, +) -> bool { + let (keys, urls): (&[&str], &[&str]) = match model.provider.as_str() { + "openai" => (&OPENAI_ENDPOINT_ENV_KEYS, &["https://api.openai.com/v1"]), + "anthropic" => ( + &ANTHROPIC_ENDPOINT_ENV_KEYS, + &["https://api.anthropic.com", "https://api.anthropic.com/v1"], + ), + _ => return false, + }; + let native = |url: &str| urls.contains(&url.trim_end_matches('/')); + if let Some(url) = &model.base_url { + return native(url); + } + keys.iter().all(|key| { + forwarded + .get(*key) + .cloned() + .or_else(|| inherited(key)) + .is_none_or(|url| native(&url)) + }) +} + +/// Startup route evidence, qualified for the provider/model actually accounted. +/// An unused custom role must not disable a different model's native tariff. +#[derive(Debug, Clone)] +pub(crate) struct NativePricingRoutes { + models: Vec, +} + +#[derive(Debug, Clone)] +struct NativeModelRoute { + provider: String, + model: String, + native: bool, +} + +impl NativePricingRoutes { + pub(crate) fn from_models( + models: &std::collections::BTreeMap, + endpoints: &std::collections::BTreeMap, + ) -> Self { + Self { + models: models + .values() + .map(|model| NativeModelRoute { + provider: model.provider.clone(), + model: model.model.clone(), + native: native_catalog_route(model, endpoints, |_| None), + }) + .collect(), + } + } + + pub(crate) fn allows(&self, provider: Option<&str>, model: &str) -> bool { + let mut exact = self + .models + .iter() + .filter(|route| Some(route.provider.as_str()) == provider && route.model == model) + .peekable(); + if exact.peek().is_some() { + return exact.all(|route| route.native); + } + // A native provider route still supports a served-model alias. Mixed + // routes for that provider cannot establish which endpoint served it. + let mut provider_routes = self + .models + .iter() + .filter(|route| Some(route.provider.as_str()) == provider) + .peekable(); + if provider_routes.peek().is_some() { + return provider_routes.all(|route| route.native); + } + // Preserve the prior all-native case for source-qualified child models. + !self.models.is_empty() && self.models.iter().all(|route| route.native) + } +} + /// Fields that must agree before a report combines catalog estimates. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] @@ -60,3 +157,587 @@ pub struct AppliedPricing { /// One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`. pub rates: RateSnapshot, } + +impl RateSnapshot { + fn from_rates(rates: TokenPricingRates) -> Result { + let convert = |rate| { + nano_usd_per_million_tokens(rate) + .and_then(|value| u64::try_from(value).ok()) + .ok_or("price_catalog_invalid") + }; + Ok(Self { + input: convert(rates.input_per_million)?, + output: convert(rates.output_per_million)?, + cache_read: rates.cache_read_per_million.map(convert).transpose()?, + cache_write: rates.cache_write_per_million.map(convert).transpose()?, + }) + } + + fn price( + &self, + input: Option, + cached: Option, + written: Option, + output: Option, + ) -> Result { + rounded_price(self.charges(input, cached, written, output)?) + } + + fn charges( + &self, + input: Option, + cached: Option, + written: Option, + output: Option, + ) -> Result { + let input = input.ok_or("pricing_input_unreported")?; + let output = output.ok_or("pricing_output_unreported")?; + // A missing cache category cannot affect the price only when it is + // charged exactly like ordinary input, or the entire input is zero. + let component = |count: Option, rate: Option| match count { + Some(count) => match rate { + Some(rate) => Ok((count, rate)), + None if count == 0 => Ok((0, 0)), + None => Err("cache_rate_unreported"), + }, + None if input == 0 || rate == Some(self.input) => Ok((0, self.input)), + None => Err("pricing_cache_unreported"), + }; + let (cached, read_rate) = component(cached, self.cache_read)?; + let (written, write_rate) = component(written, self.cache_write)?; + let uncached = input + .checked_sub(cached) + .and_then(|v| v.checked_sub(written)) + .ok_or("inconsistent_usage")?; + let charges = [ + (uncached, self.input), + (cached, read_rate), + (written, write_rate), + (output, self.output), + ]; + let sum = charges + .into_iter() + .try_fold(0_u128, |sum, (count, rate)| { + sum.checked_add(u128::from(count).checked_mul(u128::from(rate))?) + }) + .ok_or("cost_out_of_range")?; + Ok(sum) + } + + fn observed_charges( + &self, + observed: &ObservedUsage, + aligned: bool, + write_tariff_known: bool, + ) -> Option { + // Categories are disjoint. Without aligned input/cache observations, + // retain independently priceable categories instead of subtracting + // counters from different source-call subsets. + let mut charges = 0_u128; + let mut any = false; + let mut add = |count: Option, rate: Option| -> Option<()> { + if let (Some(count), Some(rate)) = (count, rate) { + charges = charges.checked_add(u128::from(count).checked_mul(u128::from(rate))?)?; + any = true; + } + Some(()) + }; + add(observed.output_tokens, Some(self.output))?; + let equal_rates = + self.cache_read == Some(self.input) && self.cache_write == Some(self.input); + if let Some(input) = observed + .input_tokens + .filter(|_| equal_rates && write_tariff_known) + { + add(Some(input), Some(self.input))?; + } else { + if aligned { + let cached = observed + .cached_input_tokens + .or((self.cache_read == Some(self.input)).then_some(0)); + let written = observed + .cache_write_input_tokens + .or((write_tariff_known && self.cache_write == Some(self.input)).then_some(0)); + if let (Some(input), Some(cached), Some(written)) = + (observed.input_tokens, cached, written) + { + add( + input + .checked_sub(cached) + .and_then(|v| v.checked_sub(written)), + Some(self.input), + )?; + } + } + add(observed.cached_input_tokens, self.cache_read)?; + if write_tariff_known { + add(observed.cache_write_input_tokens, self.cache_write)?; + } + } + any.then_some(charges) + } + + fn observed_price( + &self, + observed: &ObservedUsage, + aligned: bool, + write_tariff_known: bool, + ) -> Option { + rounded_price(self.observed_charges(observed, aligned, write_tariff_known)?).ok() + } +} + +fn rounded_price(charges: u128) -> Result { + u64::try_from(charges.checked_add(500_000).ok_or("cost_out_of_range")? / 1_000_000) + .map_err(|_| "cost_out_of_range") +} + +fn estimate(value: Result, source: MeasurementSource) -> Measurement { + Measurement { + availability: if value.is_ok() { + Availability::Measured + } else { + Availability::Unavailable + }, + value: value.ok(), + unit: MeasurementUnit::NanoUsd, + source, + attribution: if value.is_ok() { + Attribution::Invocation + } else { + Attribution::None + }, + reason: value.err().map(str::to_owned), + } +} + +fn update_price_gaps(gaps: &mut Vec, cost: &Measurement) { + gaps.retain(|gap| gap.dimensions != [AccountingDimension::EstimatedCost]); + if cost.value.is_some() { + return; + } + let mut reasons: std::collections::BTreeSet<_> = gaps + .iter() + .filter(|gap| { + gap.dimensions.iter().any(|dimension| { + matches!( + dimension, + AccountingDimension::InputTokens | AccountingDimension::OutputTokens + ) + }) + }) + .map(|gap| gap.reason) + .collect(); + if reasons.is_empty() { + reasons.insert(match cost.reason.as_deref() { + Some("model_scope_unverified" | "inconsistent_usage" | "provider_price_unverified") => { + GapReason::ScopeUnverified + } + _ => GapReason::MissingUsage, + }); + } + gaps.extend(reasons.into_iter().map(|reason| AccountingGap { + dimensions: vec![AccountingDimension::EstimatedCost], + reason, + })); +} + +/// Price validated normalized usage with a route decision for each accounted model. +/// An unqualified route permits only explicitly supplied prices. +/// +/// Price with validated original request boundaries when an aggregate spans tiers. +/// Keep this evidence explicit: normalized aggregates cannot recover request sizes. +pub(crate) fn apply_pricing_with_evidence( + usage: &mut ModelUsage, + user: Option<&UserPriceCatalog>, + bundled_allowed: impl Fn(Option<&str>, &str) -> bool, + evidence: Option<&crate::usage_accounting::UsageAccounting>, +) { + let prices = PriceCatalogs::with_user_catalog(user); + if !prices.bundled_is_valid() { + usage.token_cost.estimated = estimate( + Err("price_catalog_invalid"), + MeasurementSource::FabricPriceCatalog, + ); + return; + } + let basis = PricingBasis { + basis: "catalog".into(), + currency: "USD".into(), + normalization_version: "cache_reasoning_inclusive_v1".into(), + assumption_policy: "applicable_tariff_no_missing_counts_v1".into(), + bundled_catalog_sha256: prices.bundled_sha256().into(), + user_catalog_sha256: prices.user_sha256().map(str::to_owned), + }; + usage.token_cost.pricing = Some(basis.clone()); + if usage.accounting.is_none() { + usage.token_cost.estimated = estimate( + Err("legacy_unverified"), + MeasurementSource::FabricPriceCatalog, + ); + return; + } + let mut any_user = false; + let mut root_cache_read_required = false; + let mut root_cache_write_required = false; + for model in &mut usage.models { + let cost = &mut model.usage.token_cost; + cost.applied = None; + cost.applied_calls.clear(); + model.accounting.observed.estimated_cost_nano_usd = None; + cost.pricing = Some(basis.clone()); + let selected = prices.find( + model.provider.as_deref(), + &model.model, + !bundled_allowed(model.provider.as_deref(), &model.model), + ); + let Some((entry, origin)) = selected else { + cost.estimated = estimate( + Err("model_not_in_catalog"), + MeasurementSource::FabricPriceCatalog, + ); + update_price_gaps(&mut model.accounting.gaps, &cost.estimated); + continue; + }; + let source = match origin { + PriceOrigin::User => MeasurementSource::UserPriceCatalog, + PriceOrigin::Bundled => MeasurementSource::FabricPriceCatalog, + }; + any_user |= origin == PriceOrigin::User; + if origin == PriceOrigin::Bundled + && !model + .provider + .as_ref() + .is_some_and(|provider| provider.eq_ignore_ascii_case(&entry.provider)) + { + cost.estimated = estimate(Err("provider_price_unverified"), source); + update_price_gaps(&mut model.accounting.gaps, &cost.estimated); + continue; + } + let mut result = (|| { + if entry.unit != PricingUnit::PerToken || !entry.currency.eq_ignore_ascii_case("USD") { + return Err("price_currency_or_unit_unsupported"); + } + // A complete invocation's total input is also a safe upper bound + // on each request; a subset peak never prices a larger complete set. + let complete_peak = model + .usage + .peak_request_input_tokens + .value + .or(model.usage.input_tokens.value); + let complete_rates = entry + .rates_up_to(complete_peak) + .map(RateSnapshot::from_rates) + .transpose()?; + let observed_peak = if model.accounting.observed_tokens_aligned { + model + .accounting + .observed + .peak_request_input_tokens + .or(model.accounting.observed.input_tokens) + } else { + None + }; + let observed_rates = match &complete_rates { + Some(rates) => Some(rates.clone()), + None => entry + .rates_up_to(observed_peak) + .map(RateSnapshot::from_rates) + .transpose()?, + }; + let write_tariff_known = !entry.provider.eq_ignore_ascii_case("anthropic") + || model.usage.cache_write_input_tokens.value == Some(0); + let complete = match &complete_rates { + Some(rates) => { + root_cache_read_required |= rates.cache_read != Some(rates.input); + root_cache_write_required |= rates.cache_write != Some(rates.input); + if !write_tariff_known && model.usage.input_tokens.value != Some(0) { + Err("cache_write_tariff_unverified") + } else { + rates.price( + model.usage.input_tokens.value, + model.usage.cached_input_tokens.value, + model.usage.cache_write_input_tokens.value, + model.usage.output_tokens.value, + ) + } + } + None => Err("tiered_price_unresolvable"), + }; + let observed = complete.ok().or_else(|| { + observed_rates.as_ref().and_then(|rates| { + rates.observed_price( + &model.accounting.observed, + model.accounting.observed_tokens_aligned + || (model.usage.input_tokens.value.is_some() + && model.usage.cached_input_tokens.value.is_some() + && model.usage.cache_write_input_tokens.value.is_some()), + !entry.provider.eq_ignore_ascii_case("anthropic") + || model.accounting.observed.cache_write_input_tokens == Some(0), + ) + }) + }); + model.accounting.observed.estimated_cost_nano_usd = observed; + let rates = if complete.is_ok() { + complete_rates + } else { + observed_rates.or(complete_rates) + }; + let Some(rates) = rates else { + return complete; + }; + cost.applied = Some(AppliedPricing { + catalog_sha256: match origin { + PriceOrigin::Bundled => prices.bundled_sha256(), + PriceOrigin::User => prices.user_sha256().expect("selected user catalog"), + } + .into(), + provider: entry.provider.clone(), + model_id: entry.model_id.clone(), + pricing_as_of: entry.pricing_as_of.clone(), + pricing_source: entry.pricing_source.clone(), + rates, + }); + complete + })(); + if let Some(raw) = evidence + .and_then(|e| { + e.models + .iter() + .find(|raw| raw.model == model.model && raw.provider == model.provider) + }) + .filter(|raw| { + !raw.requests.is_empty() + && entry.unit == PricingUnit::PerToken + && entry.currency.eq_ignore_ascii_case("USD") + }) + { + let mut charges = Some(0_u128); + let mut complete = raw.requests_exhaustive; + let mut any = false; + for request in &raw.requests { + let aligned = request.gap.is_none() || request.tokens_aligned; + let normalized = + crate::usage_normalization::normalized_request(&request.usage, !aligned); + let priced: Result<(u128, RateSnapshot, bool), &'static str> = (|| { + let selected = if entry.rate_schedule.is_some() { + entry.rates_at( + normalized + .input_tokens + .value + .ok_or("pricing_input_unreported")?, + ) + } else { + entry.rates + }; + let rates = + RateSnapshot::from_rates(selected.ok_or("tiered_price_unresolvable")?)?; + let write_tariff_known = !entry.provider.eq_ignore_ascii_case("anthropic") + || normalized.cache_write_input_tokens.value == Some(0) + || normalized.input_tokens.value == Some(0); + let complete_charge = if aligned && write_tariff_known { + rates + .charges( + normalized.input_tokens.value, + normalized.cached_input_tokens.value, + normalized.cache_write_input_tokens.value, + normalized.output_tokens.value, + ) + .ok() + } else { + None + }; + let observed = ObservedUsage { + input_tokens: normalized.input_tokens.value, + cached_input_tokens: normalized.cached_input_tokens.value, + cache_write_input_tokens: normalized.cache_write_input_tokens.value, + output_tokens: normalized.output_tokens.value, + ..Default::default() + }; + let charge = complete_charge + .or_else(|| rates.observed_charges(&observed, aligned, write_tariff_known)) + .ok_or("pricing_usage_unavailable")?; + Ok((charge, rates, complete_charge.is_some())) + })(); + match priced { + Ok((charge, rates, all_parts_priced)) => { + any = true; + charges = charges.and_then(|total| total.checked_add(charge)); + root_cache_read_required |= rates.cache_read != Some(rates.input); + root_cache_write_required |= rates.cache_write != Some(rates.input); + cost.applied_calls.push(Some(AppliedPricing { + catalog_sha256: match origin { + PriceOrigin::Bundled => prices.bundled_sha256(), + PriceOrigin::User => { + prices.user_sha256().expect("selected catalog") + } + } + .into(), + provider: entry.provider.clone(), + model_id: entry.model_id.clone(), + pricing_as_of: entry.pricing_as_of.clone(), + pricing_source: entry.pricing_source.clone(), + rates, + })); + complete &= request.gap.is_none() && all_parts_priced; + } + Err(_) => { + complete = false; + cost.applied_calls.push(None); + } + } + } + let amount = charges + .and_then(|charges| rounded_price(charges).ok()) + .filter(|_| any); + if result.is_err() + && complete + && model.usage.input_tokens.value.is_some() + && model.usage.output_tokens.value.is_some() + { + result = amount.ok_or("cost_out_of_range"); + cost.applied = None; + } + if let Some(amount) = result.as_ref().ok().copied().or(amount) { + model.accounting.observed.estimated_cost_nano_usd = Some(amount); + } + } + cost.estimated = estimate(result, source); + update_price_gaps(&mut model.accounting.gaps, &cost.estimated); + } + let inconsistent = [ + &usage.input_tokens, + &usage.cached_input_tokens, + &usage.cache_write_input_tokens, + &usage.output_tokens, + ] + .iter() + .any(|m| m.reason.as_deref() == Some("inconsistent_usage")); + let accounting = usage.accounting.as_mut().expect("checked accounting"); + let complete = if !accounting.models_exhaustive || usage.models.is_empty() { + Err("model_scope_unverified") + } else if inconsistent { + Err("inconsistent_usage") + } else if usage.models.iter().any(|model| { + model.usage.token_cost.estimated.reason.as_deref() == Some("cache_write_tariff_unverified") + }) { + Err("cache_write_tariff_unverified") + } else if usage.input_tokens.value.is_none() + || usage.output_tokens.value.is_none() + || (root_cache_read_required + && usage.cached_input_tokens.value.is_none() + && usage.input_tokens.value != Some(0)) + || (root_cache_write_required + && usage.cache_write_input_tokens.value.is_none() + && usage.input_tokens.value != Some(0)) + { + Err("pricing_usage_incomplete") + } else { + usage.models.iter().try_fold(0_u64, |sum, model| { + sum.checked_add( + model + .usage + .token_cost + .estimated + .value + .ok_or("unpriced_model")?, + ) + .ok_or("cost_out_of_range") + }) + }; + let observed = if inconsistent { + None + } else { + let values: Vec<_> = usage + .models + .iter() + .filter_map(|model| model.accounting.observed.estimated_cost_nano_usd) + .collect(); + if values.is_empty() { + None + } else { + values.into_iter().try_fold(0_u64, u64::checked_add) + } + }; + accounting.observed.estimated_cost_nano_usd = complete.ok().or(observed); + usage.token_cost.estimated = estimate( + complete, + if any_user { + MeasurementSource::UserPriceCatalog + } else { + MeasurementSource::FabricPriceCatalog + }, + ); + update_price_gaps(&mut accounting.gaps, &usage.token_cost.estimated); +} + +#[cfg(test)] +mod tests { + use super::native_catalog_route; + use serde_json::json; + use std::collections::BTreeMap; + + #[test] + fn native_pricing_routes_preserve_ambiguity_and_served_aliases() { + let mut models = serde_json::from_value(json!({ + "default": {"provider": "openai", "model": "gpt-5.4"}, + "auxiliary": {"provider": "openai", "model": "gpt-5.4", + "base_url": "https://custom.invalid/v1"} + })) + .unwrap(); + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(!routes.allows(Some("openai"), "gpt-5.4")); + assert!(!routes.allows(Some("openai"), "served-alias")); + assert!(!routes.allows(Some("anthropic"), "child-model")); + + models.remove("auxiliary"); + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(routes.allows(Some("openai"), "served-alias")); + assert!(routes.allows(Some("anthropic"), "source-qualified-child")); + + models.get_mut("default").unwrap().base_url = Some("https://custom.invalid/v1".into()); + models.insert( + "unused_native".into(), + serde_json::from_value(json!({ + "provider": "openai", "model": "other-model" + })) + .unwrap(), + ); + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(!routes.allows(Some("openai"), "gpt-5.4")); + assert!(routes.allows(Some("openai"), "other-model")); + assert!(!routes.allows(Some("openai"), "served-alias")); + + let empty = super::NativePricingRoutes::from_models(&BTreeMap::new(), &BTreeMap::new()); + assert!(!empty.allows(Some("openai"), "gpt-5.4")); + } + + #[test] + fn inherited_and_forwarded_custom_routes_require_explicit_prices() { + for (configured, inherited, forwarded, expected) in [ + (None, None, None, true), + ( + Some("https://api.openai.com/v1/"), + Some("https://proxy.invalid/v1"), + None, + true, + ), + (Some("https://proxy.invalid/v1"), None, None, false), + (None, Some("https://proxy.invalid/v1"), None, false), + (None, None, Some("https://proxy.invalid/v1"), false), + (None, Some("https://api.openai.com/v1"), None, true), + ] { + let model = serde_json::from_value( + json!({"provider":"openai","model":"gpt-5.4","base_url":configured}), + ) + .unwrap(); + let env = forwarded + .map(|url| BTreeMap::from([("OPENAI_BASE_URL".into(), url.into())])) + .unwrap_or_default(); + assert_eq!( + native_catalog_route(&model, &env, |_| inherited.map(str::to_owned)), + expected + ); + } + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 2b7a780b4..349bafef5 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -873,6 +873,7 @@ struct LocalAdapterHost { relay_config: Option, // Adapter startup values stay in memory, never in a serialized handle or plan. environment_env: BTreeMap, + bundled_pricing_allowed: crate::pricing::NativePricingRoutes, } struct LocalServiceRecord { @@ -2141,6 +2142,7 @@ fn run_local_host_invocation_with_timeout( relay_config, fabric_home, fabric_invocation, + host_guard.bundled_pricing_allowed.clone(), )) } Err(error) => { @@ -2166,6 +2168,7 @@ fn run_local_host_invocation_with_timeout( relay_config, fabric_home, fabric_invocation, + bundled_pricing_allowed, ) = exchange_result?; let agent_result: AgentRunResult = serde_json::from_value(adapter_output).map_err(|error| { @@ -2324,7 +2327,13 @@ fn run_local_host_invocation_with_timeout( let model_usage = match agent_result .usage .as_ref() - .map(|raw| crate::model_usage::normalize_model_usage(raw)) + .map(|raw| { + crate::model_usage::normalize_model_usage_for_routes( + raw, + plan.user_price_catalog.as_ref(), + |provider, model| bundled_pricing_allowed.allows(provider, model), + ) + }) .transpose() { Ok(usage) => usage.flatten(), @@ -2584,6 +2593,10 @@ fn spawn_local_host( if let Some(root) = artifacts.root.as_ref() { command.env("FABRIC_ARTIFACTS", root); } + let bundled_pricing_allowed = + snapshot_pricing_route(&plan.config.models, &mut command, |name| { + std::env::var_os(name) + }); // Start suspended so the host joins its job before it can start a process. #[cfg(windows)] std::os::windows::process::CommandExt::creation_flags( @@ -2682,9 +2695,46 @@ fn spawn_local_host( artifacts, relay_config, environment_env, + bundled_pricing_allowed, }) } +// Freeze endpoint inheritance on the command itself so pricing describes the +// persistent child's startup route, not a later mutation of the controller's env. +fn snapshot_pricing_route( + models: &BTreeMap, + command: &mut Command, + inherited: impl Fn(&str) -> Option, +) -> crate::pricing::NativePricingRoutes { + let mut endpoints = BTreeMap::new(); + for name in crate::pricing::OPENAI_ENDPOINT_ENV_KEYS + .into_iter() + .chain(crate::pricing::ANTHROPIC_ENDPOINT_ENV_KEYS) + { + let value = command + .get_envs() + .find_map(|(key, value)| { + let matches = if cfg!(windows) { + key.to_string_lossy().eq_ignore_ascii_case(name) + } else { + key == name + }; + matches.then(|| value.map(std::ffi::OsStr::to_owned)) + }) + .unwrap_or_else(|| inherited(name)); + match value { + Some(value) => { + endpoints.insert(name.to_string(), value.to_string_lossy().into_owned()); + command.env(name, value); + } + None => { + command.env_remove(name); + } + } + } + crate::pricing::NativePricingRoutes::from_models(models, &endpoints) +} + fn local_host_command(plan: &RunPlan, runtime: &RuntimeHandle) -> Result<(Command, String)> { match adapter_kind(plan) { AdapterKind::Process => process_local_host_command(plan, runtime), @@ -6112,6 +6162,39 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn unused_custom_model_does_not_disable_native_call_pricing() { + let models = serde_json::from_value(serde_json::json!({ + "default": {"provider": "openai", "model": "gpt-5.4"}, + "unused": { + "provider": "openai", "model": "other-model", + "base_url": "https://custom.invalid/v1" + } + })) + .expect("model roles"); + let allowed = snapshot_pricing_route(&models, &mut Command::new("unused"), |_| None); + let raw = serde_json::from_value(serde_json::json!({ + "extensions": {"nemo_fabric.usage_accounting.v1": { + "models_exhaustive": true, "gaps": [], "models": [{ + "provider": "openai", "model": "gpt-5.4", "gaps": [], + "reported": { + "input_tokens": 100, "cached_input_tokens": 20, + "cache_write_input_tokens": 0, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true + } + }] + }} + })) + .expect("source-established usage"); + let usage = + crate::model_usage::normalize_model_usage_for_routes(&raw, None, |provider, model| { + allowed.allows(provider, model) + }) + .expect("valid accounting") + .expect("normalized accounting"); + assert!(usage.cost.estimated.amount_nano_usd.is_some()); + } + #[test] fn local_host_returns_flat_supported_accounting_with_the_normalized_result() { let (_root, plan) = local_host_plan("accounted_usage_result"); @@ -6125,10 +6208,6 @@ for line in sys.stdin: assert!(usage.models_exhaustive); assert_eq!(usage.models[0].model, "gpt-5.4"); assert_eq!(usage.observed.input_tokens, Some(100)); - assert_eq!(usage.cost.estimated.amount_nano_usd, None); - assert_eq!(usage.cost.estimated.observed_nano_usd, None); - assert!(usage.cost.pricing.is_none()); - assert!(usage.cost.applied_rates.is_empty()); } #[test] diff --git a/crates/fabric-core/src/usage_normalization.rs b/crates/fabric-core/src/usage_normalization.rs index f236d7623..9ff9fb638 100644 --- a/crates/fabric-core/src/usage_normalization.rs +++ b/crates/fabric-core/src/usage_normalization.rs @@ -588,6 +588,10 @@ fn normalize_request(raw: &ReportedUsage, observation: bool) -> [Count; 8] { counts } +pub(crate) fn normalized_request(raw: &ReportedUsage, observation: bool) -> ModelMeasurements { + measurements(normalize_request(raw, observation)) +} + fn add(left: Count, right: Count) -> Count { left.and_then(|left| { right.and_then(|right| left.checked_add(right).ok_or("token_count_out_of_range")) @@ -687,7 +691,7 @@ fn measurements(counts: [Count; 8]) -> ModelMeasurements { unit: MeasurementUnit::NanoUsd, source: MeasurementSource::FabricPriceCatalog, attribution: Attribution::None, - reason: Some("catalog_pricing_unavailable".into()), + reason: Some("price_catalog_not_configured".into()), }, }, } diff --git a/crates/fabric-core/tests/model_usage.rs b/crates/fabric-core/tests/model_usage.rs new file mode 100644 index 000000000..21e8396df --- /dev/null +++ b/crates/fabric-core/tests/model_usage.rs @@ -0,0 +1,521 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Reduced public accounting and honest partial pricing. + +use nemo_fabric_core::AgentUsage; +use nemo_fabric_core::model_usage::normalize_model_usage; +use nemo_fabric_core::price_catalog::UserPriceCatalog; +use serde_json::{Value, json}; + +fn normalize(base: Value, accounting: Value) -> Value { + let mut usage: AgentUsage = serde_json::from_value(base).unwrap(); + usage + .extensions + .insert("nemo_fabric.usage_accounting.v1".into(), accounting); + serde_json::to_value(normalize_model_usage(&usage, None, false).unwrap().unwrap()).unwrap() +} + +#[test] +fn shared_source_vectors_keep_the_reduced_complete_and_observed_contract() { + let cases: Vec = serde_json::from_str(include_str!( + "../../../tests/fixtures/usage_accounting/conformance.json" + )) + .unwrap(); + let dimensions = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ]; + for case in cases { + let result = normalize(case["usage"].clone(), case["accounting"].clone()); + for (field, expected) in case["expected"].as_object().unwrap() { + if dimensions.contains(&field.as_str()) { + assert_eq!( + result[field], expected["complete"], + "{} {field}", + case["name"] + ); + assert_eq!( + result["observed"][field], expected["observed"], + "{} {field}", + case["name"] + ); + } + } + assert!(result.get("accounting").is_none()); + assert!(result.get("reasoning_tokens").is_none()); + assert!(result.get("peak_request_input_tokens").is_none()); + assert!(result["input_tokens"].is_number() || result["input_tokens"].is_null()); + } +} + +#[test] +fn legacy_usage_does_not_gain_a_completeness_guarantee() { + let usage: AgentUsage = serde_json::from_value(json!({"input_tokens": 100})).unwrap(); + assert!( + normalize_model_usage(&usage, None, false) + .unwrap() + .is_none() + ); +} + +#[test] +fn normalized_total_does_not_replace_the_independent_raw_provider_total() { + let mut usage: AgentUsage = serde_json::from_value(json!({ + "input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "total_tokens": 999, + "extensions": {"nemo_fabric.detailed_token_usage.v1": {"output_tokens": 5, "output_tokens_include_reasoning": true}} + })).unwrap(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": false, "gaps": []}), + ); + let normalized = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(normalized.tokens.total_tokens, Some(105)); + assert_eq!(usage.total_tokens, Some(999)); +} + +fn catalog(provider: &str) -> UserPriceCatalog { + catalog_with_cache_rates(provider, 0.1, 1.25) +} + +fn catalog_with_cache_rates(provider: &str, read: f64, write: f64) -> UserPriceCatalog { + UserPriceCatalog::from_bytes("test-catalog.json".into(), &serde_json::to_vec(&json!({ + "version": 1, "entries": [{"provider": provider, "model_id": "model", "currency": "USD", "unit": "per_token", + "rates": {"input_per_million": 1.0, "output_per_million": 2.0, "cache_read_per_million": read, "cache_write_per_million": write}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"}, + "pricing_as_of": "2026-10-21", "pricing_source": "test fixture"}] + })).unwrap()).unwrap() +} + +fn priced(provider: &str, counts: Value) -> nemo_fabric_core::model_usage::ModelUsage { + let mut usage = AgentUsage::default(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "provider": provider, "reported": counts, "gaps": []}] + }), + ); + normalize_model_usage(&usage, Some(&catalog(provider)), false) + .unwrap() + .unwrap() +} + +#[test] +fn nonzero_anthropic_writes_without_tariff_keep_tokens_and_only_observed_cost() { + let result = priced( + "anthropic", + json!({ + "input_tokens": 100, "cached_input_tokens": 20, "cache_write_input_tokens": 10, + "input_tokens_include_cache": true, "output_tokens": 5, "output_tokens_include_reasoning": true + }), + ); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + assert_eq!(result.cost.estimated.observed_nano_usd, Some(82_000)); + assert!( + result.models[0] + .gaps + .iter() + .any(|gap| gap.code.as_deref() == Some("cache_write_tariff_unverified")) + ); +} + +#[test] +fn zero_anthropic_writes_need_no_expiry_evidence() { + let result = priced( + "anthropic", + json!({ + "input_tokens": 100, "cached_input_tokens": 20, "cache_write_input_tokens": 0, + "input_tokens_include_cache": true, "output_tokens": 5, "output_tokens_include_reasoning": true + }), + ); + assert_eq!(result.cost.estimated.amount_nano_usd, Some(92_000)); + assert_eq!(result.cost.estimated.observed_nano_usd, Some(92_000)); +} + +#[test] +fn output_only_observations_are_priceable_without_claiming_complete_spend() { + let result = priced( + "openai", + json!({"output_tokens": 20, "output_tokens_include_reasoning": true}), + ); + assert_eq!(result.tokens.output_tokens, Some(20)); + assert_eq!(result.tokens.input_tokens, None); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + assert_eq!(result.cost.estimated.observed_nano_usd, Some(40_000)); +} + +#[test] +fn overflow_does_not_erase_independent_valid_counts() { + let result = normalize( + json!({}), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "x", "gaps": [], "reported": {"input_tokens": u64::MAX, "input_tokens_include_cache": true, + "output_tokens": 1, "output_tokens_include_reasoning": true}}]}), + ); + assert_eq!(result["input_tokens"], json!(u64::MAX)); + assert_eq!(result["output_tokens"], 1); + assert!(result["total_tokens"].is_null()); +} + +#[test] +fn equal_catalog_rates_do_not_prove_an_unknown_anthropic_write_tariff() { + let mut usage = AgentUsage::default(); + usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ + "models_exhaustive": true, "gaps": [], "models": [{"model": "model", "provider": "anthropic", "gaps": [], + "reported": {"input_tokens": 100, "cached_input_tokens": 20, "cache_write_input_tokens": 10, + "input_tokens_include_cache": true, "output_tokens": 5, "output_tokens_include_reasoning": true}}]})); + let result = normalize_model_usage( + &usage, + Some(&catalog_with_cache_rates("anthropic", 1.0, 1.0)), + false, + ) + .unwrap() + .unwrap(); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + // Uncached input 70 + reads 20 + output 5*2; writes remain unpriced. + assert_eq!(result.cost.estimated.observed_nano_usd, Some(100_000)); +} + +#[test] +fn explicit_provider_never_uses_another_providers_catalog_rate() { + let mut usage = AgentUsage::default(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "provider": "custom-route", "gaps": [], + "reported": {"input_tokens": 100, "cached_input_tokens": 0, + "input_tokens_include_cache": true, "output_tokens": 5, + "output_tokens_include_reasoning": true} + }]}), + ); + let result = normalize_model_usage(&usage, Some(&catalog("openai")), false) + .unwrap() + .unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + assert_eq!(result.cost.estimated.observed_nano_usd, None); + assert!( + result.models[0] + .gaps + .iter() + .any(|gap| gap.code.as_deref() == Some("model_not_in_catalog")) + ); +} + +#[test] +fn unverified_route_cannot_use_native_prices_for_a_bundled_model() { + let mut usage = AgentUsage::default(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "gpt-5.4", "gaps": [], + "reported": {"input_tokens": 100, "cached_input_tokens": 0, + "input_tokens_include_cache": true, "output_tokens": 5, + "output_tokens_include_reasoning": true} + }]}), + ); + // Global permission for bundled prices cannot establish an individual route. + let custom = normalize_model_usage(&usage, None, true).unwrap().unwrap(); + assert_eq!(custom.tokens.input_tokens, Some(100)); + assert_eq!(custom.tokens.total_tokens, Some(105)); + assert_eq!(custom.cost.estimated.amount_nano_usd, None); + assert_eq!(custom.cost.estimated.observed_nano_usd, None); + assert!( + custom.models[0] + .gaps + .iter() + .any(|gap| gap.code.as_deref() == Some("provider_price_unverified")) + ); + + // The same bundled model is priceable when the source establishes OpenAI. + usage + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]["provider"] = json!("openai"); + let native = normalize_model_usage(&usage, None, true).unwrap().unwrap(); + assert!(native.cost.estimated.amount_nano_usd.is_some()); + assert_eq!(native.tokens.total_tokens, Some(105)); +} + +fn cost_with_requests(aggregate: f64, requests: &[Option], exhaustive: bool) -> AgentUsage { + let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true, "cost_usd": aggregate}); + let mut usage = AgentUsage { + input_tokens: Some(100), + input_tokens_include_cache: Some(true), + output_tokens: Some(5), + cost_usd: Some(aggregate), + ..Default::default() + }; + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); + usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "provider": "test", "model": "test-model", "gaps": [], "reported": counts, + "requests_exhaustive": exhaustive, + "requests": requests.iter().map(|cost| json!({"usage": {"cost_usd": cost}})).collect::>() + }] + })); + usage +} + +#[test] +fn contradictory_request_reported_costs_are_rejected() { + for (requests, exhaustive) in [ + (vec![Some(0.2)], true), + (vec![Some(0.2)], false), + (vec![Some(0.04)], true), + (vec![Some(10_000_000_000.0); 2], true), + ] { + let usage = cost_with_requests(0.1, &requests, exhaustive); + assert!( + normalize_model_usage(&usage, None, false).is_err(), + "contradictory cost must not become complete: {requests:?}, exhaustive={exhaustive}" + ); + } +} + +#[test] +fn independently_rounded_request_costs_keep_fractional_nanos_and_large_values() { + for (aggregate, requests) in [ + (1.6e-9, vec![Some(0.4e-9); 4]), + ( + 200_000_000.72337076, + vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], + ), + ] { + let usage = cost_with_requests(aggregate, &requests, true); + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert!(result.cost.reported.amount_nano_usd.is_some()); + } +} + +#[test] +fn partial_or_missing_request_costs_preserve_independent_complete_dimensions() { + for (requests, exhaustive) in [ + (vec![Some(0.04)], false), + (vec![Some(0.04), None], true), + (vec![None], true), + (vec![Some(0.0)], false), + ] { + let usage = cost_with_requests(0.1, &requests, exhaustive); + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.output_tokens, Some(5)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.reported.amount_nano_usd, Some(100_000_000)); + } +} + +fn omit_model_cost(usage: &mut AgentUsage) { + usage + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]["reported"] + .as_object_mut() + .unwrap() + .remove("cost_usd"); +} + +#[test] +fn request_costs_without_model_cost_cannot_certify_a_contradictory_root() { + let mut usage = cost_with_requests(0.1, &[Some(0.2)], true); + omit_model_cost(&mut usage); + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.cost.reported.amount_nano_usd, None); + assert_eq!(result.cost.reported.observed_nano_usd, None); + assert_eq!(result.tokens.total_tokens, Some(105)); +} + +#[test] +fn exhaustive_requests_recover_cost_without_losing_legitimate_root_rounding() { + for (aggregate, requests) in [ + (0.1, vec![Some(0.04), Some(0.06)]), + ( + 200_000_000.72337076, + vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], + ), + ] { + let mut usage = cost_with_requests(aggregate, &requests, true); + omit_model_cost(&mut usage); + let with_root = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!( + with_root.cost.reported.amount_nano_usd, + Some((aggregate * 1e9).round() as u64) + ); + assert_eq!(with_root.tokens.total_tokens, Some(105)); + usage.cost_usd = None; + let recovered = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert!(recovered.cost.reported.amount_nano_usd.is_some()); + assert_eq!( + recovered.cost.reported.amount_nano_usd, + recovered.models[0].cost.reported.amount_nano_usd + ); + } +} + +#[test] +fn incomplete_requests_recover_only_observed_cost_when_no_complete_cost_exists() { + for (requests, exhaustive) in [(vec![Some(0.04)], false), (vec![Some(0.04), None], true)] { + let mut usage = cost_with_requests(0.1, &requests, exhaustive); + omit_model_cost(&mut usage); + usage.cost_usd = None; + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.cost.reported.amount_nano_usd, None); + assert_eq!(result.cost.reported.observed_nano_usd, Some(40_000_000)); + assert_eq!(result.tokens.total_tokens, Some(105)); + } +} + +#[test] +fn broad_request_gap_cannot_certify_complete_model_or_invocation_counts() { + let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true}); + for gap in ["unfinished_work", "missing_usage", "scope_unverified"] { + for root_complete in [false, true] { + let mut usage = AgentUsage::default(); + if root_complete { + usage.input_tokens = Some(100); + usage.input_tokens_include_cache = Some(true); + usage.output_tokens = Some(5); + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); + } + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "gaps": [], + "reported": if root_complete { json!({}) } else { counts.clone() }, + "requests_exhaustive": true, + "requests": [{"usage": {}, "gap": gap}] + }] + }), + ); + assert!( + normalize_model_usage(&usage, None, false).is_err(), + "broad request gap must conflict with complete counts: {gap}, root={root_complete}" + ); + } + } +} + +#[test] +fn broad_request_gap_keeps_observations_when_no_complete_value_is_claimed() { + let observed = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true}); + let result = normalize( + json!({}), + json!({ + "models_exhaustive": false, "gaps": [], "models": [{ + "model": "model", "gaps": [], "observed": observed, + "observed_tokens_aligned": true, "requests_exhaustive": true, + "requests": [{"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}] + }] + }), + ); + assert!(result["input_tokens"].is_null()); + assert!(result["output_tokens"].is_null()); + assert_eq!(result["observed"]["input_tokens"], 100); + assert_eq!(result["observed"]["output_tokens"], 5); + for scope in [&result, &result["models"][0]] { + for dimension in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + "reported_cost", + ] { + assert!( + scope["gaps"].as_array().unwrap().iter().any(|gap| { + gap["dimensions"] == json!([dimension]) && gap["reason"] == "unfinished_work" + }), + "request reason missing for {dimension}: {}", + scope["gaps"] + ); + } + } + + let mixed = normalize( + json!({}), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "pending", "gaps": [{"dimensions": ["output_tokens"], "reason": "missing_usage"}], + "observed": observed, "observed_tokens_aligned": true, + "requests_exhaustive": true, + "requests": [ + {"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}, + {"usage": {}, "gap": "scope_unverified"}, + {"usage": {}, "gap": "unfinished_work"} + ] + }, {"model": "settled", "reported": observed, "gaps": []}] + }), + ); + assert!(mixed["input_tokens"].is_null()); + assert_eq!(mixed["observed"]["input_tokens"], 200); + assert_eq!(mixed["observed"]["output_tokens"], 10); + assert_eq!(mixed["models"][1]["total_tokens"], 105); + assert!( + !mixed["models"][1]["gaps"] + .as_array() + .unwrap() + .iter() + .any(|gap| { + gap["reason"] == "unfinished_work" || gap["reason"] == "scope_unverified" + }) + ); + for scope in [&mixed, &mixed["models"][0]] { + let gaps = scope["gaps"].as_array().unwrap(); + for reason in ["unfinished_work", "scope_unverified"] { + assert_eq!( + gaps.iter() + .filter(|gap| { + gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == reason + }) + .count(), + 1, + "each request reason must survive once: {gaps:?}" + ); + } + } + assert!( + mixed["models"][0]["gaps"] + .as_array() + .unwrap() + .iter() + .any(|gap| { + gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == "missing_usage" + }) + ); +} + +#[test] +fn dimension_scoped_request_omission_keeps_independent_complete_output() { + let result = normalize( + json!({}), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "gaps": [{"dimensions": ["input_tokens"], "reason": "missing_usage"}], + "reported": {"output_tokens": 5, "output_tokens_include_reasoning": true}, + "requests_exhaustive": true, + "requests": [{"usage": {"output_tokens": 5, "output_tokens_include_reasoning": true}, "tokens_aligned": true}] + }] + }), + ); + assert!(result["input_tokens"].is_null()); + assert_eq!(result["output_tokens"], 5); + assert!(result["total_tokens"].is_null()); +} diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 42cd74b99..9778985f8 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -99,7 +99,7 @@ Collectors own source identity and invocation lifetime; the helper does not wait A request-wide `UsageRequest.gap` affects every dimension of that call and cannot coexist with complete model or invocation totals. For a dimension-specific gap, omit the affected request fields and declare the scoped gap without setting the request-wide `gap`. Other dimensions can remain complete. -A provider name must identify the source-established provider, not merely an API-compatible protocol. Custom endpoints and arbitrary gateways do not establish a native provider tariff. Preserve the model and counts when provider identity cannot be established. Catalog estimation is added separately and never changes reported usage. +A provider name must identify the source-established provider, not merely an API-compatible protocol. Custom endpoints and arbitrary gateways do not establish a native provider tariff. Preserve the model and counts when provider identity cannot be established; explicit user catalogs can price that evidence with labeled assumptions. Core normalizes cache-inclusive input and reasoning-inclusive output once and returns the flat `RunResult.model_usage` view. Raw reasoning/peak/request evidence diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index 6a624bb47..f6aa0ba97 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -45,6 +45,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`models.McpServerConfig`](./nemo_fabric.models.md#class-mcpserverconfig): MCP server configuration. - [`models.MetadataConfig`](./nemo_fabric.models.md#class-metadataconfig): Human-readable agent identity. - [`models.ModelConfig`](./nemo_fabric.models.md#class-modelconfig): Configuration for one model role. +- [`models.PricingConfig`](./nemo_fabric.models.md#class-pricingconfig): Frozen Relay-compatible price-catalog override, read during planning. - [`models.RelayAtifConfig`](./nemo_fabric.models.md#class-relayatifconfig): NeMo Relay ATIF export configuration. - [`models.RelayAtofConfig`](./nemo_fabric.models.md#class-relayatofconfig): NeMo Relay ATOF export configuration. - [`models.RelayAtofFileSinkConfig`](./nemo_fabric.models.md#class-relayatoffilesinkconfig): NeMo Relay ATOF file sink configuration. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index 3aa659030..03e7e0335 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -16,7 +16,6 @@ The Rust core remains the source of truth for persisted schema snapshots. These --- - ## class `FabricBaseModel` Base class for SDK-facing Pydantic models. @@ -54,7 +53,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -65,7 +63,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -77,7 +74,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `MetadataConfig` Human-readable agent identity. @@ -125,7 +121,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -136,7 +131,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -148,7 +142,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `HarnessConfig` Harness adapter selection plus adapter-owned settings. @@ -197,7 +190,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -208,7 +200,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -220,7 +211,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `WorkflowConfig` Registered workflow target and immutable construction settings. @@ -268,7 +258,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -279,7 +268,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -291,7 +279,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `DiscoveryConfig` Explicit local descriptor discovery paths. @@ -338,7 +325,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -349,7 +335,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -361,7 +346,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `InstructionConfig` One portable instruction value. @@ -409,7 +393,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -420,7 +403,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -432,7 +414,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `InstructionsConfig` Harness-neutral agent instructions. @@ -479,7 +460,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -490,7 +470,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -502,7 +481,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RuntimeConfig` Invocation runtime contract. @@ -553,7 +531,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -564,7 +541,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -576,7 +552,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `EnvironmentConfig` Execution environment configuration supplied by the consumer. @@ -634,7 +609,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -645,7 +619,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -657,7 +630,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `ModelConfig` Configuration for one model role. @@ -711,7 +683,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -722,7 +693,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -734,7 +704,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `SkillConfig` Skill capability configuration. @@ -781,7 +750,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_path` ```python @@ -792,7 +760,6 @@ Add a skill path if absent. --- - ### classmethod `from_mapping` ```python @@ -803,7 +770,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_path` ```python @@ -814,7 +780,6 @@ Remove a skill path if present. --- - ### method `to_mapping` ```python @@ -826,7 +791,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `McpAuthenticationConfig` MCP server authentication configuration. @@ -883,7 +847,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -894,7 +857,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -906,7 +868,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `McpServerConfig` MCP server configuration. @@ -961,7 +922,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -972,7 +932,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -984,7 +943,6 @@ Return the server mapping without collapsing an explicit empty allowlist. --- - ## class `McpConfig` MCP capability configuration. @@ -1031,7 +989,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_server` ```python @@ -1057,7 +1014,6 @@ For stdio, set ``url`` to the executable and pass each command-line argument as --- - ### classmethod `from_mapping` ```python @@ -1068,7 +1024,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_server` ```python @@ -1079,7 +1034,6 @@ Remove a named MCP server if present. --- - ### method `to_mapping` ```python @@ -1091,7 +1045,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayConfigPolicy` NVIDIA NeMo Relay config validation policy. @@ -1140,7 +1093,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1151,7 +1103,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1163,7 +1114,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtofFileSinkConfig` NeMo Relay ATOF file sink configuration. @@ -1213,7 +1163,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1224,7 +1173,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1236,7 +1184,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtofStreamSinkConfig` NeMo Relay ATOF stream sink configuration. @@ -1290,7 +1237,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1301,7 +1247,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1313,7 +1258,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtofConfig` NeMo Relay ATOF export configuration. @@ -1361,7 +1305,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1372,7 +1315,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1384,7 +1326,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayS3StorageConfig` NeMo Relay ATIF S3 storage configuration. @@ -1439,7 +1380,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1450,7 +1390,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1462,7 +1401,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayHttpStorageConfig` NeMo Relay ATIF HTTP storage configuration. @@ -1513,7 +1451,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1524,7 +1461,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1536,7 +1472,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtifConfig` NeMo Relay ATIF export configuration. @@ -1591,7 +1526,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1602,7 +1536,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1614,7 +1547,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayOpenTelemetryEndpointConfig` One typed NeMo Relay OpenTelemetry destination. @@ -1674,7 +1606,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1685,7 +1616,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1697,7 +1627,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayOpenTelemetryConfig` NeMo Relay typed OpenTelemetry destination configuration. @@ -1745,7 +1674,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1756,7 +1684,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1768,7 +1695,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayObservabilityConfig` NeMo Relay observability component configuration. @@ -1820,7 +1746,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1831,7 +1756,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1843,7 +1767,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayComponentConfig` Generic NeMo Relay plugin component configuration. @@ -1892,7 +1815,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1903,7 +1825,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1915,7 +1836,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayConfig` First-class NeMo Relay integration configuration. @@ -1966,7 +1886,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1977,7 +1896,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1989,7 +1907,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `TelemetryProviderConfig` Provider-specific telemetry configuration. @@ -2036,7 +1953,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2047,7 +1963,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2059,7 +1974,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `TelemetryConfig` Telemetry configuration. @@ -2106,7 +2020,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `enable_native` ```python @@ -2117,7 +2030,6 @@ Let the selected adapter handle telemetry natively. --- - ### method `enable_relay` ```python @@ -2128,7 +2040,6 @@ Enable NeMo Relay telemetry for subsequently started runtimes. --- - ### classmethod `from_mapping` ```python @@ -2139,7 +2050,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_provider` ```python @@ -2150,7 +2060,6 @@ Remove a configured telemetry provider. --- - ### method `to_mapping` ```python @@ -2162,7 +2071,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `ToolDefinitionConfig` One named normalized tool or tool-group definition. @@ -2211,7 +2119,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2222,7 +2129,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2234,7 +2140,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `ToolsConfig` Harness-neutral tool capability configuration. @@ -2283,7 +2188,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_definition` ```python @@ -2301,7 +2205,6 @@ Add or replace one named definition and return this tools config. --- - ### classmethod `from_mapping` ```python @@ -2312,7 +2215,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_definition` ```python @@ -2323,7 +2225,6 @@ Remove one named definition and return this tools config. --- - ### method `to_mapping` ```python @@ -2335,7 +2236,6 @@ Return the tool mapping without collapsing an explicit empty policy. --- - ## class `SystemStatsSamplingConfig` Opt-in periodic collection; bounds include both endpoint samples. @@ -2383,7 +2283,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2394,7 +2293,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2406,7 +2304,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `SystemStatsConfig` Native invocation stats selection, independent of Relay and recording. @@ -2456,7 +2353,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2467,7 +2363,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2479,6 +2374,72 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- +## class `PricingConfig` + +Frozen Relay-compatible price-catalog override, read during planning. + + + +### Fields + +The model defines the following fields: + +| Field | Type | Required | Default | Constraints | Description | +| --- | --- | --- | --- | --- | --- | +| `catalog_path` | `str \| Path \| None` | No | `None` | — | — | + +--- + +### property extra_fields + +Return fields preserved by the extension point for this model. + +--- + +### property model_extra + +Get extra fields set during validation. + + + +**Returns:** + A dictionary of extra fields, or `None` if `config.extra` is not set to `"allow"`. + +--- + +### property model_fields_set + +Returns the set of fields that have been explicitly set on this model instance. + + + +**Returns:** + A set of strings representing the fields that have been set, i.e. that were not filled from defaults. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(value: Mapping[str, Any]) -> Self +``` + +Validate a mapping using this Pydantic model. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached JSON-compatible mapping for Rust/core calls. + + +--- ## class `FabricConfig` @@ -2508,6 +2469,7 @@ The model defines the following fields: | `system_stats` | `SystemStatsConfig \| None` | No | `SystemStatsConfig()` | — | — | | `telemetry` | `TelemetryConfig \| None` | No | `None` | — | — | | `relay` | `RelayConfig \| None` | No | `None` | — | — | +| `pricing` | `PricingConfig \| None` | No | `None` | — | — | | `tools` | `ToolsConfig \| None` | No | `None` | — | — | --- @@ -2542,7 +2504,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_mcp_server` ```python @@ -2568,7 +2529,6 @@ For stdio, set ``url`` to the executable and pass each command-line argument as --- - ### method `add_skill_path` ```python @@ -2579,7 +2539,6 @@ Add a skill path and return this config. --- - ### method `add_tool_definition` ```python @@ -2597,7 +2556,6 @@ Add or replace one named tool definition and return this config. --- - ### method `block_tools` ```python @@ -2608,7 +2566,6 @@ Block adapter-native tool names and return this config. --- - ### method `enable_relay` ```python @@ -2626,7 +2583,6 @@ Enable NeMo Relay telemetry and return this config. --- - ### classmethod `from_mapping` ```python @@ -2637,7 +2593,6 @@ Validate the public agent config mapping shape. --- - ### method `remove_mcp_server` ```python @@ -2648,7 +2603,6 @@ Remove a named MCP server and return this config. --- - ### method `remove_skill_path` ```python @@ -2659,7 +2613,6 @@ Remove a skill path and return this config. --- - ### method `remove_tool_definition` ```python @@ -2670,7 +2623,6 @@ Remove one named tool definition and return this config. --- - ### method `to_mapping` ```python @@ -2682,7 +2634,6 @@ Return a detached mapping matching the Rust ``FabricConfig`` schema. --- - ## class `RunRequest` One validated NeMo Fabric invocation request. @@ -2733,7 +2684,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2744,7 +2694,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 8a789e763..12f68b13e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 142 +position: 143 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 359d1c117..40d3afc67 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 143 +position: 144 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 6bbdf7d45..8d39f22be 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 144 +position: 145 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx index 40e387ace..9690f011e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Kind" sidebar-title: "CapabilityKind" description: "Capability kind." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,23 +43,23 @@ MCP server. ### `impl Clone for CapabilityKind` -
Clone for CapabilityKind"}} />
+
Clone for CapabilityKind"}} />
#### `clone` -
clone(&self) -> CapabilityKind"}} />
+
clone(&self) -> CapabilityKind"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for CapabilityKind` -
Debug for CapabilityKind"}} />
+
Debug for CapabilityKind"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for CapabilityKind` @@ -67,7 +67,7 @@ MCP server. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for CapabilityKind` @@ -75,11 +75,11 @@ MCP server. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ MCP server. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for CapabilityKind` -
PartialEq for CapabilityKind"}} />
+
PartialEq for CapabilityKind"}} />
#### `eq` -
eq(&self, other: &CapabilityKind) -> bool"}} />
+
eq(&self, other: &CapabilityKind) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for CapabilityKind` @@ -107,16 +107,16 @@ MCP server. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for CapabilityKind` -
Copy for CapabilityKind"}} />
+
Copy for CapabilityKind"}} />
### `impl Eq for CapabilityKind` -
Eq for CapabilityKind"}} />
+
Eq for CapabilityKind"}} />
### `impl StructuralPartialEq for CapabilityKind` -
StructuralPartialEq for CapabilityKind"}} />
+
StructuralPartialEq for CapabilityKind"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx index e121e6b73..b0c744403 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Target" sidebar-title: "CapabilityTarget" description: "Component responsible for executing a configured capability." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -45,23 +45,23 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. ### `impl Clone for CapabilityTarget` -
Clone for CapabilityTarget"}} />
+
Clone for CapabilityTarget"}} />
#### `clone` -
clone(&self) -> CapabilityTarget"}} />
+
clone(&self) -> CapabilityTarget"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for CapabilityTarget` -
Debug for CapabilityTarget"}} />
+
Debug for CapabilityTarget"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for CapabilityTarget` @@ -69,7 +69,7 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for CapabilityTarget` @@ -77,11 +77,11 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -89,19 +89,19 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for CapabilityTarget` -
PartialEq for CapabilityTarget"}} />
+
PartialEq for CapabilityTarget"}} />
#### `eq` -
eq(&self, other: &CapabilityTarget) -> bool"}} />
+
eq(&self, other: &CapabilityTarget) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for CapabilityTarget` @@ -109,16 +109,16 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for CapabilityTarget` -
Copy for CapabilityTarget"}} />
+
Copy for CapabilityTarget"}} />
### `impl Eq for CapabilityTarget` -
Eq for CapabilityTarget"}} />
+
Eq for CapabilityTarget"}} />
### `impl StructuralPartialEq for CapabilityTarget` -
StructuralPartialEq for CapabilityTarget"}} />
+
StructuralPartialEq for CapabilityTarget"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx index 74dc754f8..ad66e7ecf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx @@ -2,14 +2,14 @@ title: "Enum Relay Atif Storage Config" sidebar-title: "RelayAtifStorageConfig" description: "Relay ATIF remote storage configuration." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        extensions: BTreeMap<String, Value>,\n    },\n    S3 {\n        bucket: String,\n        key_prefix: Option<String>,\n        access_key_id: Option<String>,\n        secret_access_key_var: Option<String>,\n        session_token_var: Option<String>,\n        region: Option<String>,\n        endpoint_url: Option<String>,\n        allow_http: Option<bool>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
+
String,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        extensions: BTreeMap<String, Value>,\n    },\n    S3 {\n        bucket: String,\n        key_prefix: Option<String>,\n        access_key_id: Option<String>,\n        secret_access_key_var: Option<String>,\n        session_token_var: Option<String>,\n        region: Option<String>,\n        endpoint_url: Option<String>,\n        allow_http: Option<bool>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
Relay ATIF remote storage configuration. @@ -91,23 +91,23 @@ Additive S3 storage fields. ### `impl Clone for RelayAtifStorageConfig` -
Clone for RelayAtifStorageConfig"}} />
+
Clone for RelayAtifStorageConfig"}} />
#### `clone` -
clone(&self) -> RelayAtifStorageConfig"}} />
+
clone(&self) -> RelayAtifStorageConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtifStorageConfig` -
Debug for RelayAtifStorageConfig"}} />
+
Debug for RelayAtifStorageConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayAtifStorageConfig` @@ -115,7 +115,7 @@ Additive S3 storage fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtifStorageConfig` @@ -123,11 +123,11 @@ Additive S3 storage fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -135,19 +135,19 @@ Additive S3 storage fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtifStorageConfig` -
PartialEq for RelayAtifStorageConfig"}} />
+
PartialEq for RelayAtifStorageConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtifStorageConfig) -> bool"}} />
+
eq(&self, other: &RelayAtifStorageConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtifStorageConfig` @@ -155,8 +155,8 @@ Additive S3 storage fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtifStorageConfig` -
StructuralPartialEq for RelayAtifStorageConfig"}} />
+
StructuralPartialEq for RelayAtifStorageConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx index bbf8ff9e5..7f00685ce 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Mode" sidebar-title: "RelayAtofMode" description: "Relay ATOF file mode." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -36,31 +36,31 @@ Overwrite an existing ATOF file. ### `impl Clone for RelayAtofMode` -
Clone for RelayAtofMode"}} />
+
Clone for RelayAtofMode"}} />
#### `clone` -
clone(&self) -> RelayAtofMode"}} />
+
clone(&self) -> RelayAtofMode"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofMode` -
Debug for RelayAtofMode"}} />
+
Debug for RelayAtofMode"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofMode` -
Default for RelayAtofMode"}} />
+
Default for RelayAtofMode"}} />
#### `default` -
default() -> RelayAtofMode"}} />
+
default() -> RelayAtofMode"}} />
### `impl<'de> Deserialize<'de> for RelayAtofMode` @@ -68,7 +68,7 @@ Overwrite an existing ATOF file. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofMode` @@ -76,11 +76,11 @@ Overwrite an existing ATOF file. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -88,19 +88,19 @@ Overwrite an existing ATOF file. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofMode` -
PartialEq for RelayAtofMode"}} />
+
PartialEq for RelayAtofMode"}} />
#### `eq` -
eq(&self, other: &RelayAtofMode) -> bool"}} />
+
eq(&self, other: &RelayAtofMode) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofMode` @@ -108,16 +108,16 @@ Overwrite an existing ATOF file. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayAtofMode` -
Copy for RelayAtofMode"}} />
+
Copy for RelayAtofMode"}} />
### `impl Eq for RelayAtofMode` -
Eq for RelayAtofMode"}} />
+
Eq for RelayAtofMode"}} />
### `impl StructuralPartialEq for RelayAtofMode` -
StructuralPartialEq for RelayAtofMode"}} />
+
StructuralPartialEq for RelayAtofMode"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx index d53f00f8d..f71cbc8ff 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx @@ -2,14 +2,14 @@ title: "Enum Relay Atof Sink Config" sidebar-title: "RelayAtofSinkConfig" description: "Relay ATOF sink configuration." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Option<PathBuf>,\n        filename: Option<String>,\n        mode: RelayAtofMode,\n        extensions: BTreeMap<String, Value>,\n    },\n    Stream {\n        url: String,\n        transport: RelayAtofStreamTransport,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        field_name_policy: RelayAtofStreamFieldNamePolicy,\n        name: Option<String>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
+
Option<PathBuf>,\n        filename: Option<String>,\n        mode: RelayAtofMode,\n        extensions: BTreeMap<String, Value>,\n    },\n    Stream {\n        url: String,\n        transport: RelayAtofStreamTransport,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        field_name_policy: RelayAtofStreamFieldNamePolicy,\n        name: Option<String>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
Relay ATOF sink configuration. @@ -83,23 +83,23 @@ Additive stream sink fields. ### `impl Clone for RelayAtofSinkConfig` -
Clone for RelayAtofSinkConfig"}} />
+
Clone for RelayAtofSinkConfig"}} />
#### `clone` -
clone(&self) -> RelayAtofSinkConfig"}} />
+
clone(&self) -> RelayAtofSinkConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofSinkConfig` -
Debug for RelayAtofSinkConfig"}} />
+
Debug for RelayAtofSinkConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayAtofSinkConfig` @@ -107,7 +107,7 @@ Additive stream sink fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofSinkConfig` @@ -115,11 +115,11 @@ Additive stream sink fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -127,19 +127,19 @@ Additive stream sink fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofSinkConfig` -
PartialEq for RelayAtofSinkConfig"}} />
+
PartialEq for RelayAtofSinkConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtofSinkConfig) -> bool"}} />
+
eq(&self, other: &RelayAtofSinkConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofSinkConfig` @@ -147,8 +147,8 @@ Additive stream sink fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtofSinkConfig` -
StructuralPartialEq for RelayAtofSinkConfig"}} />
+
StructuralPartialEq for RelayAtofSinkConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx index 892413a9e..d01f8c91c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Field Name Policy" sidebar-title: "RelayAtofStreamFieldNamePolicy" description: "Relay ATOF stream field-name policy." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -36,31 +36,31 @@ Replace dots in field names. ### `impl Clone for RelayAtofStreamFieldNamePolicy` -
Clone for RelayAtofStreamFieldNamePolicy"}} />
+
Clone for RelayAtofStreamFieldNamePolicy"}} />
#### `clone` -
clone(&self) -> RelayAtofStreamFieldNamePolicy"}} />
+
clone(&self) -> RelayAtofStreamFieldNamePolicy"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofStreamFieldNamePolicy` -
Debug for RelayAtofStreamFieldNamePolicy"}} />
+
Debug for RelayAtofStreamFieldNamePolicy"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofStreamFieldNamePolicy` -
Default for RelayAtofStreamFieldNamePolicy"}} />
+
Default for RelayAtofStreamFieldNamePolicy"}} />
#### `default` -
default() -> RelayAtofStreamFieldNamePolicy"}} />
+
default() -> RelayAtofStreamFieldNamePolicy"}} />
### `impl<'de> Deserialize<'de> for RelayAtofStreamFieldNamePolicy` @@ -68,7 +68,7 @@ Replace dots in field names. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofStreamFieldNamePolicy` @@ -76,11 +76,11 @@ Replace dots in field names. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -88,19 +88,19 @@ Replace dots in field names. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofStreamFieldNamePolicy` -
PartialEq for RelayAtofStreamFieldNamePolicy"}} />
+
PartialEq for RelayAtofStreamFieldNamePolicy"}} />
#### `eq` -
eq(&self, other: &RelayAtofStreamFieldNamePolicy) -> bool"}} />
+
eq(&self, other: &RelayAtofStreamFieldNamePolicy) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofStreamFieldNamePolicy` @@ -108,16 +108,16 @@ Replace dots in field names. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayAtofStreamFieldNamePolicy` -
Copy for RelayAtofStreamFieldNamePolicy"}} />
+
Copy for RelayAtofStreamFieldNamePolicy"}} />
### `impl Eq for RelayAtofStreamFieldNamePolicy` -
Eq for RelayAtofStreamFieldNamePolicy"}} />
+
Eq for RelayAtofStreamFieldNamePolicy"}} />
### `impl StructuralPartialEq for RelayAtofStreamFieldNamePolicy` -
StructuralPartialEq for RelayAtofStreamFieldNamePolicy"}} />
+
StructuralPartialEq for RelayAtofStreamFieldNamePolicy"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx index ea48753dd..1f47f4384 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Transport" sidebar-title: "RelayAtofStreamTransport" description: "Relay ATOF stream transport." -position: 80 +position: 81 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,31 +43,31 @@ NDJSON transport. ### `impl Clone for RelayAtofStreamTransport` -
Clone for RelayAtofStreamTransport"}} />
+
Clone for RelayAtofStreamTransport"}} />
#### `clone` -
clone(&self) -> RelayAtofStreamTransport"}} />
+
clone(&self) -> RelayAtofStreamTransport"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofStreamTransport` -
Debug for RelayAtofStreamTransport"}} />
+
Debug for RelayAtofStreamTransport"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofStreamTransport` -
Default for RelayAtofStreamTransport"}} />
+
Default for RelayAtofStreamTransport"}} />
#### `default` -
default() -> RelayAtofStreamTransport"}} />
+
default() -> RelayAtofStreamTransport"}} />
### `impl<'de> Deserialize<'de> for RelayAtofStreamTransport` @@ -75,7 +75,7 @@ NDJSON transport. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofStreamTransport` @@ -83,11 +83,11 @@ NDJSON transport. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ NDJSON transport. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofStreamTransport` -
PartialEq for RelayAtofStreamTransport"}} />
+
PartialEq for RelayAtofStreamTransport"}} />
#### `eq` -
eq(&self, other: &RelayAtofStreamTransport) -> bool"}} />
+
eq(&self, other: &RelayAtofStreamTransport) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofStreamTransport` @@ -115,16 +115,16 @@ NDJSON transport. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayAtofStreamTransport` -
Copy for RelayAtofStreamTransport"}} />
+
Copy for RelayAtofStreamTransport"}} />
### `impl Eq for RelayAtofStreamTransport` -
Eq for RelayAtofStreamTransport"}} />
+
Eq for RelayAtofStreamTransport"}} />
### `impl StructuralPartialEq for RelayAtofStreamTransport` -
StructuralPartialEq for RelayAtofStreamTransport"}} />
+
StructuralPartialEq for RelayAtofStreamTransport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx index beb3da6eb..8689a98ef 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Endpoint Type" sidebar-title: "RelayOpenTelemetryEndpointType" description: "Relay OpenTelemetry span projection." -position: 81 +position: 82 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,23 +43,23 @@ Emit OpenInference semantic-convention spans. ### `impl Clone for RelayOpenTelemetryEndpointType` -
Clone for RelayOpenTelemetryEndpointType"}} />
+
Clone for RelayOpenTelemetryEndpointType"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryEndpointType"}} />
+
clone(&self) -> RelayOpenTelemetryEndpointType"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryEndpointType` -
Debug for RelayOpenTelemetryEndpointType"}} />
+
Debug for RelayOpenTelemetryEndpointType"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryEndpointType` @@ -67,7 +67,7 @@ Emit OpenInference semantic-convention spans. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryEndpointType` @@ -75,11 +75,11 @@ Emit OpenInference semantic-convention spans. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Emit OpenInference semantic-convention spans. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryEndpointType` -
PartialEq for RelayOpenTelemetryEndpointType"}} />
+
PartialEq for RelayOpenTelemetryEndpointType"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryEndpointType) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryEndpointType) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryEndpointType` @@ -107,16 +107,16 @@ Emit OpenInference semantic-convention spans. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayOpenTelemetryEndpointType` -
Copy for RelayOpenTelemetryEndpointType"}} />
+
Copy for RelayOpenTelemetryEndpointType"}} />
### `impl Eq for RelayOpenTelemetryEndpointType` -
Eq for RelayOpenTelemetryEndpointType"}} />
+
Eq for RelayOpenTelemetryEndpointType"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryEndpointType` -
StructuralPartialEq for RelayOpenTelemetryEndpointType"}} />
+
StructuralPartialEq for RelayOpenTelemetryEndpointType"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx index da13057aa..4069e7d1b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Mark Projection" sidebar-title: "RelayOpenTelemetryMarkProjection" description: "Relay OpenTelemetry mark projection policy." -position: 82 +position: 83 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,31 +43,31 @@ Project marks as tool spans. ### `impl Clone for RelayOpenTelemetryMarkProjection` -
Clone for RelayOpenTelemetryMarkProjection"}} />
+
Clone for RelayOpenTelemetryMarkProjection"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryMarkProjection"}} />
+
clone(&self) -> RelayOpenTelemetryMarkProjection"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryMarkProjection` -
Debug for RelayOpenTelemetryMarkProjection"}} />
+
Debug for RelayOpenTelemetryMarkProjection"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayOpenTelemetryMarkProjection` -
Default for RelayOpenTelemetryMarkProjection"}} />
+
Default for RelayOpenTelemetryMarkProjection"}} />
#### `default` -
default() -> RelayOpenTelemetryMarkProjection"}} />
+
default() -> RelayOpenTelemetryMarkProjection"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryMarkProjection` @@ -75,7 +75,7 @@ Project marks as tool spans. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryMarkProjection` @@ -83,11 +83,11 @@ Project marks as tool spans. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ Project marks as tool spans. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryMarkProjection` -
PartialEq for RelayOpenTelemetryMarkProjection"}} />
+
PartialEq for RelayOpenTelemetryMarkProjection"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryMarkProjection) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryMarkProjection) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryMarkProjection` @@ -115,16 +115,16 @@ Project marks as tool spans. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayOpenTelemetryMarkProjection` -
Copy for RelayOpenTelemetryMarkProjection"}} />
+
Copy for RelayOpenTelemetryMarkProjection"}} />
### `impl Eq for RelayOpenTelemetryMarkProjection` -
Eq for RelayOpenTelemetryMarkProjection"}} />
+
Eq for RelayOpenTelemetryMarkProjection"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryMarkProjection` -
StructuralPartialEq for RelayOpenTelemetryMarkProjection"}} />
+
StructuralPartialEq for RelayOpenTelemetryMarkProjection"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx index 8db6e0872..287821dd1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Otlp Transport" sidebar-title: "RelayOtlpTransport" description: "Relay OTLP transport." -position: 83 +position: 84 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -36,31 +36,31 @@ OTLP gRPC transport. ### `impl Clone for RelayOtlpTransport` -
Clone for RelayOtlpTransport"}} />
+
Clone for RelayOtlpTransport"}} />
#### `clone` -
clone(&self) -> RelayOtlpTransport"}} />
+
clone(&self) -> RelayOtlpTransport"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOtlpTransport` -
Debug for RelayOtlpTransport"}} />
+
Debug for RelayOtlpTransport"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayOtlpTransport` -
Default for RelayOtlpTransport"}} />
+
Default for RelayOtlpTransport"}} />
#### `default` -
default() -> RelayOtlpTransport"}} />
+
default() -> RelayOtlpTransport"}} />
### `impl<'de> Deserialize<'de> for RelayOtlpTransport` @@ -68,7 +68,7 @@ OTLP gRPC transport. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOtlpTransport` @@ -76,11 +76,11 @@ OTLP gRPC transport. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -88,19 +88,19 @@ OTLP gRPC transport. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOtlpTransport` -
PartialEq for RelayOtlpTransport"}} />
+
PartialEq for RelayOtlpTransport"}} />
#### `eq` -
eq(&self, other: &RelayOtlpTransport) -> bool"}} />
+
eq(&self, other: &RelayOtlpTransport) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOtlpTransport` @@ -108,16 +108,16 @@ OTLP gRPC transport. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayOtlpTransport` -
Copy for RelayOtlpTransport"}} />
+
Copy for RelayOtlpTransport"}} />
### `impl Eq for RelayOtlpTransport` -
Eq for RelayOtlpTransport"}} />
+
Eq for RelayOtlpTransport"}} />
### `impl StructuralPartialEq for RelayOtlpTransport` -
StructuralPartialEq for RelayOtlpTransport"}} />
+
StructuralPartialEq for RelayOtlpTransport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx index 95ec26616..bd688cc9f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Unsupported Behavior" sidebar-title: "RelayUnsupportedBehavior" description: "Relay unsupported/unknown config handling." -position: 84 +position: 85 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,31 +43,31 @@ Error on the unsupported or unknown value. ### `impl Clone for RelayUnsupportedBehavior` -
Clone for RelayUnsupportedBehavior"}} />
+
Clone for RelayUnsupportedBehavior"}} />
#### `clone` -
clone(&self) -> RelayUnsupportedBehavior"}} />
+
clone(&self) -> RelayUnsupportedBehavior"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayUnsupportedBehavior` -
Debug for RelayUnsupportedBehavior"}} />
+
Debug for RelayUnsupportedBehavior"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayUnsupportedBehavior` -
Default for RelayUnsupportedBehavior"}} />
+
Default for RelayUnsupportedBehavior"}} />
#### `default` -
default() -> RelayUnsupportedBehavior"}} />
+
default() -> RelayUnsupportedBehavior"}} />
### `impl<'de> Deserialize<'de> for RelayUnsupportedBehavior` @@ -75,7 +75,7 @@ Error on the unsupported or unknown value. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayUnsupportedBehavior` @@ -83,11 +83,11 @@ Error on the unsupported or unknown value. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ Error on the unsupported or unknown value. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayUnsupportedBehavior` -
PartialEq for RelayUnsupportedBehavior"}} />
+
PartialEq for RelayUnsupportedBehavior"}} />
#### `eq` -
eq(&self, other: &RelayUnsupportedBehavior) -> bool"}} />
+
eq(&self, other: &RelayUnsupportedBehavior) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayUnsupportedBehavior` @@ -115,16 +115,16 @@ Error on the unsupported or unknown value. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayUnsupportedBehavior` -
Copy for RelayUnsupportedBehavior"}} />
+
Copy for RelayUnsupportedBehavior"}} />
### `impl Eq for RelayUnsupportedBehavior` -
Eq for RelayUnsupportedBehavior"}} />
+
Eq for RelayUnsupportedBehavior"}} />
### `impl StructuralPartialEq for RelayUnsupportedBehavior` -
StructuralPartialEq for RelayUnsupportedBehavior"}} />
+
StructuralPartialEq for RelayUnsupportedBehavior"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx index f130d1764..ee84e47b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx @@ -2,7 +2,7 @@ title: "Enum Resolution Strategy" sidebar-title: "ResolutionStrategy" description: "Adapter install or availability strategy." -position: 54 +position: 55 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -71,23 +71,23 @@ Adapter is installed through a harness-native plugin manager. ### `impl Clone for ResolutionStrategy` -
Clone for ResolutionStrategy"}} />
+
Clone for ResolutionStrategy"}} />
#### `clone` -
clone(&self) -> ResolutionStrategy"}} />
+
clone(&self) -> ResolutionStrategy"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolutionStrategy` -
Debug for ResolutionStrategy"}} />
+
Debug for ResolutionStrategy"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ResolutionStrategy` @@ -95,7 +95,7 @@ Adapter is installed through a harness-native plugin manager. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ResolutionStrategy` @@ -103,11 +103,11 @@ Adapter is installed through a harness-native plugin manager. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -115,19 +115,19 @@ Adapter is installed through a harness-native plugin manager. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ResolutionStrategy` -
PartialEq for ResolutionStrategy"}} />
+
PartialEq for ResolutionStrategy"}} />
#### `eq` -
eq(&self, other: &ResolutionStrategy) -> bool"}} />
+
eq(&self, other: &ResolutionStrategy) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ResolutionStrategy` @@ -135,16 +135,16 @@ Adapter is installed through a harness-native plugin manager. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for ResolutionStrategy` -
Copy for ResolutionStrategy"}} />
+
Copy for ResolutionStrategy"}} />
### `impl Eq for ResolutionStrategy` -
Eq for ResolutionStrategy"}} />
+
Eq for ResolutionStrategy"}} />
### `impl StructuralPartialEq for ResolutionStrategy` -
StructuralPartialEq for ResolutionStrategy"}} />
+
StructuralPartialEq for ResolutionStrategy"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx index 8ce5c9908..17b80b5bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx @@ -2,7 +2,7 @@ title: "Enum Telemetry Provider" sidebar-title: "TelemetryProvider" description: "Telemetry runtime provider." -position: 64 +position: 65 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -40,7 +40,7 @@ Let the selected adapter handle telemetry natively. #### `as_str` -
str"}} />
+
str"}} />
Return the stable configuration value for this provider. @@ -48,31 +48,31 @@ Return the stable configuration value for this provider. ### `impl Clone for TelemetryProvider` -
Clone for TelemetryProvider"}} />
+
Clone for TelemetryProvider"}} />
#### `clone` -
clone(&self) -> TelemetryProvider"}} />
+
clone(&self) -> TelemetryProvider"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryProvider` -
Debug for TelemetryProvider"}} />
+
Debug for TelemetryProvider"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for TelemetryProvider` -
Default for TelemetryProvider"}} />
+
Default for TelemetryProvider"}} />
#### `default` -
default() -> TelemetryProvider"}} />
+
default() -> TelemetryProvider"}} />
### `impl<'de> Deserialize<'de> for TelemetryProvider` @@ -80,7 +80,7 @@ Return the stable configuration value for this provider. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryProvider` @@ -88,11 +88,11 @@ Return the stable configuration value for this provider. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -100,63 +100,63 @@ Return the stable configuration value for this provider. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl Ord for TelemetryProvider` -
Ord for TelemetryProvider"}} />
+
Ord for TelemetryProvider"}} />
#### `cmp` -
cmp(&self, other: &TelemetryProvider) -> Ordering"}} />
+
cmp(&self, other: &TelemetryProvider) -> Ordering"}} />
#### `max` -
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
#### `min` -
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
#### `clamp` -
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
+
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
### `impl PartialEq for TelemetryProvider` -
PartialEq for TelemetryProvider"}} />
+
PartialEq for TelemetryProvider"}} />
#### `eq` -
eq(&self, other: &TelemetryProvider) -> bool"}} />
+
eq(&self, other: &TelemetryProvider) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl PartialOrd for TelemetryProvider` -
PartialOrd for TelemetryProvider"}} />
+
PartialOrd for TelemetryProvider"}} />
#### `partial_cmp` -
partial_cmp(&self, other: &TelemetryProvider) -> Option<Ordering>"}} />
+
partial_cmp(&self, other: &TelemetryProvider) -> Option<Ordering>"}} />
#### `lt` -
lt(&self, other: &Rhs) -> bool"}} />
+
lt(&self, other: &Rhs) -> bool"}} />
#### `le` -
le(&self, other: &Rhs) -> bool"}} />
+
le(&self, other: &Rhs) -> bool"}} />
#### `gt` -
gt(&self, other: &Rhs) -> bool"}} />
+
gt(&self, other: &Rhs) -> bool"}} />
#### `ge` -
ge(&self, other: &Rhs) -> bool"}} />
+
ge(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryProvider` @@ -164,16 +164,16 @@ Return the stable configuration value for this provider. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for TelemetryProvider` -
Copy for TelemetryProvider"}} />
+
Copy for TelemetryProvider"}} />
### `impl Eq for TelemetryProvider` -
Eq for TelemetryProvider"}} />
+
Eq for TelemetryProvider"}} />
### `impl StructuralPartialEq for TelemetryProvider` -
StructuralPartialEq for TelemetryProvider"}} />
+
StructuralPartialEq for TelemetryProvider"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx index fc493bae7..377b2a224 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx @@ -2,13 +2,13 @@ title: "Function load_adapter_descriptor" sidebar-title: "load_adapter_descriptor" description: "Load an adapter descriptor from JSON package metadata." -position: 71 +position: 72 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
AsRef<Path>,\n) -> Result<AdapterDescriptor>"}} />
+
AsRef<Path>,\n) -> Result<AdapterDescriptor>"}} />
Load an adapter descriptor from JSON package metadata. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx index 142fdba30..de11d4144 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx @@ -2,13 +2,13 @@ title: "Function load_adapter_target_descriptor" sidebar-title: "load_adapter_target_descriptor" description: "Load an Adapter Target Descriptor from declarative JSON metadata." -position: 72 +position: 73 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
AsRef<Path>,\n) -> Result<AdapterTargetDescriptor>"}} />
+
AsRef<Path>,\n) -> Result<AdapterTargetDescriptor>"}} />
Load an Adapter Target Descriptor from declarative JSON metadata. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx index 6bc8a973b..a6d607703 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx @@ -2,7 +2,7 @@ title: "Function resolve_run_plan_from_config" sidebar-title: "resolve_run_plan_from_config" description: "Resolve a typed NVIDIA NeMo Fabric config into a runnable plan." -position: 73 +position: 74 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index f031abd58..61f33ba88 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 145 +position: 146 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -53,6 +53,7 @@ NeMo Fabric config models and loading helpers. - [McpServerPlan](struct-mcpserverplan.mdx): Resolved MCP server exposure. - [MetadataConfig](struct-metadataconfig.mdx): Human-readable metadata. - [ModelConfig](struct-modelconfig.mdx): Model configuration. +- [PricingConfig](struct-pricingconfig.mdx): Central token-price configuration; never changes provider requests. - [RelayAtifConfig](struct-relayatifconfig.mdx): Relay ATIF export configuration. - [RelayAtofConfig](struct-relayatofconfig.mdx): Relay ATOF export configuration. - [RelayComponentConfig](struct-relaycomponentconfig.mdx): Generic NeMo Relay plugin component configuration. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx index bfc4a1fcd..47ec3308d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub metadata: MetadataConfig,\n    pub harness: Option<HarnessConfig>,\n    pub workflow: Option<WorkflowConfig>,\n    pub discovery: Option<DiscoveryConfig>,\n    pub models: BTreeMap<String, ModelConfig>,\n    pub instructions: Option<InstructionsConfig>,\n    pub runtime: RuntimeConfig,\n    pub environment: Option<EnvironmentConfig>,\n    pub tools: Option<ToolsConfig>,\n    pub skills: Option<SkillConfig>,\n    pub mcp: Option<McpConfig>,\n    pub telemetry: Option<TelemetryConfig>,\n    pub system_stats: Option<SystemStatsConfig>,\n    pub relay: Option<RelayConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub metadata: MetadataConfig,\n    pub harness: Option<HarnessConfig>,\n    pub workflow: Option<WorkflowConfig>,\n    pub discovery: Option<DiscoveryConfig>,\n    pub models: BTreeMap<String, ModelConfig>,\n    pub instructions: Option<InstructionsConfig>,\n    pub runtime: RuntimeConfig,\n    pub environment: Option<EnvironmentConfig>,\n    pub tools: Option<ToolsConfig>,\n    pub skills: Option<SkillConfig>,\n    pub mcp: Option<McpConfig>,\n    pub telemetry: Option<TelemetryConfig>,\n    pub system_stats: Option<SystemStatsConfig>,\n    pub relay: Option<RelayConfig>,\n    pub pricing: Option<PricingConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Versioned NVIDIA NeMo Fabric agent config. @@ -77,6 +77,10 @@ Invocation stats: omitted means all endpoint groups, explicit null disables. First-class NeMo Relay integration configuration. +### `pricing: Option` + +Optional catalog override for central token-price estimates. + ### `extensions: BTreeMap` Additive fields not yet recognized by this core version. @@ -85,23 +89,23 @@ Additive fields not yet recognized by this core version. ### `impl Clone for FabricConfig` -
Clone for FabricConfig"}} />
+
Clone for FabricConfig"}} />
#### `clone` -
clone(&self) -> FabricConfig"}} />
+
clone(&self) -> FabricConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for FabricConfig` -
Debug for FabricConfig"}} />
+
Debug for FabricConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for FabricConfig` @@ -109,7 +113,7 @@ Additive fields not yet recognized by this core version. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for FabricConfig` @@ -117,11 +121,11 @@ Additive fields not yet recognized by this core version. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -129,19 +133,19 @@ Additive fields not yet recognized by this core version. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for FabricConfig` -
PartialEq for FabricConfig"}} />
+
PartialEq for FabricConfig"}} />
#### `eq` -
eq(&self, other: &FabricConfig) -> bool"}} />
+
eq(&self, other: &FabricConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for FabricConfig` @@ -149,8 +153,8 @@ Additive fields not yet recognized by this core version. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for FabricConfig` -
StructuralPartialEq for FabricConfig"}} />
+
StructuralPartialEq for FabricConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx new file mode 100644 index 000000000..db976091d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx @@ -0,0 +1,106 @@ +--- +title: "Struct Pricing Config" +sidebar-title: "PricingConfig" +description: "Central token-price configuration; never changes provider requests." +position: 54 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<PathBuf>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+ +Central token-price configuration; never changes provider requests. + +## Fields + +### `catalog_path: Option` + +Relay-compatible v1 catalog, resolved relative to the config base directory. + +### `extensions: BTreeMap` + +Additive pricing configuration, preserved for future versions. + +## Trait Implementations + +### `impl Clone for PricingConfig` + +
Clone for PricingConfig"}} />
+ +#### `clone` + +
clone(&self) -> PricingConfig"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingConfig` + +
Debug for PricingConfig"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for PricingConfig` + +
Default for PricingConfig"}} />
+ +#### `default` + +
default() -> PricingConfig"}} />
+ +### `impl<'de> Deserialize<'de> for PricingConfig` + +
Deserialize<'de> for PricingConfig"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingConfig` + +
PricingConfig"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingConfig` + +
PartialEq for PricingConfig"}} />
+ +#### `eq` + +
eq(&self, other: &PricingConfig) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingConfig` + +
Serialize for PricingConfig"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for PricingConfig` + +
StructuralPartialEq for PricingConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx index 4a2989ac6..4968c698e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Atif Config" sidebar-title: "RelayAtifConfig" description: "Relay ATIF export configuration." -position: 38 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub agent_name: String,\n    pub agent_version: Option<String>,\n    pub model_name: Option<String>,\n    pub tool_definitions: Option<Vec<Value>>,\n    pub extra: Option<Value>,\n    pub output_directory: Option<PathBuf>,\n    pub filename_template: String,\n    pub storage: Option<Vec<RelayAtifStorageConfig>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub agent_name: String,\n    pub agent_version: Option<String>,\n    pub model_name: Option<String>,\n    pub tool_definitions: Option<Vec<Value>>,\n    pub extra: Option<Value>,\n    pub output_directory: Option<PathBuf>,\n    pub filename_template: String,\n    pub storage: Option<Vec<RelayAtifStorageConfig>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Relay ATIF export configuration. @@ -59,31 +59,31 @@ Additive ATIF fields. ### `impl Clone for RelayAtifConfig` -
Clone for RelayAtifConfig"}} />
+
Clone for RelayAtifConfig"}} />
#### `clone` -
clone(&self) -> RelayAtifConfig"}} />
+
clone(&self) -> RelayAtifConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtifConfig` -
Debug for RelayAtifConfig"}} />
+
Debug for RelayAtifConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtifConfig` -
Default for RelayAtifConfig"}} />
+
Default for RelayAtifConfig"}} />
#### `default` -
default() -> Self"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RelayAtifConfig` @@ -91,7 +91,7 @@ Additive ATIF fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtifConfig` @@ -99,11 +99,11 @@ Additive ATIF fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -111,19 +111,19 @@ Additive ATIF fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtifConfig` -
PartialEq for RelayAtifConfig"}} />
+
PartialEq for RelayAtifConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtifConfig) -> bool"}} />
+
eq(&self, other: &RelayAtifConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtifConfig` @@ -131,8 +131,8 @@ Additive ATIF fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtifConfig` -
StructuralPartialEq for RelayAtifConfig"}} />
+
StructuralPartialEq for RelayAtifConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx index 4ff5d7de9..a9819899e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Atof Config" sidebar-title: "RelayAtofConfig" description: "Relay ATOF export configuration." -position: 39 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub sinks: Vec<RelayAtofSinkConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub sinks: Vec<RelayAtofSinkConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Relay ATOF export configuration. @@ -31,31 +31,31 @@ Additive ATOF fields. ### `impl Clone for RelayAtofConfig` -
Clone for RelayAtofConfig"}} />
+
Clone for RelayAtofConfig"}} />
#### `clone` -
clone(&self) -> RelayAtofConfig"}} />
+
clone(&self) -> RelayAtofConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofConfig` -
Debug for RelayAtofConfig"}} />
+
Debug for RelayAtofConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofConfig` -
Default for RelayAtofConfig"}} />
+
Default for RelayAtofConfig"}} />
#### `default` -
default() -> RelayAtofConfig"}} />
+
default() -> RelayAtofConfig"}} />
### `impl<'de> Deserialize<'de> for RelayAtofConfig` @@ -63,7 +63,7 @@ Additive ATOF fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofConfig` @@ -71,11 +71,11 @@ Additive ATOF fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Additive ATOF fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofConfig` -
PartialEq for RelayAtofConfig"}} />
+
PartialEq for RelayAtofConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtofConfig) -> bool"}} />
+
eq(&self, other: &RelayAtofConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofConfig` @@ -103,8 +103,8 @@ Additive ATOF fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtofConfig` -
StructuralPartialEq for RelayAtofConfig"}} />
+
StructuralPartialEq for RelayAtofConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx index 56259d7a5..7c1330a78 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Component Config" sidebar-title: "RelayComponentConfig" description: "Generic NeMo Relay plugin component configuration." -position: 40 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub enabled: bool,\n    pub config: BTreeMap<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub enabled: bool,\n    pub config: BTreeMap<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Generic NeMo Relay plugin component configuration. @@ -35,23 +35,23 @@ Additive component fields. ### `impl Clone for RelayComponentConfig` -
Clone for RelayComponentConfig"}} />
+
Clone for RelayComponentConfig"}} />
#### `clone` -
clone(&self) -> RelayComponentConfig"}} />
+
clone(&self) -> RelayComponentConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayComponentConfig` -
Debug for RelayComponentConfig"}} />
+
Debug for RelayComponentConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayComponentConfig` @@ -59,7 +59,7 @@ Additive component fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayComponentConfig` @@ -67,11 +67,11 @@ Additive component fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive component fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayComponentConfig` -
PartialEq for RelayComponentConfig"}} />
+
PartialEq for RelayComponentConfig"}} />
#### `eq` -
eq(&self, other: &RelayComponentConfig) -> bool"}} />
+
eq(&self, other: &RelayComponentConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayComponentConfig` @@ -99,8 +99,8 @@ Additive component fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayComponentConfig` -
StructuralPartialEq for RelayComponentConfig"}} />
+
StructuralPartialEq for RelayComponentConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx index 570ef8d8e..f1aae1fe7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Config" sidebar-title: "RelayConfig" description: "NeMo Relay integration configuration." -position: 41 +position: 42 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Option<String>,\n    pub output_dir: Option<PathBuf>,\n    pub observability: Option<RelayObservabilityConfig>,\n    pub components: Vec<RelayComponentConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
Option<String>,\n    pub output_dir: Option<PathBuf>,\n    pub observability: Option<RelayObservabilityConfig>,\n    pub components: Vec<RelayComponentConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
NeMo Relay integration configuration. @@ -43,31 +43,31 @@ Additive Relay fields. ### `impl Clone for RelayConfig` -
Clone for RelayConfig"}} />
+
Clone for RelayConfig"}} />
#### `clone` -
clone(&self) -> RelayConfig"}} />
+
clone(&self) -> RelayConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayConfig` -
Debug for RelayConfig"}} />
+
Debug for RelayConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayConfig` -
Default for RelayConfig"}} />
+
Default for RelayConfig"}} />
#### `default` -
default() -> RelayConfig"}} />
+
default() -> RelayConfig"}} />
### `impl<'de> Deserialize<'de> for RelayConfig` @@ -75,7 +75,7 @@ Additive Relay fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayConfig` @@ -83,11 +83,11 @@ Additive Relay fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ Additive Relay fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayConfig` -
PartialEq for RelayConfig"}} />
+
PartialEq for RelayConfig"}} />
#### `eq` -
eq(&self, other: &RelayConfig) -> bool"}} />
+
eq(&self, other: &RelayConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayConfig` @@ -115,8 +115,8 @@ Additive Relay fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayConfig` -
StructuralPartialEq for RelayConfig"}} />
+
StructuralPartialEq for RelayConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx index 712c72795..753715ba6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Config Policy" sidebar-title: "RelayConfigPolicy" description: "Relay validation policy." -position: 42 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -31,31 +31,31 @@ Policy for unsupported values. ### `impl Clone for RelayConfigPolicy` -
Clone for RelayConfigPolicy"}} />
+
Clone for RelayConfigPolicy"}} />
#### `clone` -
clone(&self) -> RelayConfigPolicy"}} />
+
clone(&self) -> RelayConfigPolicy"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayConfigPolicy` -
Debug for RelayConfigPolicy"}} />
+
Debug for RelayConfigPolicy"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayConfigPolicy` -
Default for RelayConfigPolicy"}} />
+
Default for RelayConfigPolicy"}} />
#### `default` -
default() -> Self"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RelayConfigPolicy` @@ -63,7 +63,7 @@ Policy for unsupported values. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayConfigPolicy` @@ -71,11 +71,11 @@ Policy for unsupported values. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Policy for unsupported values. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayConfigPolicy` -
PartialEq for RelayConfigPolicy"}} />
+
PartialEq for RelayConfigPolicy"}} />
#### `eq` -
eq(&self, other: &RelayConfigPolicy) -> bool"}} />
+
eq(&self, other: &RelayConfigPolicy) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayConfigPolicy` @@ -103,8 +103,8 @@ Policy for unsupported values. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayConfigPolicy` -
StructuralPartialEq for RelayConfigPolicy"}} />
+
StructuralPartialEq for RelayConfigPolicy"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx index a01db8716..eb68b2061 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Observability Config" sidebar-title: "RelayObservabilityConfig" description: "NeMo Relay observability component configuration." -position: 43 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
u32,\n    pub atof: Option<RelayAtofConfig>,\n    pub atif: Option<RelayAtifConfig>,\n    pub opentelemetry: Option<RelayOpenTelemetryConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub enable_full_payloads: bool,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
u32,\n    pub atof: Option<RelayAtofConfig>,\n    pub atif: Option<RelayAtifConfig>,\n    pub opentelemetry: Option<RelayOpenTelemetryConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub enable_full_payloads: bool,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
NeMo Relay observability component configuration. @@ -47,31 +47,31 @@ Additive observability fields. ### `impl Clone for RelayObservabilityConfig` -
Clone for RelayObservabilityConfig"}} />
+
Clone for RelayObservabilityConfig"}} />
#### `clone` -
clone(&self) -> RelayObservabilityConfig"}} />
+
clone(&self) -> RelayObservabilityConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayObservabilityConfig` -
Debug for RelayObservabilityConfig"}} />
+
Debug for RelayObservabilityConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayObservabilityConfig` -
Default for RelayObservabilityConfig"}} />
+
Default for RelayObservabilityConfig"}} />
#### `default` -
default() -> Self"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RelayObservabilityConfig` @@ -79,7 +79,7 @@ Additive observability fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayObservabilityConfig` @@ -87,11 +87,11 @@ Additive observability fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -99,19 +99,19 @@ Additive observability fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayObservabilityConfig` -
PartialEq for RelayObservabilityConfig"}} />
+
PartialEq for RelayObservabilityConfig"}} />
#### `eq` -
eq(&self, other: &RelayObservabilityConfig) -> bool"}} />
+
eq(&self, other: &RelayObservabilityConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayObservabilityConfig` @@ -119,8 +119,8 @@ Additive observability fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayObservabilityConfig` -
StructuralPartialEq for RelayObservabilityConfig"}} />
+
StructuralPartialEq for RelayObservabilityConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx index 4bade1462..e46864ec7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Open Telemetry Config" sidebar-title: "RelayOpenTelemetryConfig" description: "Relay OpenTelemetry export section." -position: 44 +position: 45 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub endpoints: Vec<RelayOpenTelemetryEndpointConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub endpoints: Vec<RelayOpenTelemetryEndpointConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Relay OpenTelemetry export section. @@ -31,31 +31,31 @@ Additive OpenTelemetry section fields. ### `impl Clone for RelayOpenTelemetryConfig` -
Clone for RelayOpenTelemetryConfig"}} />
+
Clone for RelayOpenTelemetryConfig"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryConfig"}} />
+
clone(&self) -> RelayOpenTelemetryConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryConfig` -
Debug for RelayOpenTelemetryConfig"}} />
+
Debug for RelayOpenTelemetryConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayOpenTelemetryConfig` -
Default for RelayOpenTelemetryConfig"}} />
+
Default for RelayOpenTelemetryConfig"}} />
#### `default` -
default() -> RelayOpenTelemetryConfig"}} />
+
default() -> RelayOpenTelemetryConfig"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryConfig` @@ -63,7 +63,7 @@ Additive OpenTelemetry section fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryConfig` @@ -71,11 +71,11 @@ Additive OpenTelemetry section fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Additive OpenTelemetry section fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryConfig` -
PartialEq for RelayOpenTelemetryConfig"}} />
+
PartialEq for RelayOpenTelemetryConfig"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryConfig) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryConfig` @@ -103,8 +103,8 @@ Additive OpenTelemetry section fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryConfig` -
StructuralPartialEq for RelayOpenTelemetryConfig"}} />
+
StructuralPartialEq for RelayOpenTelemetryConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx index da8138f4e..aafe4a8a3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Open Telemetry Endpoint Config" sidebar-title: "RelayOpenTelemetryEndpointConfig" description: "One typed Relay OpenTelemetry OTLP destination." -position: 45 +position: 46 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
RelayOpenTelemetryEndpointType,\n    pub endpoint: String,\n    pub mark_projection: RelayOpenTelemetryMarkProjection,\n    pub mark_exclude_names: Vec<String>,\n    pub attribute_mappings: Vec<BTreeMap<String, String>>,\n    pub transport: RelayOtlpTransport,\n    pub service_name: String,\n    pub service_namespace: Option<String>,\n    pub service_version: Option<String>,\n    pub instrumentation_scope: String,\n    pub timeout_millis: u64,\n    pub headers: BTreeMap<String, String>,\n    pub header_env: BTreeMap<String, String>,\n    pub resource_attributes: BTreeMap<String, String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
RelayOpenTelemetryEndpointType,\n    pub endpoint: String,\n    pub mark_projection: RelayOpenTelemetryMarkProjection,\n    pub mark_exclude_names: Vec<String>,\n    pub attribute_mappings: Vec<BTreeMap<String, String>>,\n    pub transport: RelayOtlpTransport,\n    pub service_name: String,\n    pub service_namespace: Option<String>,\n    pub service_version: Option<String>,\n    pub instrumentation_scope: String,\n    pub timeout_millis: u64,\n    pub headers: BTreeMap<String, String>,\n    pub header_env: BTreeMap<String, String>,\n    pub resource_attributes: BTreeMap<String, String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
One typed Relay OpenTelemetry OTLP destination. @@ -79,23 +79,23 @@ Additive endpoint fields. ### `impl Clone for RelayOpenTelemetryEndpointConfig` -
Clone for RelayOpenTelemetryEndpointConfig"}} />
+
Clone for RelayOpenTelemetryEndpointConfig"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryEndpointConfig"}} />
+
clone(&self) -> RelayOpenTelemetryEndpointConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryEndpointConfig` -
Debug for RelayOpenTelemetryEndpointConfig"}} />
+
Debug for RelayOpenTelemetryEndpointConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryEndpointConfig` @@ -103,7 +103,7 @@ Additive endpoint fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryEndpointConfig` @@ -111,11 +111,11 @@ Additive endpoint fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -123,19 +123,19 @@ Additive endpoint fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryEndpointConfig` -
PartialEq for RelayOpenTelemetryEndpointConfig"}} />
+
PartialEq for RelayOpenTelemetryEndpointConfig"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryEndpointConfig) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryEndpointConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryEndpointConfig` @@ -143,8 +143,8 @@ Additive endpoint fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryEndpointConfig` -
StructuralPartialEq for RelayOpenTelemetryEndpointConfig"}} />
+
StructuralPartialEq for RelayOpenTelemetryEndpointConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx index 8e70ade88..a2f7b346d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx @@ -2,14 +2,14 @@ title: "Struct Resolve Context" sidebar-title: "ResolveContext" description: "Source context used when resolving an in-memory NeMo Fabric config." -position: 55 +position: 56 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n}"}} />
+
PathBuf,\n}"}} />
Source context used when resolving an in-memory NeMo Fabric config. @@ -27,7 +27,7 @@ Base directory used to resolve relative NeMo Fabric paths. #### `new` -
Into<PathBuf>) -> Self"}} />
+
Into<PathBuf>) -> Self"}} />
Build a context with an explicit base directory. @@ -35,36 +35,36 @@ Build a context with an explicit base directory. ### `impl Clone for ResolveContext` -
Clone for ResolveContext"}} />
+
Clone for ResolveContext"}} />
#### `clone` -
clone(&self) -> ResolveContext"}} />
+
clone(&self) -> ResolveContext"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolveContext` -
Debug for ResolveContext"}} />
+
Debug for ResolveContext"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl PartialEq for ResolveContext` -
PartialEq for ResolveContext"}} />
+
PartialEq for ResolveContext"}} />
#### `eq` -
eq(&self, other: &ResolveContext) -> bool"}} />
+
eq(&self, other: &ResolveContext) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl StructuralPartialEq for ResolveContext` -
StructuralPartialEq for ResolveContext"}} />
+
StructuralPartialEq for ResolveContext"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx index 7c1e65663..e246b640b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx @@ -2,14 +2,14 @@ title: "Struct Resolved Adapter Descriptor" sidebar-title: "ResolvedAdapterDescriptor" description: "Adapter descriptor selected for a run plan." -position: 56 +position: 57 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterDescriptor,\n}"}} />
+
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterDescriptor,\n}"}} />
Adapter descriptor selected for a run plan. @@ -27,23 +27,23 @@ Adapter-owned compatibility and capability metadata. ### `impl Clone for ResolvedAdapterDescriptor` -
Clone for ResolvedAdapterDescriptor"}} />
+
Clone for ResolvedAdapterDescriptor"}} />
#### `clone` -
clone(&self) -> ResolvedAdapterDescriptor"}} />
+
clone(&self) -> ResolvedAdapterDescriptor"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolvedAdapterDescriptor` -
Debug for ResolvedAdapterDescriptor"}} />
+
Debug for ResolvedAdapterDescriptor"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ResolvedAdapterDescriptor` @@ -51,7 +51,7 @@ Adapter-owned compatibility and capability metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ResolvedAdapterDescriptor` @@ -59,11 +59,11 @@ Adapter-owned compatibility and capability metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -71,19 +71,19 @@ Adapter-owned compatibility and capability metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ResolvedAdapterDescriptor` -
PartialEq for ResolvedAdapterDescriptor"}} />
+
PartialEq for ResolvedAdapterDescriptor"}} />
#### `eq` -
eq(&self, other: &ResolvedAdapterDescriptor) -> bool"}} />
+
eq(&self, other: &ResolvedAdapterDescriptor) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ResolvedAdapterDescriptor` @@ -91,8 +91,8 @@ Adapter-owned compatibility and capability metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ResolvedAdapterDescriptor` -
StructuralPartialEq for ResolvedAdapterDescriptor"}} />
+
StructuralPartialEq for ResolvedAdapterDescriptor"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx index 8adac3efc..3a0831bf1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx @@ -2,14 +2,14 @@ title: "Struct Resolved Adapter Target Descriptor" sidebar-title: "ResolvedAdapterTargetDescriptor" description: "Adapter target descriptor selected for a run plan." -position: 57 +position: 58 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterTargetDescriptor,\n}"}} />
+
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterTargetDescriptor,\n}"}} />
Adapter target descriptor selected for a run plan. @@ -27,23 +27,23 @@ Target-specific resolution and validation metadata. ### `impl Clone for ResolvedAdapterTargetDescriptor` -
Clone for ResolvedAdapterTargetDescriptor"}} />
+
Clone for ResolvedAdapterTargetDescriptor"}} />
#### `clone` -
clone(&self) -> ResolvedAdapterTargetDescriptor"}} />
+
clone(&self) -> ResolvedAdapterTargetDescriptor"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolvedAdapterTargetDescriptor` -
Debug for ResolvedAdapterTargetDescriptor"}} />
+
Debug for ResolvedAdapterTargetDescriptor"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ResolvedAdapterTargetDescriptor` @@ -51,7 +51,7 @@ Target-specific resolution and validation metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ResolvedAdapterTargetDescriptor` @@ -59,11 +59,11 @@ Target-specific resolution and validation metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -71,19 +71,19 @@ Target-specific resolution and validation metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ResolvedAdapterTargetDescriptor` -
PartialEq for ResolvedAdapterTargetDescriptor"}} />
+
PartialEq for ResolvedAdapterTargetDescriptor"}} />
#### `eq` -
eq(&self, other: &ResolvedAdapterTargetDescriptor) -> bool"}} />
+
eq(&self, other: &ResolvedAdapterTargetDescriptor) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ResolvedAdapterTargetDescriptor` @@ -91,8 +91,8 @@ Target-specific resolution and validation metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ResolvedAdapterTargetDescriptor` -
StructuralPartialEq for ResolvedAdapterTargetDescriptor"}} />
+
StructuralPartialEq for ResolvedAdapterTargetDescriptor"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx index e3c43e4e1..62cae0e6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx @@ -2,19 +2,23 @@ title: "Struct RunPlan" sidebar-title: "RunPlan" description: "Resolved NeMo Fabric run plan." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub base_dir: PathBuf,\n    pub config: FabricConfig,\n    pub agent_config: AgentConfig,\n    pub adapter_descriptor: Option<ResolvedAdapterDescriptor>,\n    pub adapter_target_descriptor: Option<ResolvedAdapterTargetDescriptor>,\n    pub resolution: Option<ResolutionStrategy>,\n    pub environment_plan: Option<EnvironmentPlan>,\n    pub capability_plan: CapabilityPlan,\n    pub capabilities: RuntimeCapabilities,\n    pub telemetry_plan: Option<TelemetryPlan>,\n}"}} />
+
Option<UserPriceCatalog>,\n    pub agent_name: String,\n    pub base_dir: PathBuf,\n    pub config: FabricConfig,\n    pub agent_config: AgentConfig,\n    pub adapter_descriptor: Option<ResolvedAdapterDescriptor>,\n    pub adapter_target_descriptor: Option<ResolvedAdapterTargetDescriptor>,\n    pub resolution: Option<ResolutionStrategy>,\n    pub environment_plan: Option<EnvironmentPlan>,\n    pub capability_plan: CapabilityPlan,\n    pub capabilities: RuntimeCapabilities,\n    pub telemetry_plan: Option<TelemetryPlan>,\n}"}} />
Resolved NeMo Fabric run plan. ## Fields +### `user_price_catalog: Option` + +Catalog bytes are frozen during planning. + ### `agent_name: String` Stable agent name. @@ -63,23 +67,23 @@ Resolved telemetry pass-through plan. ### `impl Clone for RunPlan` -
Clone for RunPlan"}} />
+
Clone for RunPlan"}} />
#### `clone` -
clone(&self) -> RunPlan"}} />
+
clone(&self) -> RunPlan"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RunPlan` -
Debug for RunPlan"}} />
+
Debug for RunPlan"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RunPlan` @@ -87,7 +91,7 @@ Resolved telemetry pass-through plan. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RunPlan` @@ -95,11 +99,11 @@ Resolved telemetry pass-through plan. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -107,19 +111,19 @@ Resolved telemetry pass-through plan. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RunPlan` -
PartialEq for RunPlan"}} />
+
PartialEq for RunPlan"}} />
#### `eq` -
eq(&self, other: &RunPlan) -> bool"}} />
+
eq(&self, other: &RunPlan) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RunPlan` @@ -127,8 +131,8 @@ Resolved telemetry pass-through plan. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RunPlan` -
StructuralPartialEq for RunPlan"}} />
+
StructuralPartialEq for RunPlan"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx index 0d972cb87..6df2d0128 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx @@ -2,14 +2,14 @@ title: "Struct Runtime Capabilities" sidebar-title: "RuntimeCapabilities" description: "Lifecycle behavior implemented by a resolved runtime path." -position: 59 +position: 60 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub streaming: bool,\n    pub updates: bool,\n    pub cancellation: bool,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub streaming: bool,\n    pub updates: bool,\n    pub cancellation: bool,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
Lifecycle behavior implemented by a resolved runtime path. @@ -39,31 +39,31 @@ Additional adapter-specific capability metadata. ### `impl Clone for RuntimeCapabilities` -
Clone for RuntimeCapabilities"}} />
+
Clone for RuntimeCapabilities"}} />
#### `clone` -
clone(&self) -> RuntimeCapabilities"}} />
+
clone(&self) -> RuntimeCapabilities"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RuntimeCapabilities` -
Debug for RuntimeCapabilities"}} />
+
Debug for RuntimeCapabilities"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RuntimeCapabilities` -
Default for RuntimeCapabilities"}} />
+
Default for RuntimeCapabilities"}} />
#### `default` -
default() -> RuntimeCapabilities"}} />
+
default() -> RuntimeCapabilities"}} />
### `impl<'de> Deserialize<'de> for RuntimeCapabilities` @@ -71,7 +71,7 @@ Additional adapter-specific capability metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RuntimeCapabilities` @@ -79,11 +79,11 @@ Additional adapter-specific capability metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -91,19 +91,19 @@ Additional adapter-specific capability metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RuntimeCapabilities` -
PartialEq for RuntimeCapabilities"}} />
+
PartialEq for RuntimeCapabilities"}} />
#### `eq` -
eq(&self, other: &RuntimeCapabilities) -> bool"}} />
+
eq(&self, other: &RuntimeCapabilities) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RuntimeCapabilities` @@ -111,8 +111,8 @@ Additional adapter-specific capability metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RuntimeCapabilities` -
StructuralPartialEq for RuntimeCapabilities"}} />
+
StructuralPartialEq for RuntimeCapabilities"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx index 2c89a11b2..662d07246 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Runtime Config" sidebar-title: "RuntimeConfig" description: "Invocation runtime contract." -position: 60 +position: 61 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Invocation runtime contract. @@ -43,23 +43,23 @@ Additive normalized runtime fields. ### `impl Clone for RuntimeConfig` -
Clone for RuntimeConfig"}} />
+
Clone for RuntimeConfig"}} />
#### `clone` -
clone(&self) -> RuntimeConfig"}} />
+
clone(&self) -> RuntimeConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RuntimeConfig` -
Debug for RuntimeConfig"}} />
+
Debug for RuntimeConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RuntimeConfig` @@ -67,7 +67,7 @@ Additive normalized runtime fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RuntimeConfig` @@ -75,11 +75,11 @@ Additive normalized runtime fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Additive normalized runtime fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RuntimeConfig` -
PartialEq for RuntimeConfig"}} />
+
PartialEq for RuntimeConfig"}} />
#### `eq` -
eq(&self, other: &RuntimeConfig) -> bool"}} />
+
eq(&self, other: &RuntimeConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RuntimeConfig` @@ -107,8 +107,8 @@ Additive normalized runtime fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RuntimeConfig` -
StructuralPartialEq for RuntimeConfig"}} />
+
StructuralPartialEq for RuntimeConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx index 47644f228..9fac093e1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Skill Config" sidebar-title: "SkillConfig" description: "Skill capability configuration." -position: 61 +position: 62 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<PathBuf>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
Vec<PathBuf>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Skill capability configuration. @@ -27,31 +27,31 @@ Additive skill fields. ### `impl Clone for SkillConfig` -
Clone for SkillConfig"}} />
+
Clone for SkillConfig"}} />
#### `clone` -
clone(&self) -> SkillConfig"}} />
+
clone(&self) -> SkillConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for SkillConfig` -
Debug for SkillConfig"}} />
+
Debug for SkillConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for SkillConfig` -
Default for SkillConfig"}} />
+
Default for SkillConfig"}} />
#### `default` -
default() -> SkillConfig"}} />
+
default() -> SkillConfig"}} />
### `impl<'de> Deserialize<'de> for SkillConfig` @@ -59,7 +59,7 @@ Additive skill fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for SkillConfig` @@ -67,11 +67,11 @@ Additive skill fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive skill fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for SkillConfig` -
PartialEq for SkillConfig"}} />
+
PartialEq for SkillConfig"}} />
#### `eq` -
eq(&self, other: &SkillConfig) -> bool"}} />
+
eq(&self, other: &SkillConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for SkillConfig` @@ -99,8 +99,8 @@ Additive skill fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for SkillConfig` -
StructuralPartialEq for SkillConfig"}} />
+
StructuralPartialEq for SkillConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx index 2e1b4af6b..7f1987b33 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Telemetry Config" sidebar-title: "TelemetryConfig" description: "Telemetry configuration." -position: 62 +position: 63 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<TelemetryProvider, TelemetryProviderConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
BTreeMap<TelemetryProvider, TelemetryProviderConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Telemetry configuration. @@ -27,23 +27,23 @@ Additive telemetry fields. ### `impl Clone for TelemetryConfig` -
Clone for TelemetryConfig"}} />
+
Clone for TelemetryConfig"}} />
#### `clone` -
clone(&self) -> TelemetryConfig"}} />
+
clone(&self) -> TelemetryConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryConfig` -
Debug for TelemetryConfig"}} />
+
Debug for TelemetryConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for TelemetryConfig` @@ -51,7 +51,7 @@ Additive telemetry fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryConfig` @@ -59,11 +59,11 @@ Additive telemetry fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -71,19 +71,19 @@ Additive telemetry fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for TelemetryConfig` -
PartialEq for TelemetryConfig"}} />
+
PartialEq for TelemetryConfig"}} />
#### `eq` -
eq(&self, other: &TelemetryConfig) -> bool"}} />
+
eq(&self, other: &TelemetryConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryConfig` @@ -91,8 +91,8 @@ Additive telemetry fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for TelemetryConfig` -
StructuralPartialEq for TelemetryConfig"}} />
+
StructuralPartialEq for TelemetryConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx index f12176331..20f00922c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx @@ -2,14 +2,14 @@ title: "Struct Telemetry Plan" sidebar-title: "TelemetryPlan" description: "Resolved telemetry plan." -position: 63 +position: 64 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<TelemetryProvider>,\n    pub relay_enabled: bool,\n    pub relay_project: Option<String>,\n    pub relay_output_dir: Option<PathBuf>,\n    pub relay_config: Option<Value>,\n    pub native_config: Option<Value>,\n    pub adapter_outputs: Vec<String>,\n}"}} />
+
Vec<TelemetryProvider>,\n    pub relay_enabled: bool,\n    pub relay_project: Option<String>,\n    pub relay_output_dir: Option<PathBuf>,\n    pub relay_config: Option<Value>,\n    pub native_config: Option<Value>,\n    pub adapter_outputs: Vec<String>,\n}"}} />
Resolved telemetry plan. @@ -47,23 +47,23 @@ Telemetry outputs declared by the selected adapter descriptor. ### `impl Clone for TelemetryPlan` -
Clone for TelemetryPlan"}} />
+
Clone for TelemetryPlan"}} />
#### `clone` -
clone(&self) -> TelemetryPlan"}} />
+
clone(&self) -> TelemetryPlan"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryPlan` -
Debug for TelemetryPlan"}} />
+
Debug for TelemetryPlan"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for TelemetryPlan` @@ -71,7 +71,7 @@ Telemetry outputs declared by the selected adapter descriptor. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryPlan` @@ -79,11 +79,11 @@ Telemetry outputs declared by the selected adapter descriptor. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -91,19 +91,19 @@ Telemetry outputs declared by the selected adapter descriptor. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for TelemetryPlan` -
PartialEq for TelemetryPlan"}} />
+
PartialEq for TelemetryPlan"}} />
#### `eq` -
eq(&self, other: &TelemetryPlan) -> bool"}} />
+
eq(&self, other: &TelemetryPlan) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryPlan` @@ -111,8 +111,8 @@ Telemetry outputs declared by the selected adapter descriptor. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for TelemetryPlan` -
StructuralPartialEq for TelemetryPlan"}} />
+
StructuralPartialEq for TelemetryPlan"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx index f683b0886..cd185c152 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Telemetry Provider Config" sidebar-title: "TelemetryProviderConfig" description: "Provider-specific telemetry configuration." -position: 65 +position: 66 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Option<Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
Option<Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Provider-specific telemetry configuration. @@ -27,31 +27,31 @@ Additive provider fields. ### `impl Clone for TelemetryProviderConfig` -
Clone for TelemetryProviderConfig"}} />
+
Clone for TelemetryProviderConfig"}} />
#### `clone` -
clone(&self) -> TelemetryProviderConfig"}} />
+
clone(&self) -> TelemetryProviderConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryProviderConfig` -
Debug for TelemetryProviderConfig"}} />
+
Debug for TelemetryProviderConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for TelemetryProviderConfig` -
Default for TelemetryProviderConfig"}} />
+
Default for TelemetryProviderConfig"}} />
#### `default` -
default() -> TelemetryProviderConfig"}} />
+
default() -> TelemetryProviderConfig"}} />
### `impl<'de> Deserialize<'de> for TelemetryProviderConfig` @@ -59,7 +59,7 @@ Additive provider fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryProviderConfig` @@ -67,11 +67,11 @@ Additive provider fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive provider fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for TelemetryProviderConfig` -
PartialEq for TelemetryProviderConfig"}} />
+
PartialEq for TelemetryProviderConfig"}} />
#### `eq` -
eq(&self, other: &TelemetryProviderConfig) -> bool"}} />
+
eq(&self, other: &TelemetryProviderConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryProviderConfig` @@ -99,8 +99,8 @@ Additive provider fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for TelemetryProviderConfig` -
StructuralPartialEq for TelemetryProviderConfig"}} />
+
StructuralPartialEq for TelemetryProviderConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx index b5e0b460f..1a22133b4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Tool Definition Config" sidebar-title: "ToolDefinitionConfig" description: "One named normalized tool or tool-group definition." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub ref: String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub ref: String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
One named normalized tool or tool-group definition. @@ -35,23 +35,23 @@ Additive definition fields. ### `impl Clone for ToolDefinitionConfig` -
Clone for ToolDefinitionConfig"}} />
+
Clone for ToolDefinitionConfig"}} />
#### `clone` -
clone(&self) -> ToolDefinitionConfig"}} />
+
clone(&self) -> ToolDefinitionConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ToolDefinitionConfig` -
Debug for ToolDefinitionConfig"}} />
+
Debug for ToolDefinitionConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ToolDefinitionConfig` @@ -59,7 +59,7 @@ Additive definition fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ToolDefinitionConfig` @@ -67,11 +67,11 @@ Additive definition fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive definition fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ToolDefinitionConfig` -
PartialEq for ToolDefinitionConfig"}} />
+
PartialEq for ToolDefinitionConfig"}} />
#### `eq` -
eq(&self, other: &ToolDefinitionConfig) -> bool"}} />
+
eq(&self, other: &ToolDefinitionConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ToolDefinitionConfig` @@ -99,8 +99,8 @@ Additive definition fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ToolDefinitionConfig` -
StructuralPartialEq for ToolDefinitionConfig"}} />
+
StructuralPartialEq for ToolDefinitionConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx index 64b2d6eec..655ac9d26 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Tools Config" sidebar-title: "ToolsConfig" description: "Harness-neutral tool capability configuration." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Harness-neutral tool capability configuration. @@ -35,31 +35,31 @@ Additive tool configuration fields. ### `impl Clone for ToolsConfig` -
Clone for ToolsConfig"}} />
+
Clone for ToolsConfig"}} />
#### `clone` -
clone(&self) -> ToolsConfig"}} />
+
clone(&self) -> ToolsConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ToolsConfig` -
Debug for ToolsConfig"}} />
+
Debug for ToolsConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for ToolsConfig` -
Default for ToolsConfig"}} />
+
Default for ToolsConfig"}} />
#### `default` -
default() -> ToolsConfig"}} />
+
default() -> ToolsConfig"}} />
### `impl<'de> Deserialize<'de> for ToolsConfig` @@ -67,7 +67,7 @@ Additive tool configuration fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ToolsConfig` @@ -75,11 +75,11 @@ Additive tool configuration fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Additive tool configuration fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ToolsConfig` -
PartialEq for ToolsConfig"}} />
+
PartialEq for ToolsConfig"}} />
#### `eq` -
eq(&self, other: &ToolsConfig) -> bool"}} />
+
eq(&self, other: &ToolsConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ToolsConfig` @@ -107,8 +107,8 @@ Additive tool configuration fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ToolsConfig` -
StructuralPartialEq for ToolsConfig"}} />
+
StructuralPartialEq for ToolsConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx index d2b51fe8b..f9466c489 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx @@ -2,14 +2,14 @@ title: "Struct Tools Plan" sidebar-title: "ToolsPlan" description: "Normalized tool policy for a run." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n}"}} />
+
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n}"}} />
Normalized tool policy for a run. @@ -31,31 +31,31 @@ Adapter-native tool names to block. ### `impl Clone for ToolsPlan` -
Clone for ToolsPlan"}} />
+
Clone for ToolsPlan"}} />
#### `clone` -
clone(&self) -> ToolsPlan"}} />
+
clone(&self) -> ToolsPlan"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ToolsPlan` -
Debug for ToolsPlan"}} />
+
Debug for ToolsPlan"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for ToolsPlan` -
Default for ToolsPlan"}} />
+
Default for ToolsPlan"}} />
#### `default` -
default() -> ToolsPlan"}} />
+
default() -> ToolsPlan"}} />
### `impl<'de> Deserialize<'de> for ToolsPlan` @@ -63,7 +63,7 @@ Adapter-native tool names to block. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ToolsPlan` @@ -71,11 +71,11 @@ Adapter-native tool names to block. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Adapter-native tool names to block. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ToolsPlan` -
PartialEq for ToolsPlan"}} />
+
PartialEq for ToolsPlan"}} />
#### `eq` -
eq(&self, other: &ToolsPlan) -> bool"}} />
+
eq(&self, other: &ToolsPlan) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ToolsPlan` @@ -103,8 +103,8 @@ Adapter-native tool names to block. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ToolsPlan` -
StructuralPartialEq for ToolsPlan"}} />
+
StructuralPartialEq for ToolsPlan"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx index 21beec0df..92f3dbaf8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Workflow Config" sidebar-title: "WorkflowConfig" description: "Registered workflow target and immutable construction settings." -position: 68 +position: 69 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Registered workflow target and immutable construction settings. @@ -31,23 +31,23 @@ Additive workflow fields. ### `impl Clone for WorkflowConfig` -
Clone for WorkflowConfig"}} />
+
Clone for WorkflowConfig"}} />
#### `clone` -
clone(&self) -> WorkflowConfig"}} />
+
clone(&self) -> WorkflowConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for WorkflowConfig` -
Debug for WorkflowConfig"}} />
+
Debug for WorkflowConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for WorkflowConfig` @@ -55,7 +55,7 @@ Additive workflow fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for WorkflowConfig` @@ -63,11 +63,11 @@ Additive workflow fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Additive workflow fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for WorkflowConfig` -
PartialEq for WorkflowConfig"}} />
+
PartialEq for WorkflowConfig"}} />
#### `eq` -
eq(&self, other: &WorkflowConfig) -> bool"}} />
+
eq(&self, other: &WorkflowConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for WorkflowConfig` @@ -95,8 +95,8 @@ Additive workflow fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for WorkflowConfig` -
StructuralPartialEq for WorkflowConfig"}} />
+
StructuralPartialEq for WorkflowConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx index a002d8fc3..6c99169ea 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Workflow Entrypoint Config" sidebar-title: "WorkflowEntrypointConfig" description: "Adapter-owned workflow entry point supplied by an Adapter Target Descriptor." -position: 69 +position: 70 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub ref: String,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub ref: String,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Adapter-owned workflow entry point supplied by an Adapter Target Descriptor. @@ -31,23 +31,23 @@ Additive workflow entry-point fields. ### `impl Clone for WorkflowEntrypointConfig` -
Clone for WorkflowEntrypointConfig"}} />
+
Clone for WorkflowEntrypointConfig"}} />
#### `clone` -
clone(&self) -> WorkflowEntrypointConfig"}} />
+
clone(&self) -> WorkflowEntrypointConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for WorkflowEntrypointConfig` -
Debug for WorkflowEntrypointConfig"}} />
+
Debug for WorkflowEntrypointConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for WorkflowEntrypointConfig` @@ -55,7 +55,7 @@ Additive workflow entry-point fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for WorkflowEntrypointConfig` @@ -63,11 +63,11 @@ Additive workflow entry-point fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Additive workflow entry-point fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for WorkflowEntrypointConfig` -
PartialEq for WorkflowEntrypointConfig"}} />
+
PartialEq for WorkflowEntrypointConfig"}} />
#### `eq` -
eq(&self, other: &WorkflowEntrypointConfig) -> bool"}} />
+
eq(&self, other: &WorkflowEntrypointConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for WorkflowEntrypointConfig` @@ -95,8 +95,8 @@ Additive workflow entry-point fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for WorkflowEntrypointConfig` -
StructuralPartialEq for WorkflowEntrypointConfig"}} />
+
StructuralPartialEq for WorkflowEntrypointConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx index e66041abb..3937f9f5f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx @@ -2,14 +2,14 @@ title: "Struct Workflow Target Spec" sidebar-title: "WorkflowTargetSpec" description: "Workflow-specific Adapter Target Descriptor fields." -position: 70 +position: 71 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
WorkflowEntrypointConfig,\n    pub settings_schema: Option<Map<String, Value>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
WorkflowEntrypointConfig,\n    pub settings_schema: Option<Map<String, Value>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Workflow-specific Adapter Target Descriptor fields. @@ -31,23 +31,23 @@ Additive workflow target fields. ### `impl Clone for WorkflowTargetSpec` -
Clone for WorkflowTargetSpec"}} />
+
Clone for WorkflowTargetSpec"}} />
#### `clone` -
clone(&self) -> WorkflowTargetSpec"}} />
+
clone(&self) -> WorkflowTargetSpec"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for WorkflowTargetSpec` -
Debug for WorkflowTargetSpec"}} />
+
Debug for WorkflowTargetSpec"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for WorkflowTargetSpec` @@ -55,7 +55,7 @@ Additive workflow target fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for WorkflowTargetSpec` @@ -63,11 +63,11 @@ Additive workflow target fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Additive workflow target fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for WorkflowTargetSpec` -
PartialEq for WorkflowTargetSpec"}} />
+
PartialEq for WorkflowTargetSpec"}} />
#### `eq` -
eq(&self, other: &WorkflowTargetSpec) -> bool"}} />
+
eq(&self, other: &WorkflowTargetSpec) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for WorkflowTargetSpec` @@ -95,8 +95,8 @@ Additive workflow target fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for WorkflowTargetSpec` -
StructuralPartialEq for WorkflowTargetSpec"}} />
+
StructuralPartialEq for WorkflowTargetSpec"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx index d8c0693d4..c00ba879b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx @@ -2,7 +2,7 @@ title: "Enum Doctor Status" sidebar-title: "DoctorStatus" description: "Diagnostic status." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,23 +43,23 @@ Check failed. ### `impl Clone for DoctorStatus` -
Clone for DoctorStatus"}} />
+
Clone for DoctorStatus"}} />
#### `clone` -
clone(&self) -> DoctorStatus"}} />
+
clone(&self) -> DoctorStatus"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for DoctorStatus` -
Debug for DoctorStatus"}} />
+
Debug for DoctorStatus"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for DoctorStatus` @@ -67,7 +67,7 @@ Check failed. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for DoctorStatus` @@ -75,11 +75,11 @@ Check failed. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Check failed. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for DoctorStatus` -
PartialEq for DoctorStatus"}} />
+
PartialEq for DoctorStatus"}} />
#### `eq` -
eq(&self, other: &DoctorStatus) -> bool"}} />
+
eq(&self, other: &DoctorStatus) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for DoctorStatus` @@ -107,16 +107,16 @@ Check failed. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for DoctorStatus` -
Copy for DoctorStatus"}} />
+
Copy for DoctorStatus"}} />
### `impl Eq for DoctorStatus` -
Eq for DoctorStatus"}} />
+
Eq for DoctorStatus"}} />
### `impl StructuralPartialEq for DoctorStatus` -
StructuralPartialEq for DoctorStatus"}} />
+
StructuralPartialEq for DoctorStatus"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx index bb8aa23e8..b97461157 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx @@ -2,7 +2,7 @@ title: "Function doctor_plan" sidebar-title: "doctor_plan" description: "Inspect a resolved run plan without mutating the environment." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index 6a1929704..e11cf4d12 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 146 +position: 147 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx index c64d5792e..6ce3b87f8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx @@ -2,14 +2,14 @@ title: "Struct Doctor Check" sidebar-title: "DoctorCheck" description: "Diagnostic check result." -position: 74 +position: 75 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub status: DoctorStatus,\n    pub message: String,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub status: DoctorStatus,\n    pub message: String,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
Diagnostic check result. @@ -35,23 +35,23 @@ Optional structured metadata. ### `impl Clone for DoctorCheck` -
Clone for DoctorCheck"}} />
+
Clone for DoctorCheck"}} />
#### `clone` -
clone(&self) -> DoctorCheck"}} />
+
clone(&self) -> DoctorCheck"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for DoctorCheck` -
Debug for DoctorCheck"}} />
+
Debug for DoctorCheck"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for DoctorCheck` @@ -59,7 +59,7 @@ Optional structured metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for DoctorCheck` @@ -67,11 +67,11 @@ Optional structured metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Optional structured metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for DoctorCheck` -
PartialEq for DoctorCheck"}} />
+
PartialEq for DoctorCheck"}} />
#### `eq` -
eq(&self, other: &DoctorCheck) -> bool"}} />
+
eq(&self, other: &DoctorCheck) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for DoctorCheck` @@ -99,8 +99,8 @@ Optional structured metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for DoctorCheck` -
StructuralPartialEq for DoctorCheck"}} />
+
StructuralPartialEq for DoctorCheck"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx index c7d65e76f..a7672fc4c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx @@ -2,14 +2,14 @@ title: "Struct Doctor Report" sidebar-title: "DoctorReport" description: "Diagnostic report for a resolved run plan." -position: 75 +position: 76 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub status: DoctorStatus,\n    pub checks: Vec<DoctorCheck>,\n}"}} />
+
String,\n    pub status: DoctorStatus,\n    pub checks: Vec<DoctorCheck>,\n}"}} />
Diagnostic report for a resolved run plan. @@ -31,23 +31,23 @@ Checks. ### `impl Clone for DoctorReport` -
Clone for DoctorReport"}} />
+
Clone for DoctorReport"}} />
#### `clone` -
clone(&self) -> DoctorReport"}} />
+
clone(&self) -> DoctorReport"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for DoctorReport` -
Debug for DoctorReport"}} />
+
Debug for DoctorReport"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for DoctorReport` @@ -55,7 +55,7 @@ Checks. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for DoctorReport` @@ -63,11 +63,11 @@ Checks. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Checks. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for DoctorReport` -
PartialEq for DoctorReport"}} />
+
PartialEq for DoctorReport"}} />
#### `eq` -
eq(&self, other: &DoctorReport) -> bool"}} />
+
eq(&self, other: &DoctorReport) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for DoctorReport` @@ -95,8 +95,8 @@ Checks. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for DoctorReport` -
StructuralPartialEq for DoctorReport"}} />
+
StructuralPartialEq for DoctorReport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 0d0570533..bed56c4a4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -2,14 +2,14 @@ title: "Enum Fabric Error" sidebar-title: "FabricError" description: "Errors raised by NeMo Fabric config loading and validation." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -33,7 +33,7 @@ Underlying path-resolution error. ### `PathNotFound(PathBuf)` -
PathBuf)"}} />
+
PathBuf)"}} />
The requested path does not exist. @@ -755,36 +755,36 @@ Underlying JSON error. ### `impl Debug for FabricError` -
Debug for FabricError"}} />
+
Debug for FabricError"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Display for FabricError` -
Display for FabricError"}} />
+
Display for FabricError"}} />
#### `fmt` -
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
### `impl Error for FabricError` -
Error for FabricError"}} />
+
Error for FabricError"}} />
#### `source` -
source(&self) -> Option<&(dyn Error + 'static)>"}} />
+
source(&self) -> Option<&(dyn Error + 'static)>"}} />
#### `description` -
description(&self) -> &str"}} />
+
description(&self) -> &str"}} />
#### `cause` -
cause(&self) -> Option<&dyn Error>"}} />
+
cause(&self) -> Option<&dyn Error>"}} />
#### `provide` -
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
+
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index 0e2ed4c29..0cf8cb514 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 147 +position: 148 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx index 0a91b0129..aaf087962 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx @@ -2,14 +2,14 @@ title: "Type Alias Result" sidebar-title: "Result" description: "Core NeMo Fabric result type." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Result<T, FabricError>;"}} />
+
Result<T, FabricError>;"}} />
Core NeMo Fabric result type. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index f5fedaa31..fb4cfff97 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,13 +2,13 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 155 +position: 157 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
str"}} />
+
str"}} />
Returns the crate version compiled into this build. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 0c4ece762..5a9180fa7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -66,6 +66,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use config::MetadataConfig;` - `pub use config::ModelConfig;` - `pub use config::OAuthTokenEndpointAuthMethod;` +- `pub use config::PricingConfig;` - `pub use config::ResolutionStrategy;` - `pub use config::ResolveContext;` - `pub use config::ResolvedAdapterDescriptor;` @@ -164,7 +165,8 @@ Core config and runtime contract for NeMo Fabric. - [doctor](doctor/index.mdx): Plan diagnostics for NeMo Fabric. - [error](error/index.mdx): Error types for NeMo Fabric core. - [model_usage](model-usage/index.mdx): Invocation-owned normalized counts and separately identified costs. -- [pricing](pricing/index.mdx): Persisted pricing-basis types for the stable model-usage result contract. +- [price_catalog](price-catalog/index.mdx): Model price catalogs for estimating token cost. +- [pricing](pricing/index.mdx): Catalog prices are estimates, separate from provider-reported amounts. - [resource_snapshot](resource-snapshot/index.mdx): Resource snapshots of a runtime's local adapter host and its descendants. - [runtime](runtime/index.mdx): Runtime invocation helpers. - [schema](schema/index.mdx): JSON Schema generation for the public NeMo Fabric contract. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx new file mode 100644 index 000000000..f6e285726 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx @@ -0,0 +1,15 @@ +--- +title: "Function normalize_model_usage" +sidebar-title: "normalize_model_usage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/fn-normalize-model-usage" +description: "Normalize supported evidence. Legacy usage remains raw rather than gaining a completeness guarantee merely because a newer core read it." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
AgentUsage,\n    catalog: Option<&UserPriceCatalog>,\n    bundled_pricing_allowed: bool,\n) -> Result<Option<ModelUsage>, AccountingError>"}} />
+ +Normalize supported evidence. Legacy usage remains raw rather than gaining a completeness guarantee merely because a newer core read it. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx index 7176f9683..6685d65d7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx @@ -3,7 +3,7 @@ title: "Module model_usage" sidebar-title: "model_usage" slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage" description: "Invocation-owned normalized counts and separately identified costs." -position: 148 +position: 149 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -24,3 +24,7 @@ Invocation-owned normalized counts and separately identified costs. ## Enums - [UsageDimension](enum-usagedimension.mdx): Dimensions whose complete values can be unavailable independently. + +## Functions + +- [normalize_model_usage](fn-normalize-model-usage.mdx): Normalize supported evidence. Legacy usage remains raw rather than gaining a completeness guarantee merely because a newer core read it. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx new file mode 100644 index 000000000..ded366199 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx @@ -0,0 +1,116 @@ +--- +title: "Enum Cache Read Accounting" +sidebar-title: "CacheReadAccounting" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-cachereadaccounting" +description: "Whether a provider's prompt token count includes cached reads." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum CacheReadAccounting { + IncludedInPromptTokens, + Separate, +} +``` + +Whether a provider's prompt token count includes cached reads. + +## Variants + +### `IncludedInPromptTokens` + +
+ +Prompt tokens include cached reads. + +### `Separate` + +
+ +Prompt tokens exclude cached reads. + +## Trait Implementations + +### `impl Clone for CacheReadAccounting` + +
Clone for CacheReadAccounting"}} />
+ +#### `clone` + +
clone(&self) -> CacheReadAccounting"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for CacheReadAccounting` + +
Debug for CacheReadAccounting"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for CacheReadAccounting` + +
Deserialize<'de> for CacheReadAccounting"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for CacheReadAccounting` + +
CacheReadAccounting"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for CacheReadAccounting` + +
PartialEq for CacheReadAccounting"}} />
+ +#### `eq` + +
eq(&self, other: &CacheReadAccounting) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for CacheReadAccounting` + +
Serialize for CacheReadAccounting"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for CacheReadAccounting` + +
Copy for CacheReadAccounting"}} />
+ +### `impl Eq for CacheReadAccounting` + +
Eq for CacheReadAccounting"}} />
+ +### `impl StructuralPartialEq for CacheReadAccounting` + +
StructuralPartialEq for CacheReadAccounting"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx new file mode 100644 index 000000000..69d57502d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx @@ -0,0 +1,129 @@ +--- +title: "Enum Pricing Catalog Error" +sidebar-title: "PricingCatalogError" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-pricingcatalogerror" +description: "Why a catalog is invalid. The messages match NeMo Relay's." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Error),\n    UnsupportedVersion {\n        version: u32,\n    },\n    DuplicateModelAlias {\n        model: String,\n    },\n    EmptyField {\n        entry_index: usize,\n        field: String,\n    },\n    InvalidRate {\n        entry_index: usize,\n        field: String,\n        value: f64,\n    },\n}"}} />
+ +Why a catalog is invalid. The messages match NeMo Relay's. + +## Variants + +### `Json(Error)` + +
Error)"}} />
+ +The catalog is not JSON of the catalog format. + +### `UnsupportedVersion` + +
+ +The catalog format version is not 1. + +#### Fields + +##### `version: u32` + +Version in the catalog. + +### `DuplicateModelAlias` + +
+ +Two entries or aliases of one provider name the same model. + +#### Fields + +##### `model: String` + +Normalized `provider/model` key. + +### `EmptyField` + +
+ +A required text field is blank. + +#### Fields + +##### `entry_index: usize` + +Zero-based entry index. + +##### `field: String` + +Field name. + +### `InvalidRate` + +
+ +A rate or a tier bound is out of range. + +#### Fields + +##### `entry_index: usize` + +Zero-based entry index. + +##### `field: String` + +Field name. + +##### `value: f64` + +Invalid value. + +## Trait Implementations + +### `impl Debug for PricingCatalogError` + +
Debug for PricingCatalogError"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Display for PricingCatalogError` + +
Display for PricingCatalogError"}} />
+ +#### `fmt` + +
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Error for PricingCatalogError` + +
Error for PricingCatalogError"}} />
+ +#### `source` + +
source(&self) -> Option<&(dyn Error + 'static)>"}} />
+ +#### `description` + +
description(&self) -> &str"}} />
+ +#### `cause` + +
cause(&self) -> Option<&dyn Error>"}} />
+ +#### `provide` + +
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
+ +### `impl From< Error> for PricingCatalogError` + +
From<Error> for PricingCatalogError"}} />
+ +#### `from` + +
from(source: Error) -> Self"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx new file mode 100644 index 000000000..d4ecc5b7d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx @@ -0,0 +1,138 @@ +--- +title: "Enum Pricing Unit" +sidebar-title: "PricingUnit" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-pricingunit" +description: "What a model price is charged per." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum PricingUnit { + PerToken, + PerRequest, + PerSecond, + GpuHour, +} +``` + +What a model price is charged per. + +## Variants + +### `PerToken` + +
+ +Per token. + +### `PerRequest` + +
+ +Per request. + +### `PerSecond` + +
+ +Per second. + +### `GpuHour` + +
+ +Per GPU hour, for self-hosted models. + +## Trait Implementations + +### `impl Clone for PricingUnit` + +
Clone for PricingUnit"}} />
+ +#### `clone` + +
clone(&self) -> PricingUnit"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingUnit` + +
Debug for PricingUnit"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for PricingUnit` + +
Default for PricingUnit"}} />
+ +#### `default` + +
default() -> PricingUnit"}} />
+ +### `impl<'de> Deserialize<'de> for PricingUnit` + +
Deserialize<'de> for PricingUnit"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingUnit` + +
PricingUnit"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingUnit` + +
PartialEq for PricingUnit"}} />
+ +#### `eq` + +
eq(&self, other: &PricingUnit) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingUnit` + +
Serialize for PricingUnit"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for PricingUnit` + +
Copy for PricingUnit"}} />
+ +### `impl Eq for PricingUnit` + +
Eq for PricingUnit"}} />
+ +### `impl StructuralPartialEq for PricingUnit` + +
StructuralPartialEq for PricingUnit"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx new file mode 100644 index 000000000..ce7fa57bc --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx @@ -0,0 +1,117 @@ +--- +title: "Enum Rate Schedule Application" +sidebar-title: "RateScheduleApplication" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-ratescheduleapplication" +description: "How a chosen rate tier applies to a request." +position: 10 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum RateScheduleApplication { + FullRequest, +} +``` + +How a chosen rate tier applies to a request. + +## Variants + +### `FullRequest` + +
+ +The tier's rates apply to every token of the request. + +## Trait Implementations + +### `impl Clone for RateScheduleApplication` + +
Clone for RateScheduleApplication"}} />
+ +#### `clone` + +
clone(&self) -> RateScheduleApplication"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RateScheduleApplication` + +
Debug for RateScheduleApplication"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for RateScheduleApplication` + +
Default for RateScheduleApplication"}} />
+ +#### `default` + +
default() -> RateScheduleApplication"}} />
+ +### `impl<'de> Deserialize<'de> for RateScheduleApplication` + +
Deserialize<'de> for RateScheduleApplication"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RateScheduleApplication` + +
RateScheduleApplication"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RateScheduleApplication` + +
PartialEq for RateScheduleApplication"}} />
+ +#### `eq` + +
eq(&self, other: &RateScheduleApplication) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RateScheduleApplication` + +
Serialize for RateScheduleApplication"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for RateScheduleApplication` + +
Copy for RateScheduleApplication"}} />
+ +### `impl Eq for RateScheduleApplication` + +
Eq for RateScheduleApplication"}} />
+ +### `impl StructuralPartialEq for RateScheduleApplication` + +
StructuralPartialEq for RateScheduleApplication"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx new file mode 100644 index 000000000..3fa7398d9 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx @@ -0,0 +1,107 @@ +--- +title: "Enum Token Rate Schedule" +sidebar-title: "TokenRateSchedule" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-tokenrateschedule" +description: "Rates that depend on the size of a request." +position: 11 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
RateScheduleApplication,\n        tiers: Vec<TokenRateTier>,\n    },\n}"}} />
+ +Rates that depend on the size of a request. + +## Variants + +### `PromptTokenThreshold` + +
+ +One tier's rates apply to a whole request, chosen by the request's prompt tokens. + +#### Fields + +##### `applies_to: RateScheduleApplication` + +How the chosen tier's rates apply. + +##### `tiers: Vec` + +Tiers in the order they are tried. The first tier whose bounds contain a request's prompt tokens applies to it. + +## Trait Implementations + +### `impl Clone for TokenRateSchedule` + +
Clone for TokenRateSchedule"}} />
+ +#### `clone` + +
clone(&self) -> TokenRateSchedule"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenRateSchedule` + +
Debug for TokenRateSchedule"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenRateSchedule` + +
Deserialize<'de> for TokenRateSchedule"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenRateSchedule` + +
TokenRateSchedule"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenRateSchedule` + +
PartialEq for TokenRateSchedule"}} />
+ +#### `eq` + +
eq(&self, other: &TokenRateSchedule) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenRateSchedule` + +
Serialize for TokenRateSchedule"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for TokenRateSchedule` + +
StructuralPartialEq for TokenRateSchedule"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx new file mode 100644 index 000000000..c9aa13b29 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx @@ -0,0 +1,34 @@ +--- +title: "Module price_catalog" +sidebar-title: "price_catalog" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog" +description: "Model price catalogs for estimating token cost." +position: 150 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Model price catalogs for estimating token cost. + +A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, model-only lookup uses the first matching entry in listed order. + +NeMo Fabric ships one catalog, embedded at build time, with the list prices of models that its presets, docs, and examples use. Each entry cites the provider pricing page it was taken from and the date it was checked. A caller can add a catalog of their own, whose entries take precedence. + +## Structs + +- [ModelPricing](struct-modelpricing.mdx): Prices of one model, in `currency` per million tokens. +- [PricingCatalog](struct-pricingcatalog.mdx): A model price catalog in the NeMo Relay model pricing catalog format, version 1. +- [PromptCachePricing](struct-promptcachepricing.mdx): How a provider counts cached prompt tokens in its usage reports. +- [TokenPricingRates](struct-tokenpricingrates.mdx): Token rates in currency units per million tokens. +- [TokenRateTier](struct-tokenratetier.mdx): Rates for requests whose prompt tokens fall within inclusive bounds. +- [UserPriceCatalog](struct-userpricecatalog.mdx): A price catalog the caller supplied, read and validated when the run plan is resolved. + +## Enums + +- [CacheReadAccounting](enum-cachereadaccounting.mdx): Whether a provider's prompt token count includes cached reads. +- [PricingCatalogError](enum-pricingcatalogerror.mdx): Why a catalog is invalid. The messages match NeMo Relay's. +- [PricingUnit](enum-pricingunit.mdx): What a model price is charged per. +- [RateScheduleApplication](enum-ratescheduleapplication.mdx): How a chosen rate tier applies to a request. +- [TokenRateSchedule](enum-tokenrateschedule.mdx): Rates that depend on the size of a request. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx new file mode 100644 index 000000000..814233c4e --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx @@ -0,0 +1,131 @@ +--- +title: "Struct Model Pricing" +sidebar-title: "ModelPricing" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-modelpricing" +description: "Prices of one model, in `currency` per million tokens." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub model_id: String,\n    pub aliases: Vec<String>,\n    pub currency: String,\n    pub unit: PricingUnit,\n    pub rates: Option<TokenPricingRates>,\n    pub rate_schedule: Option<TokenRateSchedule>,\n    pub prompt_cache: PromptCachePricing,\n    pub pricing_as_of: String,\n    pub pricing_source: String,\n}"}} />
+ +Prices of one model, in `currency` per million tokens. + +## Fields + +### `provider: String` + +Provider that sets the price, such as `anthropic` or `openai`. + +### `model_id: String` + +Canonical model identifier. + +### `aliases: Vec` + +Other identifiers of the same model, such as dated snapshots. + +### `currency: String` + +ISO 4217 currency of the rates. NeMo Fabric estimates cost only from `USD` entries. + +### `unit: PricingUnit` + +What a rate is charged per. NeMo Fabric estimates cost only from `per_token` entries. + +### `rates: Option` + +Flat rates. Ignored when `rate_schedule` is set. + +### `rate_schedule: Option` + +Rates that depend on the number of prompt tokens in a request. + +### `prompt_cache: PromptCachePricing` + +How the provider counts cached prompt tokens. NeMo Relay uses it to split provider usage; NeMo Fabric prices usage that is already normalized, so it does not read it. + +### `pricing_as_of: String` + +Date the price was taken from `pricing_source`, as `YYYY-MM-DD` in the bundled catalog. + +### `pricing_source: String` + +Where the price was taken from, such as the provider's pricing page. + +## Trait Implementations + +### `impl Clone for ModelPricing` + +
Clone for ModelPricing"}} />
+ +#### `clone` + +
clone(&self) -> ModelPricing"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ModelPricing` + +
Debug for ModelPricing"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for ModelPricing` + +
Deserialize<'de> for ModelPricing"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ModelPricing` + +
ModelPricing"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ModelPricing` + +
PartialEq for ModelPricing"}} />
+ +#### `eq` + +
eq(&self, other: &ModelPricing) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ModelPricing` + +
Serialize for ModelPricing"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for ModelPricing` + +
StructuralPartialEq for ModelPricing"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx new file mode 100644 index 000000000..01d5d4765 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx @@ -0,0 +1,113 @@ +--- +title: "Struct Pricing Catalog" +sidebar-title: "PricingCatalog" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-pricingcatalog" +description: "A model price catalog in the NeMo Relay model pricing catalog format, version 1." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
u32,\n    pub entries: Vec<ModelPricing>,\n}"}} />
+ +A model price catalog in the NeMo Relay model pricing catalog format, version 1. + +NeMo Fabric reads and writes the format unchanged, with one extra check: every rate must be below one million US dollars per million tokens. + +## Fields + +### `version: u32` + +Catalog format version. Must be `1`. + +### `entries: Vec` + +Price entries. Lookup uses the first entry that matches. + +## Implementations + +### `impl PricingCatalog` + +
PricingCatalog"}} />
+ +#### `from_json` + +
u8]) -> Result<Self, PricingCatalogError>"}} />
+ +Parse and validate a catalog. + +## Trait Implementations + +### `impl Clone for PricingCatalog` + +
Clone for PricingCatalog"}} />
+ +#### `clone` + +
clone(&self) -> PricingCatalog"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingCatalog` + +
Debug for PricingCatalog"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PricingCatalog` + +
Deserialize<'de> for PricingCatalog"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingCatalog` + +
PricingCatalog"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingCatalog` + +
PartialEq for PricingCatalog"}} />
+ +#### `eq` + +
eq(&self, other: &PricingCatalog) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingCatalog` + +
Serialize for PricingCatalog"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for PricingCatalog` + +
StructuralPartialEq for PricingCatalog"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx new file mode 100644 index 000000000..423b73286 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Prompt Cache Pricing" +sidebar-title: "PromptCachePricing" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-promptcachepricing" +description: "How a provider counts cached prompt tokens in its usage reports." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
CacheReadAccounting,\n}"}} />
+ +How a provider counts cached prompt tokens in its usage reports. + +## Fields + +### `read_accounting: CacheReadAccounting` + +Whether the provider's prompt token count includes cached reads. + +## Trait Implementations + +### `impl Clone for PromptCachePricing` + +
Clone for PromptCachePricing"}} />
+ +#### `clone` + +
clone(&self) -> PromptCachePricing"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PromptCachePricing` + +
Debug for PromptCachePricing"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PromptCachePricing` + +
Deserialize<'de> for PromptCachePricing"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PromptCachePricing` + +
PromptCachePricing"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PromptCachePricing` + +
PartialEq for PromptCachePricing"}} />
+ +#### `eq` + +
eq(&self, other: &PromptCachePricing) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PromptCachePricing` + +
Serialize for PromptCachePricing"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for PromptCachePricing` + +
Copy for PromptCachePricing"}} />
+ +### `impl Eq for PromptCachePricing` + +
Eq for PromptCachePricing"}} />
+ +### `impl StructuralPartialEq for PromptCachePricing` + +
StructuralPartialEq for PromptCachePricing"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx new file mode 100644 index 000000000..8c6ea1f0b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx @@ -0,0 +1,111 @@ +--- +title: "Struct Token Pricing Rates" +sidebar-title: "TokenPricingRates" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-tokenpricingrates" +description: "Token rates in currency units per million tokens." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
f64,\n    pub output_per_million: f64,\n    pub cache_read_per_million: Option<f64>,\n    pub cache_write_per_million: Option<f64>,\n}"}} />
+ +Token rates in currency units per million tokens. + +## Fields + +### `input_per_million: f64` + +Rate for input tokens that were neither read from nor written to the prompt cache. + +### `output_per_million: f64` + +Rate for output tokens, including reasoning tokens. + +### `cache_read_per_million: Option` + +Rate for input tokens read from the prompt cache. Without it, cached reads cannot be priced; set `0.0` when they are free. + +### `cache_write_per_million: Option` + +Rate for input tokens written to the prompt cache. Without it, cache writes cannot be priced; set `0.0` when they are free. + +## Trait Implementations + +### `impl Clone for TokenPricingRates` + +
Clone for TokenPricingRates"}} />
+ +#### `clone` + +
clone(&self) -> TokenPricingRates"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenPricingRates` + +
Debug for TokenPricingRates"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenPricingRates` + +
Deserialize<'de> for TokenPricingRates"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenPricingRates` + +
TokenPricingRates"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenPricingRates` + +
PartialEq for TokenPricingRates"}} />
+ +#### `eq` + +
eq(&self, other: &TokenPricingRates) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenPricingRates` + +
Serialize for TokenPricingRates"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for TokenPricingRates` + +
Copy for TokenPricingRates"}} />
+ +### `impl StructuralPartialEq for TokenPricingRates` + +
StructuralPartialEq for TokenPricingRates"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx new file mode 100644 index 000000000..4ff3cf607 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Token Rate Tier" +sidebar-title: "TokenRateTier" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-tokenratetier" +description: "Rates for requests whose prompt tokens fall within inclusive bounds." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub max_prompt_tokens: Option<u64>,\n    pub rates: TokenPricingRates,\n}"}} />
+ +Rates for requests whose prompt tokens fall within inclusive bounds. + +## Fields + +### `min_prompt_tokens: Option` + +Smallest number of prompt tokens in the tier, or no lower bound. + +### `max_prompt_tokens: Option` + +Largest number of prompt tokens in the tier, or no upper bound. + +### `rates: TokenPricingRates` + +Rates of the tier. + +## Trait Implementations + +### `impl Clone for TokenRateTier` + +
Clone for TokenRateTier"}} />
+ +#### `clone` + +
clone(&self) -> TokenRateTier"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenRateTier` + +
Debug for TokenRateTier"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenRateTier` + +
Deserialize<'de> for TokenRateTier"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenRateTier` + +
TokenRateTier"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenRateTier` + +
PartialEq for TokenRateTier"}} />
+ +#### `eq` + +
eq(&self, other: &TokenRateTier) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenRateTier` + +
Serialize for TokenRateTier"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for TokenRateTier` + +
Copy for TokenRateTier"}} />
+ +### `impl StructuralPartialEq for TokenRateTier` + +
StructuralPartialEq for TokenRateTier"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx new file mode 100644 index 000000000..63d5f8211 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx @@ -0,0 +1,115 @@ +--- +title: "Struct User Price Catalog" +sidebar-title: "UserPriceCatalog" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-userpricecatalog" +description: "A price catalog the caller supplied, read and validated when the run plan is resolved." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
PathBuf,\n    pub sha256: String,\n    pub catalog: PricingCatalog,\n}"}} />
+ +A price catalog the caller supplied, read and validated when the run plan is resolved. + +## Fields + +### `path: PathBuf` + +Absolute path the catalog was read from. + +### `sha256: String` + +`sha256:` followed by the lowercase hexadecimal SHA-256 of the file. + +### `catalog: PricingCatalog` + +The validated catalog. + +## Implementations + +### `impl UserPriceCatalog` + +
UserPriceCatalog"}} />
+ +#### `from_bytes` + +
PathBuf,\n    bytes: &[u8],\n) -> Result<Self, PricingCatalogError>"}} />
+ +Validate bytes and retain their exact catalog fingerprint. + +## Trait Implementations + +### `impl Clone for UserPriceCatalog` + +
Clone for UserPriceCatalog"}} />
+ +#### `clone` + +
clone(&self) -> UserPriceCatalog"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UserPriceCatalog` + +
Debug for UserPriceCatalog"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UserPriceCatalog` + +
Deserialize<'de> for UserPriceCatalog"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UserPriceCatalog` + +
UserPriceCatalog"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UserPriceCatalog` + +
PartialEq for UserPriceCatalog"}} />
+ +#### `eq` + +
eq(&self, other: &UserPriceCatalog) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UserPriceCatalog` + +
Serialize for UserPriceCatalog"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UserPriceCatalog` + +
StructuralPartialEq for UserPriceCatalog"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx index a62a00c14..4a1105670 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx @@ -1,15 +1,15 @@ --- title: "Module pricing" sidebar-title: "pricing" -description: "Persisted pricing-basis types for the stable model-usage result contract." -position: 149 +description: "Catalog prices are estimates, separate from provider-reported amounts." +position: 151 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -Persisted pricing-basis types for the stable model-usage result contract. +Catalog prices are estimates, separate from provider-reported amounts. ## Structs diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index 931c9f916..c0e90786d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -3,7 +3,7 @@ title: "Module resource_snapshot" sidebar-title: "resource_snapshot" slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" description: "Resource snapshots of a runtime's local adapter host and its descendants." -position: 150 +position: 152 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 12cea418d..2ff453e3b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 151 +position: 153 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index a194d1995..08a1933e1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 152 +position: 154 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx index 9356d0466..ca7d260a5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx @@ -3,7 +3,7 @@ title: "Module system_stats" sidebar-title: "system_stats" slug: "/reference/api/rust-library-reference/nemo-fabric-core/system_stats" description: "Invocation measurements collected by NeMo Fabric in the executing runtime." -position: 153 +position: 155 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx index df06788a3..a942c30e8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx @@ -3,7 +3,7 @@ title: "Module usage_accounting" sidebar-title: "usage_accounting" slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting" description: "Validated accounting evidence carried through the existing usage extension." -position: 154 +position: 156 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index d337ed45c..88b07bc25 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1155,7 +1155,7 @@ Important fields: | `status` | Terminal invocation status such as success, failure, or cancellation. | | `output` | Harness output normalized to the configured output schema. | | `error` | Structured failure metadata when available. | -| `model_usage` | Optional normalized complete counts, observations, gaps and provider-reported cost. Catalog estimates remain unavailable. | +| `model_usage` | Optional normalized complete counts, observations, gaps and separately reported/estimated cost. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | | `system_stats` | Optional `InvocationSystemStats`: monotonic adapter exchange duration, before/after observations with availability, and an optional CPU counter difference estimate. | | `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. | @@ -1212,7 +1212,7 @@ if result.model_usage is not None: print(result.model_usage.input_tokens) # Complete count, or None. print(result.model_usage.observed.input_tokens) # Available observations. print(result.model_usage.gaps) # Affected dimensions/reasons. - print(result.model_usage.cost.reported.amount_nano_usd) + print(result.model_usage.cost.estimated.amount_nano_usd) summary = summarize_usage( [result], expected=[(result.runtime_id, result.invocation_id)] @@ -1231,10 +1231,16 @@ extensions still fail the adapter contract. Legacy usage without supported accounting remains available in `result.usage` and is not certified complete by `summarize_usage`. -`cost.reported` contains only provider-reported amounts. Complete and observed -amounts use integer nano-USD. Catalog estimates remain unavailable in this slice; -`cost.estimated` has no amounts and no typed pricing setting is exposed. The stable -result shape retains optional pricing-basis fields for persisted evidence. +`cost.reported` contains only provider-reported amounts; `cost.estimated` contains +catalog estimates. Both distinguish `amount_nano_usd` from `observed_nano_usd`. +Estimates retain their pricing basis and distinct selected rate snapshots. +Configure `PricingConfig(catalog_path="prices.json")` for a Relay-compatible +catalog override; planning freezes its bytes. An explicitly established provider must match the catalog entry's provider, including aliases and shorter model paths. A different provider's entry cannot price that source solely because the model name matches. When no provider identity is available, an explicit user catalog can match the model name alone. Bundled provider prices apply only +when the configured route establishes the native provider. Unknown model, +cache-price or request-tier evidence affects estimates without erasing tokens. +Nonzero Anthropic cache writes currently lack expiry-price evidence, so their +complete estimate remains unknown; independently priceable parts remain observed. +Known zero writes need no expiry evidence. Estimates are not invoices. `summarize_usage` requires an explicit expected invocation inventory. It retains missing results in coverage, keeps complete totals separate from observed sums, diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index cfd2150de..2cb00c651 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -558,6 +558,20 @@ } ] }, + "PricingConfig": { + "additionalProperties": true, + "description": "Central token-price configuration; never changes provider requests.", + "properties": { + "catalog_path": { + "description": "Relay-compatible v1 catalog, resolved relative to the config base directory.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -1611,6 +1625,17 @@ "description": "Named model roles.", "type": "object" }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/PricingConfig" + }, + { + "type": "null" + } + ], + "description": "Optional catalog override for central token-price estimates." + }, "relay": { "anyOf": [ { diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index e183bca75..ec8bcc2c7 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -894,6 +894,21 @@ ], "type": "object" }, + "CacheReadAccounting": { + "description": "Whether a provider's prompt token count includes cached reads.", + "oneOf": [ + { + "const": "included_in_prompt_tokens", + "description": "Prompt tokens include cached reads.", + "type": "string" + }, + { + "const": "separate", + "description": "Prompt tokens exclude cached reads.", + "type": "string" + } + ] + }, "CapabilityKind": { "description": "Capability kind.", "oneOf": [ @@ -1341,6 +1356,17 @@ "description": "Named model roles.", "type": "object" }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/PricingConfig" + }, + { + "type": "null" + } + ], + "description": "Optional catalog override for central token-price estimates." + }, "relay": { "anyOf": [ { @@ -1925,6 +1951,80 @@ ], "type": "object" }, + "ModelPricing": { + "additionalProperties": false, + "description": "Prices of one model, in `currency` per million tokens.", + "properties": { + "aliases": { + "default": [], + "description": "Other identifiers of the same model, such as dated snapshots.", + "items": { + "type": "string" + }, + "type": "array" + }, + "currency": { + "default": "USD", + "description": "ISO 4217 currency of the rates. NeMo Fabric estimates cost only from\n`USD` entries.", + "type": "string" + }, + "model_id": { + "description": "Canonical model identifier.", + "type": "string" + }, + "pricing_as_of": { + "description": "Date the price was taken from `pricing_source`, as `YYYY-MM-DD` in\nthe bundled catalog.", + "type": "string" + }, + "pricing_source": { + "description": "Where the price was taken from, such as the provider's pricing page.", + "type": "string" + }, + "prompt_cache": { + "$ref": "#/$defs/PromptCachePricing", + "description": "How the provider counts cached prompt tokens. NeMo Relay uses it to\nsplit provider usage; NeMo Fabric prices usage that is already\nnormalized, so it does not read it." + }, + "provider": { + "description": "Provider that sets the price, such as `anthropic` or `openai`.", + "type": "string" + }, + "rate_schedule": { + "anyOf": [ + { + "$ref": "#/$defs/TokenRateSchedule" + }, + { + "type": "null" + } + ], + "description": "Rates that depend on the number of prompt tokens in a request." + }, + "rates": { + "anyOf": [ + { + "$ref": "#/$defs/TokenPricingRates" + }, + { + "type": "null" + } + ], + "description": "Flat rates. Ignored when `rate_schedule` is set." + }, + "unit": { + "$ref": "#/$defs/PricingUnit", + "default": "per_token", + "description": "What a rate is charged per. NeMo Fabric estimates cost only from\n`per_token` entries." + } + }, + "required": [ + "provider", + "model_id", + "prompt_cache", + "pricing_as_of", + "pricing_source" + ], + "type": "object" + }, "OAuthTokenEndpointAuthMethod": { "description": "OAuth client authentication method used at the token endpoint.", "oneOf": [ @@ -1945,6 +2045,93 @@ } ] }, + "PricingCatalog": { + "additionalProperties": false, + "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", + "properties": { + "entries": { + "description": "Price entries. Lookup uses the first entry that matches.", + "items": { + "$ref": "#/$defs/ModelPricing" + }, + "type": "array" + }, + "version": { + "description": "Catalog format version. Must be `1`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "version", + "entries" + ], + "type": "object" + }, + "PricingConfig": { + "additionalProperties": true, + "description": "Central token-price configuration; never changes provider requests.", + "properties": { + "catalog_path": { + "description": "Relay-compatible v1 catalog, resolved relative to the config base directory.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "PricingUnit": { + "description": "What a model price is charged per.", + "oneOf": [ + { + "const": "per_token", + "description": "Per token.", + "type": "string" + }, + { + "const": "per_request", + "description": "Per request.", + "type": "string" + }, + { + "const": "per_second", + "description": "Per second.", + "type": "string" + }, + { + "const": "gpu_hour", + "description": "Per GPU hour, for self-hosted models.", + "type": "string" + } + ] + }, + "PromptCachePricing": { + "additionalProperties": false, + "description": "How a provider counts cached prompt tokens in its usage reports.", + "properties": { + "read_accounting": { + "$ref": "#/$defs/CacheReadAccounting", + "description": "Whether the provider's prompt token count includes cached reads." + } + }, + "required": [ + "read_accounting" + ], + "type": "object" + }, + "RateScheduleApplication": { + "description": "How a chosen rate tier applies to a request.", + "oneOf": [ + { + "const": "full_request", + "description": "The tier's rates apply to every token of the request.", + "type": "string" + } + ] + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -2985,6 +3172,107 @@ }, "type": "object" }, + "TokenPricingRates": { + "additionalProperties": false, + "description": "Token rates in currency units per million tokens.", + "properties": { + "cache_read_per_million": { + "description": "Rate for input tokens read from the prompt cache. Without it, cached\nreads cannot be priced; set `0.0` when they are free.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "cache_write_per_million": { + "description": "Rate for input tokens written to the prompt cache. Without it, cache\nwrites cannot be priced; set `0.0` when they are free.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "input_per_million": { + "description": "Rate for input tokens that were neither read from nor written to the\nprompt cache.", + "format": "double", + "type": "number" + }, + "output_per_million": { + "description": "Rate for output tokens, including reasoning tokens.", + "format": "double", + "type": "number" + } + }, + "required": [ + "input_per_million", + "output_per_million" + ], + "type": "object" + }, + "TokenRateSchedule": { + "description": "Rates that depend on the size of a request.", + "oneOf": [ + { + "additionalProperties": false, + "description": "One tier's rates apply to a whole request, chosen by the request's\nprompt tokens.", + "properties": { + "applies_to": { + "$ref": "#/$defs/RateScheduleApplication", + "default": "full_request", + "description": "How the chosen tier's rates apply." + }, + "tiers": { + "description": "Tiers in the order they are tried. The first tier whose bounds\ncontain a request's prompt tokens applies to it.", + "items": { + "$ref": "#/$defs/TokenRateTier" + }, + "type": "array" + }, + "type": { + "const": "prompt_token_threshold", + "type": "string" + } + }, + "required": [ + "type", + "tiers" + ], + "type": "object" + } + ] + }, + "TokenRateTier": { + "additionalProperties": false, + "description": "Rates for requests whose prompt tokens fall within inclusive bounds.", + "properties": { + "max_prompt_tokens": { + "description": "Largest number of prompt tokens in the tier, or no upper bound.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "min_prompt_tokens": { + "description": "Smallest number of prompt tokens in the tier, or no lower bound.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "rates": { + "$ref": "#/$defs/TokenPricingRates", + "description": "Rates of the tier." + } + }, + "required": [ + "rates" + ], + "type": "object" + }, "ToolDefinitionConfig": { "additionalProperties": true, "description": "One named normalized tool or tool-group definition.", @@ -3074,6 +3362,29 @@ }, "type": "object" }, + "UserPriceCatalog": { + "description": "A price catalog the caller supplied, read and validated when the run plan\nis resolved.", + "properties": { + "catalog": { + "$ref": "#/$defs/PricingCatalog", + "description": "The validated catalog." + }, + "path": { + "description": "Absolute path the catalog was read from.", + "type": "string" + }, + "sha256": { + "description": "`sha256:` followed by the lowercase hexadecimal SHA-256 of the file.", + "type": "string" + } + }, + "required": [ + "path", + "sha256", + "catalog" + ], + "type": "object" + }, "WorkflowConfig": { "additionalProperties": true, "description": "Registered workflow target and immutable construction settings.", @@ -3235,6 +3546,17 @@ } ], "description": "Resolved telemetry pass-through plan." + }, + "user_price_catalog": { + "anyOf": [ + { + "$ref": "#/$defs/UserPriceCatalog" + }, + { + "type": "null" + } + ], + "description": "Catalog bytes are frozen during planning." } }, "required": [ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index 9d817384f..f19f256ee 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -4,6 +4,7 @@ """Python SDK surface for NeMo Fabric.""" from nemo_fabric.accounting import summarize_usage +from nemo_fabric.models import PricingConfig from nemo_fabric.types import ModelUsage from nemo_fabric.types import PerModelUsage from nemo_fabric.types import PricingBasis @@ -93,6 +94,7 @@ __all__ = [ "summarize_usage", + "PricingConfig", "ModelUsage", "PerModelUsage", "PricingBasis", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index 7a9bbd627..fa3f6c6ce 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -1118,6 +1118,19 @@ def _validate_collect(cls, value: list[str]) -> list[str]: return value +class PricingConfig(FabricBaseModel): + """Frozen Relay-compatible price-catalog override, read during planning.""" + + catalog_path: str | Path | None = None + + @field_validator("catalog_path") + @classmethod + def _validate_catalog_path(cls, value: str | Path | None) -> str | Path | None: + if value is not None and not str(value).strip(): + raise ValueError("pricing catalog path must not be empty") + return value + + class FabricConfig(FabricBaseModel): """SDK-facing typed NeMo Fabric agent configuration. @@ -1141,6 +1154,7 @@ class FabricConfig(FabricBaseModel): system_stats: SystemStatsConfig | None = Field(default_factory=SystemStatsConfig) telemetry: TelemetryConfig | None = None relay: RelayConfig | None = None + pricing: PricingConfig | None = None tools: ToolsConfig | None = None @model_serializer(mode="wrap") diff --git a/tests/python/conftest.py b/tests/python/conftest.py index ba0e484c1..40a104a58 100644 --- a/tests/python/conftest.py +++ b/tests/python/conftest.py @@ -6,11 +6,16 @@ from pathlib import Path import pytest -from nemo_fabric import Fabric, FabricConfig +from nemo_fabric import Fabric, FabricConfig, PricingConfig + + +@pytest.fixture(name="accounting_catalog") +def accounting_catalog_fixture(): + return None @pytest.fixture(name="accounting_runtime") -async def accounting_runtime_fixture(tmp_path): +async def accounting_runtime_fixture(tmp_path, accounting_catalog): """Normalize controlled adapter evidence through the native public runtime.""" root = Path(__file__).resolve().parents[2] accounting_schema = json.loads( @@ -68,6 +73,10 @@ async def accounting_runtime_fixture(tmp_path): "runtime": {"artifacts": "./artifacts"}, } ) + if accounting_catalog is not None: + catalog_path = tmp_path / "catalog.json" + catalog_path.write_text(json.dumps(accounting_catalog)) + config.pricing = PricingConfig(catalog_path=catalog_path) async with await Fabric().start_runtime(config, base_dir=tmp_path) as runtime: yield runtime diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py index c305b4b33..dca8697bb 100644 --- a/tests/python/test_usage_reducer_parity.py +++ b/tests/python/test_usage_reducer_parity.py @@ -104,8 +104,39 @@ async def test_actual_calls_with_mixed_conventions_recover_complete_counts_in_co @pytest.mark.integration -async def test_scoped_request_gap_preserves_cache_uncertainty_in_core( - reduced, accounting_runtime +@pytest.mark.parametrize( + "accounting_catalog, complete_cost", + [ + ( + { + "version": 1, + "entries": [ + { + "provider": "openai", + "model_id": "model", + "currency": "USD", + "unit": "per_token", + "rates": { + "input_per_million": 1.0, + "output_per_million": 2.0, + "cache_read_per_million": cache_rate, + "cache_write_per_million": 1.25, + }, + "prompt_cache": { + "read_accounting": "included_in_prompt_tokens" + }, + "pricing_as_of": "2026-10-21", + "pricing_source": "test fixture", + } + ], + }, + complete_cost, + ) + for cache_rate, complete_cost in [(0.1, None), (1.0, 110_000)] + ], +) +async def test_scoped_request_gap_preserves_cache_uncertainty_and_independent_pricing( + reduced, accounting_runtime, complete_cost ): index = next( i @@ -132,3 +163,4 @@ async def test_scoped_request_gap_preserves_cache_uncertainty_in_core( assert usage.cached_input_tokens is None assert usage.observed.cached_input_tokens == 20 assert usage.models[0].cached_input_tokens is None + assert usage.cost.estimated.amount_nano_usd == complete_cost From cbb06993ec9db38d0b7e2dfdd072c94eecfd16ae Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 14:59:52 -0700 Subject: [PATCH 073/102] fix: validate recovered usage and optional pricing rates Signed-off-by: mnajafian-nv --- .../generated/usage-accounting-vectors.ts | 180 ++++++++++++++++++ crates/fabric-core/src/runtime.rs | 7 +- crates/fabric-core/src/usage_normalization.rs | 23 +++ docs/adapter-contract/tutorials/results.md | 2 +- .../nemo_fabric.types.md | 1 + docs/sdk/python.mdx | 3 - schemas/SCHEMA.md | 9 +- .../src/nemo_fabric/types.py | 7 +- tests/adapters/test_deepagents_accounting.py | 4 +- tests/docs/test_python_api_docs.py | 4 +- .../usage_accounting/conformance.json | 180 ++++++++++++++++++ tests/python/test_model_usage.py | 22 ++- 12 files changed, 426 insertions(+), 16 deletions(-) diff --git a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts index 05a913834..cbc90c193 100644 --- a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts +++ b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts @@ -493,6 +493,186 @@ export const accountingVectors = [ "observed": 20 } } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } } ] satisfies { name: string; diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 2b7a780b4..b03484eee 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -2324,7 +2324,7 @@ fn run_local_host_invocation_with_timeout( let model_usage = match agent_result .usage .as_ref() - .map(|raw| crate::model_usage::normalize_model_usage(raw)) + .map(crate::model_usage::normalize_model_usage) .transpose() { Ok(usage) => usage.flatten(), @@ -6114,7 +6114,7 @@ for line in sys.stdin: #[test] fn local_host_returns_flat_supported_accounting_with_the_normalized_result() { - let (_root, plan) = local_host_plan("accounted_usage_result"); + let (root, plan) = local_host_plan("accounted_usage_result"); let result = run_plan(&plan, RunRequest::text("accounted")).expect("normalized result"); assert_eq!(result.status, RunStatus::Succeeded); let usage = result.model_usage.expect("supported accounting"); @@ -6129,6 +6129,7 @@ for line in sys.stdin: assert_eq!(usage.cost.estimated.observed_nano_usd, None); assert!(usage.cost.pricing.is_none()); assert!(usage.cost.applied_rates.is_empty()); + let _ = fs::remove_dir_all(root); } #[test] @@ -6688,7 +6689,7 @@ for line in sys.stdin: let diagnostic = serde_json::to_string(events[0]).unwrap(); assert!(!diagnostic.contains("private-accounting-identity")); assert!(!diagnostic.contains("request totals")); - let _ = root; + let _ = fs::remove_dir_all(root); } } } diff --git a/crates/fabric-core/src/usage_normalization.rs b/crates/fabric-core/src/usage_normalization.rs index f236d7623..f8924207e 100644 --- a/crates/fabric-core/src/usage_normalization.rs +++ b/crates/fabric-core/src/usage_normalization.rs @@ -348,6 +348,7 @@ pub fn describe_model_usage(usage: &AgentUsage) -> Result Result Result input) || peak.is_ok_and(|peak| peak > input) + }) { + counts[INPUT] = Err("inconsistent_usage"); + observed[INPUT] = Err("inconsistent_usage"); + } + let reasoning = counts[UsageDimension::ReasoningTokens as usize]; + if counts[OUTPUT].is_ok_and(|output| reasoning.is_ok_and(|reasoning| reasoning > output)) { + counts[OUTPUT] = Err("inconsistent_usage"); + observed[OUTPUT] = Err("inconsistent_usage"); + } +} + // An unconvertible aggregate must not erase a qualified model/request subset. // Contradictions and overflow stay errors; these reasons only lack a usable // normalized observation. Recovery never certifies a complete invocation. diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 42cd74b99..f361e00ba 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -108,7 +108,7 @@ A model aggregate is not a single request. No adapter should fabricate call boundaries, hide execution features or change retry settings to make usage look complete. Costs computed from a price table never become `cost_usd`. -## 2. Separate Failure Classes +### 2. Separate Failure Classes Use these failure classes consistently: diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 57ba50589..83cdfab2b 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -2634,6 +2634,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - `error`: Structured failure, or ``None`` on success. - `usage`: Normalized invocation usage, or ``None`` when unavailable. + - `model_usage`: Complete counts, observations and accounting gaps, or ``None`` when supported accounting evidence is unavailable. - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. - `system_stats`: Invocation duration and endpoint observations, or None for older results or execution backends without collection. - `artifacts`: Normalized artifact manifest. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index d337ed45c..f55fe8ba9 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1194,9 +1194,6 @@ the SDK raises a `FabricError` subclass instead of returning a partial ### Complete counts and useful observations -The bundled Deep Agents adapter supplies this evidence for its documented collection scope. - - `result.model_usage` is present when the adapter supplies the declared `nemo_fabric.usage_accounting.v1` evidence. Its token fields are independently nullable counts: input includes cache reads and writes, output includes diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 6e90b6cec..8f43fc0a8 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -26,6 +26,8 @@ schemas/ │ ├── agent.schema.json │ ├── run-plan.schema.json │ └── ... +├── extensions/ # Descriptor-declared extension payloads +│ └── usage-accounting.schema.json └── adapter-contract/ # Southbound adapter-facing contract ├── adapter-descriptor.schema.json ├── adapter-target-descriptor.schema.json @@ -38,9 +40,10 @@ schemas/ └── openai-stream-record.schema.json ``` -An adapter author can treat `adapter-contract/` as the complete schema entry -point. Contract versions remain explicit fields in the wire format rather than -directory names. +`adapter-contract/` defines the base adapter wire objects. Adapters returning +supported extension payloads must also declare the applicable schema from +`extensions/`. Contract versions remain explicit fields in the wire format +rather than directory names. Start with the [adapter contract overview](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/README.md) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 72b8ed64d..bea167f4a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2587,8 +2587,11 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if re.fullmatch(r"sha256:[0-9a-f]{64}", data["catalog_sha256"]) is None: raise FabricConfigError("invalid applied pricing fingerprint") rates = _mapping(data["rates"], "pricing rates") - if set(rates) != {"input", "output", "cache_read", "cache_write"}: + allowed = {"input", "output", "cache_read", "cache_write"} + if not {"input", "output"} <= set(rates) <= allowed: raise FabricConfigError("incomplete pricing rates") + rates.setdefault("cache_read", None) + rates.setdefault("cache_write", None) for key, value in rates.items(): if value is not None or key in {"input", "output"}: _usage_counter(value, "pricing rate") @@ -2902,6 +2905,8 @@ class RunResult(FabricMapping): are preserved as-is. error: Structured failure, or ``None`` on success. usage: Normalized invocation usage, or ``None`` when unavailable. + model_usage: Complete counts, observations and accounting gaps, or + ``None`` when supported accounting evidence is unavailable. resource_snapshot: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py index 5ba450330..dccaf29a3 100644 --- a/tests/adapters/test_deepagents_accounting.py +++ b/tests/adapters/test_deepagents_accounting.py @@ -365,8 +365,8 @@ async def test_non_model_failure_keeps_completed_call_usage(make_model): @pytest.fixture(name="make_runtime") async def make_runtime_fixture(tmp_path, monkeypatch): runtimes = [] - os.environ["SCRIPTED_MODEL_KEY"] = "unused-scripted" - os.environ.pop("OPENAI_PROXY", None) + monkeypatch.setenv("SCRIPTED_MODEL_KEY", "unused-scripted") + monkeypatch.delenv("OPENAI_PROXY", raising=False) async def make(replies, *, identity="runtime", profile=None, barrier=None): ledger = [] diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index 73a2c51ac..6fbcbfec2 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -177,7 +177,7 @@ def test_python_reference_exactly_covers_exported_sdk_classes() -> None: for module, names in functions.items(): page = REFERENCE_DIR / f"{module}.md" documented = { - match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text()) + match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } assert names <= documented assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") @@ -274,7 +274,7 @@ def test_generated_signatures_preserve_python_callable_semantics(): page = REFERENCE_DIR / f"{module_name}.md" rendered = { match.group(1): match.group(2) - for match in FUNCTION_SIGNATURE.finditer(page.read_text()) + for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } for function_name in function_names: function = getattr(module, function_name) diff --git a/tests/fixtures/usage_accounting/conformance.json b/tests/fixtures/usage_accounting/conformance.json index dc9d5cf47..b5e026d61 100644 --- a/tests/fixtures/usage_accounting/conformance.json +++ b/tests/fixtures/usage_accounting/conformance.json @@ -484,5 +484,185 @@ "observed": 20 } } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } } ] diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 93d31dd63..fb018ffaa 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -10,7 +10,7 @@ from nemo_fabric.accounting import summarize_usage from nemo_fabric.errors import FabricConfigError -from nemo_fabric.types import ModelUsage, PerModelUsage, TokenCounts +from nemo_fabric.types import AppliedPricing, ModelUsage, PerModelUsage, TokenCounts def test_per_model_complete_counts_have_the_public_token_types(): @@ -472,3 +472,23 @@ def test_independently_rounded_provider_cost_is_not_a_token_reconciliation_error per_model_usage["cost"]["reported"] = {"amount_nano_usd": 9} usage["models"] = [per_model_usage] assert ModelUsage.from_mapping(usage).cost.reported.amount_nano_usd == 10 + + +@pytest.mark.parametrize("cache_rates", [{}, {"cache_read": 0}, {"cache_write": 0}]) +def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): + parsed = AppliedPricing.from_mapping( + { + "catalog_sha256": "sha256:" + "a" * 64, + "provider": "fixture-provider", + "model_id": "fixture-model", + "pricing_as_of": "2026-10-22", + "pricing_source": "fixture-catalog", + "rates": {"input": 10, "output": 20, **cache_rates}, + } + ) + assert parsed.rates == { + "input": 10, + "output": 20, + "cache_read": cache_rates.get("cache_read"), + "cache_write": cache_rates.get("cache_write"), + } From 559ce0a558bb9faaa258b7ec17b1ac870b9d3248 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 15:05:43 -0700 Subject: [PATCH 074/102] fix: preserve fractional cost precision across usage reducers Signed-off-by: mnajafian-nv --- adapter-contract/typescript/src/usage.ts | 38 ++++++++++++++++++- .../usage_accounting/reducer-cases.json | 36 ++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index 726b5aff0..cb8f0dd9f 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -44,6 +44,42 @@ function validate(record: UsageObservation): void { } } +// Keep low-order cost contributions until the final round, matching math.fsum. +// Inputs have already passed the finite, nonnegative cost validation. +function sumCosts(values: readonly number[]): number { + const partials: number[] = []; + for (let high of values) { + let retained = 0; + for (let low of partials) { + if (Math.abs(high) < Math.abs(low)) [high, low] = [low, high]; + const sum = high + low; + const residual = low - (sum - high); + if (residual !== 0) partials[retained++] = residual; + high = sum; + } + partials.length = retained; + partials.push(high); + } + let index = partials.length; + let total = index === 0 ? 0 : partials[--index]!; + let residual = 0; + while (index > 0) { + const high = total; + const low = partials[--index]!; + total = high + low; + residual = low - (total - high); + if (residual !== 0) break; + } + // Preserve round-to-even when the remaining low-order terms share a sign. + if (index > 0 && ((residual < 0 && partials[index - 1]! < 0) || + (residual > 0 && partials[index - 1]! > 0))) { + const correction = residual * 2; + const rounded = total + correction; + if (correction === rounded - total) total = rounded; + } + return total; +} + function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage; observed: ReportedUsage; gaps: UsageGap[] } { const reported: ReportedUsage = {}; const observed: ReportedUsage = {}; @@ -58,7 +94,7 @@ function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage let value: number | undefined; if (values.length) { if (key === "cost_usd") { - const total = values.reduce((sum, item) => sum + item, 0); + const total = sumCosts(values); if (Number.isFinite(total) && total * 1e9 < 2 ** 64) value = total; } else if (key === "peak_request_input_tokens") { value = values.reduce((maximum, item) => Math.max(maximum, item)); diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json index e79697c63..cba63db2e 100644 --- a/tests/fixtures/usage_accounting/reducer-cases.json +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -437,5 +437,41 @@ "observed_input": 100, "exhaustive": true } + }, + { + "name": "reported_cost_keeps_fractional_sum_precision", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.2000000005 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + } + ], + "expected": { + "input_tokens": null, + "cost_usd": 0.4000000005, + "observed_input": null, + "observed_output": null, + "exhaustive": true + } } ] From 87bce87287cf7008d50ea72880b0f9d0b259c670 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 15:34:02 -0700 Subject: [PATCH 075/102] feat: compare task attempts with accounting coverage Signed-off-by: mnajafian-nv --- docs/index.yml | 2 + .../api/python-library-reference/index.md | 5 + .../nemo_fabric.comparisons.md | 78 ++ .../nemo_fabric.reporting.md | 3 +- docs/sdk/python.mdx | 20 + examples/README.md | 5 + examples/attempt_comparison/README.md | 132 +++ examples/attempt_comparison/__main__.py | 98 +++ scripts/docs/enhance_python_api_reference.py | 1 + scripts/generate_api_docs.sh | 6 + .../src/nemo_fabric/comparisons.py | 708 ++++++++++++++++ tests/docs/test_python_api_docs.py | 3 +- tests/python/test_comparison_capture.py | 174 ++++ tests/python/test_comparisons.py | 768 ++++++++++++++++++ 14 files changed, 2000 insertions(+), 3 deletions(-) create mode 100644 docs/reference/api/python-library-reference/nemo_fabric.comparisons.md create mode 100644 examples/attempt_comparison/README.md create mode 100644 examples/attempt_comparison/__main__.py create mode 100644 sdk/python/nemo-fabric-runtime/src/nemo_fabric/comparisons.py create mode 100644 tests/python/test_comparison_capture.py create mode 100644 tests/python/test_comparisons.py diff --git a/docs/index.yml b/docs/index.yml index 90765f0e7..39af3d5ec 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -102,6 +102,8 @@ navigation: path: ./reference/api/python-library-reference/nemo_fabric.accounting.md - page: Run Evidence Reporting path: ./reference/api/python-library-reference/nemo_fabric.reporting.md + - page: Task Comparisons + path: ./reference/api/python-library-reference/nemo_fabric.comparisons.md - page: Client path: ./reference/api/python-library-reference/nemo_fabric.client.md - page: Adapter Capability Inspection diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index 3fd5ff0e4..fcda2e405 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -12,6 +12,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`nemo_fabric.accounting`](./nemo_fabric.accounting.md#module-nemo_fabricaccounting): Typed model accounting and invocation summaries with explicit coverage. - [`nemo_fabric.reporting`](./nemo_fabric.reporting.md#module-nemo_fabricreporting): Inspect saved invocation evidence without a benchmark manifest or trace import. +- [`nemo_fabric.comparisons`](./nemo_fabric.comparisons.md#module-nemo_fabriccomparisons): Compare caller-defined task attempts using saved Fabric invocation evidence. - [`nemo_fabric.client`](./nemo_fabric.client.md#module-nemo_fabricclient): Native Python client for resolving and running NVIDIA NeMo Fabric agents. - [`nemo_fabric.capabilities`](./nemo_fabric.capabilities.md#module-nemo_fabriccapabilities): Metadata-only adapter inspection for host-side admission. - [`nemo_fabric.runtime`](./nemo_fabric.runtime.md#module-nemo_fabricruntime): Runtime lifecycle support for the NVIDIA NeMo Fabric Python SDK. @@ -126,6 +127,10 @@ SPDX-License-Identifier: Apache-2.0 --> - [`reporting.render_result_summary`](./nemo_fabric.reporting.md#function-render_result_summary): Render a concise overview; details add resource evidence. - [`reporting.summarize_request`](./nemo_fabric.reporting.md#function-summarize_request): Describe outer dispatch evidence without fabricating an invocation result. - [`reporting.summarize_result`](./nemo_fabric.reporting.md#function-summarize_result): Describe one returned invocation, preserving independent usable evidence. +- [`comparisons.main`](./nemo_fabric.comparisons.md#function-main): Print an explicit comparison manifest as a table or JSON without running agents. +- [`comparisons.read_comparison_manifest`](./nemo_fabric.comparisons.md#function-read_comparison_manifest): Read a bounded manifest; evidence paths are relative to its explicit root. +- [`comparisons.render_comparison_summary`](./nemo_fabric.comparisons.md#function-render_comparison_summary): Render per-metric coverage; details retain identities and pricing evidence. +- [`comparisons.summarize_attempts`](./nemo_fabric.comparisons.md#function-summarize_attempts): Summarize an explicit inventory of scheduled custom-task attempts. - [`capabilities.inspect_adapter`](./nemo_fabric.capabilities.md#function-inspect_adapter): Inspect a standalone runtime using descriptor metadata without loading its SDK. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.comparisons.md b/docs/reference/api/python-library-reference/nemo_fabric.comparisons.md new file mode 100644 index 000000000..a73dfd699 --- /dev/null +++ b/docs/reference/api/python-library-reference/nemo_fabric.comparisons.md @@ -0,0 +1,78 @@ +--- +title: "Task Comparisons" +slug: "/reference/api/python-library-reference/comparisons" +description: "Compare caller-defined task attempts with explicit evidence coverage." +--- + + +# module `nemo_fabric.comparisons` + +Compare caller-defined task attempts using saved Fabric invocation evidence. + +**Global Variables** +--------------- +- **MAX_RESULT_BYTES** + +--- + +## function `summarize_attempts` + +```python +def summarize_attempts( + attempts: Iterable[Mapping[str, Any]], + *, + root: Path, + inventory_complete: bool, +) -> dict[str, Any] +``` + +Summarize an explicit inventory of scheduled custom-task attempts. + +Each attempt names its task/revision/configuration and expected requests. ``inventory_complete`` asserts all scheduled attempts, including retries, are listed; ``requests_complete`` makes the corresponding per-attempt claim. Unknown inventory never establishes a complete cohort denominator. Evaluator verdicts and caller elapsed boundaries remain independent of execution. Resources retain per-invocation scopes. Logs and traces are references only. ``pricing_alignment`` compares contributing shared-reducer bases, not cohort eligibility or accounting completeness; per-configuration totals are retained. + + +--- + +## function `read_comparison_manifest` + +```python +def read_comparison_manifest(path: Path, *, root: Path | None = None) -> dict[str, Any] +``` + +Read a bounded manifest; evidence paths are relative to its explicit root. + + +--- + +## function `render_comparison_summary` + +```python +def render_comparison_summary( + report: Mapping[str, Any], + *, + details: bool = False, +) -> str +``` + +Render per-metric coverage; details retain identities and pricing evidence. + +Timing describes observed caller attempts with one common scope, never a sum of invocation durations. Neither partial spend nor task quality is ranked. + + +--- + +## function `main` + +```python +def main() -> None +``` + +Print an explicit comparison manifest as a table or JSON without running agents. + + + + +--- + +_This file was automatically generated via [lazydocs](https://github.com/ml-tooling/lazydocs)._ diff --git a/docs/reference/api/python-library-reference/nemo_fabric.reporting.md b/docs/reference/api/python-library-reference/nemo_fabric.reporting.md index 8a6ea68d3..19cc6235e 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.reporting.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.reporting.md @@ -13,7 +13,6 @@ Inspect saved invocation evidence without a benchmark manifest or trace import. **Global Variables** --------------- - **MAX_RESULT_BYTES** -- **RESOURCE_SCOPE** --- @@ -110,7 +109,7 @@ Describe the invocation interval without assuming whole-runtime success. def render_result_summary(summary: Mapping[str, Any], *, details: bool = False) -> str ``` -Render a concise overview; details add resource evidence. +Render a concise overview; details add resource and pricing evidence. Token and cost completeness remain separate. JSON quantities and accounting semantics are unchanged by this presentation. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 15ca4c52f..cd04a4462 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1562,3 +1562,23 @@ reference automatically or treats it as proof that publication succeeded. An outer timeout does not establish that inner or remote work stopped. Existing invocation status and error stage/code remain the stop evidence for returned results; this adds no competing stop-reason field. + + +### Compare Your Own Tasks + +The comparison module works with your own task runner, test suite or benchmark framework. Ordinary recording needs no task labels. For a comparison, the caller supplies an explicit inventory of attempts, task revisions, configuration IDs and expected request IDs with relative saved-evidence paths: + +```python +from pathlib import Path + +from nemo_fabric.comparisons import summarize_attempts, render_comparison_summary + +report = summarize_attempts(attempts, root=Path("./runs"), inventory_complete=True) +print(render_comparison_summary(report)) +``` + +Set `inventory_complete=True` only when every scheduled attempt, including retries and attempts with no returned result, is listed. Each attempt separately declares `requests_complete`. Missing evidence stays in the denominator. An optional named and versioned evaluator supplies task verdicts; execution success never supplies one. Caller-measured elapsed time requires its own scope, and parallel invocation times and resource windows are not summed. + +The summary separates complete totals, observed contributions, accounting gaps and result availability. A request can explicitly name an invocation-record fallback when its full result is unavailable. A valid full result remains authoritative even when its accounting is unknown. Accounting recovered from a record is labeled `core_invocation`; it does not establish the outer execution outcome. Incompatible price catalogs remain separate groups. The comparison table labels each configuration's contributing pricing bases and warns when they differ or are unverified. A complete estimate establishes accounting coverage within its configuration, not cost comparability across different tariffs. Different task/revision inventories are shown without selecting a cohort or ranking partial spend. + +Use `python -m nemo_fabric.comparisons manifest.json` for an overview, add `--details` for all token dimensions and pricing evidence, or add `--json` for structured output. The SDK renderer accepts `render_comparison_summary(report, details=True)` for the same detailed view. Each displayed usage metric retains its own complete/expected-request coverage. Caller timing reports the median of available attempt observations only when their measured scopes agree; it never sums overlapping invocation intervals. Partial timing retains its observation count. Task inventories, evaluators and pricing compatibility remain independent requirements for a fair comparison. The [custom-task example](https://github.com/NVIDIA/NeMo-Fabric/tree/main/examples/attempt_comparison) documents the manifest and produces explicitly synthetic evidence without model calls. Logs and optional Relay traces remain references, never accounting backfill. diff --git a/examples/README.md b/examples/README.md index 889387a82..421b02270 100644 --- a/examples/README.md +++ b/examples/README.md @@ -87,3 +87,8 @@ Start with the shared setup and execution model in the ## Native System Stats [`system_stats`](system_stats/README.md) demonstrates a real temporary worker-memory spike, native sampled peak/mean/range, retained per-invocation artifacts, and Linux cgroup constraints without a model endpoint or Relay collector. + + +## Compare Your Own Tasks + +[`attempt_comparison`](attempt_comparison/README.md) demonstrates complete, partial and missing evidence for caller-defined tasks. It uses synthetic saved results, makes no model calls, and shows explicit request coverage without a benchmark-runner dependency. diff --git a/examples/attempt_comparison/README.md b/examples/attempt_comparison/README.md new file mode 100644 index 000000000..7d498e405 --- /dev/null +++ b/examples/attempt_comparison/README.md @@ -0,0 +1,132 @@ + + +# Compare Your Own Task Attempts + +Use saved Fabric evidence to compare your tasks across harnesses or model +configurations. Task IDs, revisions and configuration labels are caller-defined. +A task can be a code review, document review, investigation or benchmark problem. +An ordinary Fabric run does not require these labels. + +## Try the Example + +From a matching development checkout: + +```bash +python -m examples.attempt_comparison /tmp/fabric-comparison-demo +python -m nemo_fabric.comparisons /tmp/fabric-comparison-demo/manifest.json +python -m nemo_fabric.comparisons /tmp/fabric-comparison-demo/manifest.json --details +python -m nemo_fabric.comparisons /tmp/fabric-comparison-demo/manifest.json --json +``` + +Use a new or empty directory. This creates synthetic normal, partial and missing +results. It makes no model calls and invents no latency, evaluator verdict or +price. The candidate's extra listed attempt stays visible, with a workload warning. +The report does not choose a matched subset or rank partial spend. + +## Capture a Real Task + +List every scheduled attempt, including retries, before dispatch. Give each +expected Fabric request a distinct `request_id`. Recording saves full results +and request evidence under the configured artifact directory. Keep missing +requests in the inventory when startup, timeout or recording prevents a result. + +The caller can capture a request and its own elapsed boundary: + +```python +from time import monotonic_ns +from nemo_fabric import Fabric, RunRequest + +started = monotonic_ns() +result = await Fabric().run( + config, base_dir=workspace, + request=RunRequest(request_id="review-1", input="Review my change"), +) +elapsed_ms = (monotonic_ns() - started) // 1_000_000 +``` + +Enable `runtime.record.enabled` and set `runtime.artifacts` in `config`. The +returned `run_result` artifact identifies the full saved result. Add its path, +relative to your evidence root, to this caller-owned manifest: + +```json +{ + "schema": "nemo.fabric.comparison_manifest", + "version": "v1alpha1", + "inventory_complete": true, + "attempts": [{ + "attempt_id": "review-attempt-1", + "task_id": "my-change-review", + "task_revision": "source-revision-1", + "configuration_id": "harness-and-model-settings-1", + "requests_complete": true, + "requests": [{"request_id": "review-1", "evidence": "review-result.json"}], + "elapsed_ms": 4200, + "elapsed_scope": "caller_one_shot" + }] +} +``` + +Replace the illustrative path and elapsed value with the captured values. Set +`inventory_complete` only when every scheduled attempt is listed, and +`requests_complete` only when the attempt lists every expected request. Both +default to false. An empty scheduled attempt remains visible and never becomes +measured zero spend. Request IDs and attempt IDs must be unique in the inventory. + +Use `from nemo_fabric.comparisons import summarize_attempts` with the `attempts` +list, `root=Path(...)` and explicit `inventory_complete` for the Python API. +The CLI defaults to the manifest's directory as the evidence root; `--root` +selects another root. Paths cannot escape that root, including through symlinks. + +## Interpret the Report + +- Execution status describes Fabric execution. It does not establish that your + task was solved. Add an optional caller verdict as + `"evaluation": {"status": "passed", "evaluator": "my-tests", "version": "v1"}` + or use `failed`. Otherwise the task remains `not_evaluated`. Mixed evaluator + identities have no combined pass fraction. +- Complete token totals and observed subtotals retain independent coverage. + Missing cost does not erase known tokens, execution or quality. The same + shared accounting reducer handles invocations and comparisons. +- Catalog estimates retain their pricing basis. The overview shows a compact basis label and warns when bases differ or a configuration has no priced evidence. `--details` retains full bases and all token dimensions. Every displayed usage metric shows its own complete/expected-request coverage. Caller timing shows the median of available attempts only when measured scopes agree, with the observation count; invocation intervals are not summed. JSON exposes `pricing_alignment`: `aligned` means every configuration has priced evidence under one shared basis, `different_bases` means the contributing bases differ, `unverified` means only some configurations have priced evidence, and `unavailable` means none do. These labels describe pricing evidence, not complete accounting or equivalent workloads. Individual valid totals remain available. Incompatible bases within one configuration remain separate groups in JSON; their amounts are not added together. Observed spend is incomplete and cannot establish which configuration is cheaper. +- Caller elapsed time names its measurement boundary. Parallel invocation + durations are never added to create attempt latency. Runtime CPU/RSS snapshots + retain each invocation's host scope; the report does not sum incompatible trees. +- Task IDs/revisions and unmatched workloads remain visible. Matching labels do + not prove that served models, tools or environments were equivalent. Inspect + available configuration and artifact evidence when checking conditions. + +Logs, telemetry and Relay references remain links to supporting evidence. Their +payloads are not read or used to fill missing Fabric accounting. Harbor, +OpenShell, a benchmark runner and a telemetry backend are not required. + +## Use a Bounded Invocation Record When the Full Result Is Missing + +A request can name an explicit fallback: + +```json +{"request_id": "review-1", "evidence": "review-result.json", "record": "run-record.jsonl"} +``` + +A valid full result is authoritative, including unknown or malformed accounting. +The reader uses the explicit matching record only when the full result itself +is missing, unreadable, malformed or exceeds the bounded reader's 16 MiB limit. Both paths +use the canonical evidence reader. Request IDs must match; known runtime and +invocation IDs must agree. No referenced log, trace or arbitrary artifact is opened. + +A record path can also be the primary `evidence`. Typed normalized accounting in +a complete end record can remain useful even when the full result was not saved. +The report labels `accounting_source: "record"` and +`accounting_boundary: "core_invocation"`. Its pre-shutdown status never becomes +a returned one-shot result or a task verdict. Legacy-only record counters stay +unverified. A result and its record never count twice. + +`execution.missing_result_count` counts absent full results. Accounting coverage +counts authoritative returned results or usable typed record projections, so a +record can provide accounting while execution remains without a full result. +Top-level `missing_accounting_evidence_count` includes every listed request without an authoritative accounting source. Each pricing group covers only invocations assigned to that known basis; missing accounting cannot be assigned to a pricing group. A returned result with missing or invalid normalized accounting is incomplete, not missing. Inspect +`accounting_sources`, `accounting_evidence_count`, `normalized_accounting_count`, +and each request's +source boundary when these counts differ. diff --git a/examples/attempt_comparison/__main__.py b/examples/attempt_comparison/__main__.py new file mode 100644 index 000000000..a9323a8d6 --- /dev/null +++ b/examples/attempt_comparison/__main__.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Create a synthetic custom-task comparison; no harness or model calls.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from nemo_fabric.comparisons import read_comparison_manifest, render_comparison_summary + + +def _usage(partial: bool) -> dict[str, Any]: + return { + "input_tokens": None if partial else 100, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 20, + "total_tokens": None if partial else 120, + "observed": {"input_tokens": 80} if partial else {}, + "models_exhaustive": False, + "models": [], + "gaps": [ + { + "dimensions": ["reported_cost", "estimated_cost"], + "reason": "missing_usage", + }, + *( + [ + { + "dimensions": ["input_tokens", "total_tokens"], + "reason": "missing_usage", + } + ] + if partial + else [] + ), + ], + "cost": {"reported": {}, "estimated": {}}, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path, help="New or empty output directory") + args = parser.parse_args() + root = args.directory + root.mkdir(parents=True, exist_ok=True) + if any(root.iterdir()): + parser.error( + "use a new or empty directory; existing evidence is never overwritten" + ) + attempts = [] + for name, configuration in [ + ("normal", "baseline"), + ("partial", "candidate"), + ("missing", "candidate"), + ]: + filename = f"{name}.json" + if name != "missing": + result = { + "request_id": name, + "runtime_id": f"runtime-{name}", + "invocation_id": f"invocation-{name}", + "status": "succeeded", + "model_usage": _usage(name == "partial"), + "artifacts": {"artifacts": []}, + } + with (root / filename).open("x", encoding="utf-8") as stream: + json.dump(result, stream, indent=2) + attempts.append( + { + "attempt_id": name, + "task_id": "review-my-change", + "task_revision": "v1", + "configuration_id": configuration, + "requests_complete": True, + "requests": [{"request_id": name, "evidence": filename}], + } + ) + manifest = { + "schema": "nemo.fabric.comparison_manifest", + "version": "v1alpha1", + "inventory_complete": True, + "attempts": attempts, + } + path = root / "manifest.json" + with path.open("x", encoding="utf-8") as stream: + json.dump(manifest, stream, indent=2) + print("Synthetic evidence; no model calls, measured latency or evaluator verdicts.") + print(render_comparison_summary(read_comparison_manifest(path))) + + +if __name__ == "__main__": + main() diff --git a/scripts/docs/enhance_python_api_reference.py b/scripts/docs/enhance_python_api_reference.py index 8af0ae17a..90e05c98b 100644 --- a/scripts/docs/enhance_python_api_reference.py +++ b/scripts/docs/enhance_python_api_reference.py @@ -18,6 +18,7 @@ MODULE_NAMES = ( "nemo_fabric.accounting", "nemo_fabric.reporting", + "nemo_fabric.comparisons", "nemo_fabric.client", "nemo_fabric.capabilities", "nemo_fabric.runtime", diff --git a/scripts/generate_api_docs.sh b/scripts/generate_api_docs.sh index bef86d5b1..d84c84a72 100755 --- a/scripts/generate_api_docs.sh +++ b/scripts/generate_api_docs.sh @@ -24,6 +24,7 @@ PYTHONPATH="sdk/python/nemo-fabric-runtime/src" lazydocs \ --overview-file "index.md" \ "nemo_fabric.accounting" \ "nemo_fabric.reporting" \ + "nemo_fabric.comparisons" \ "nemo_fabric.client" \ "nemo_fabric.capabilities" \ "nemo_fabric.runtime" \ @@ -130,6 +131,11 @@ add_frontmatter \ "Run Evidence Reporting" \ "Inspect saved invocation results, accounting coverage, and diagnostic references." \ "/reference/api/python-library-reference/reporting" +add_frontmatter \ + "$out/nemo_fabric.comparisons.md" \ + "Task Comparisons" \ + "Compare caller-defined task attempts with explicit evidence coverage." \ + "/reference/api/python-library-reference/comparisons" add_frontmatter \ "$out/nemo_fabric.errors.md" \ "Errors" \ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/comparisons.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/comparisons.py new file mode 100644 index 000000000..5e1033e2a --- /dev/null +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/comparisons.py @@ -0,0 +1,708 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compare caller-defined task attempts using saved Fabric invocation evidence.""" + +from __future__ import annotations + +import argparse +from collections import Counter +from collections.abc import Iterable, Mapping +import json +import math +from statistics import median +from pathlib import Path +from typing import Any + +from nemo_fabric.accounting import summarize_usage +from nemo_fabric._report_format import format_duration_ms, format_money +from nemo_fabric.reporting import MAX_RESULT_BYTES, read_evidence + + +def _text(value: Any, label: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{label} must be a nonempty string") + return value + + +def _path(root: Path, value: Any) -> Path: + relative = Path(_text(value, "evidence path")) + resolved = (root / relative).resolve() + if relative.is_absolute() or not resolved.is_relative_to(root): + raise ValueError("evidence paths must remain within the comparison root") + return resolved + + +def _evaluation(value: Any) -> dict[str, str]: + if value is None: + return {"status": "not_evaluated"} + if ( + not isinstance(value, Mapping) + or not isinstance(value.get("status"), str) + or value.get("status") not in {"passed", "failed", "not_evaluated"} + ): + raise ValueError("evaluation must name passed, failed or not_evaluated") + status = value["status"] + if status == "not_evaluated": + return {"status": status} + return { + "status": status, + "evaluator": _text(value.get("evaluator"), "evaluator"), + "version": _text(value.get("version"), "evaluator version"), + } + + +def _accounting_source(summary: Mapping[str, Any]) -> dict[str, Any]: + return { + key: summary[key] + for key in ("request_id", "runtime_id", "invocation_id", "model_usage") + } + + +def _record_accounting( + view: dict[str, Any], summary: Mapping[str, Any] +) -> dict[str, Any] | None: + if ( + summary.get("schema") != "nemo.fabric.invocation_record_summary" + or summary.get("model_usage") is None + ): + return None + if summary.get("outcome_boundary") != "invocation": + view["record_error"] = "unsupported_record_boundary" + return None + view.update(accounting_source="record", accounting_boundary="core_invocation") + return _accounting_source(summary) + + +def _request( + root: Path, spec: Mapping[str, Any] +) -> tuple[dict[str, Any], dict[str, Any] | None]: + request_id = _text(spec.get("request_id"), "request_id") + evidence = _text(spec.get("evidence"), "evidence path") + path = _path(root, evidence) + record_path = ( + _path(root, spec["record"]) if spec.get("record") is not None else None + ) + view: dict[str, Any] = {"request_id": request_id, "evidence": evidence} + summary = None + try: + summary = read_evidence(path) + except FileNotFoundError: + view["state"] = "missing_evidence" + except (OSError, ValueError, UnicodeError): + view["state"] = "invalid_evidence" + else: + if summary["request_id"] != request_id: + view["state"] = "identity_mismatch" + # A valid result for a different request cannot authorize a fallback. + return view, None + view["summary"] = summary + if summary["schema"] == "nemo.fabric.invocation_summary": + # A valid full result remains authoritative, including unknown or + # malformed accounting. Never repair it from an earlier projection. + view.update( + state="available", + accounting_source="result", + accounting_boundary="invocation", + ) + return view, _accounting_source(summary) + view["state"] = summary.get("state", "record_only") + accounting = _record_accounting(view, summary) + if accounting is not None: + return view, accounting + if record_path is None: + return view, None + view["record"] = spec["record"] + try: + record = read_evidence(record_path) + except (OSError, ValueError, UnicodeError): + view["record_error"] = "invalid_or_unreadable_record" + return view, None + if ( + record["request_id"] != request_id + or record["schema"] != "nemo.fabric.invocation_record_summary" + ): + view["record_error"] = "identity_or_record_type_mismatch" + return view, None + if summary is not None: + if any( + field in summary and summary[field] != record[field] + for field in ("runtime_id", "invocation_id") + ): + view["record_error"] = "identity_mismatch" + return view, None + if summary.get("state") == "no_result" and summary.get("stage") == "start": + view["record_error"] = "conflicting_request_evidence" + return view, None + view["record_summary"] = record + return view, _record_accounting(view, record) + + +def _usage( + results: list[dict[str, Any]], + listed_requests: int, + *, + inventory_complete: bool, + attempt_inventory_complete: bool = True, +) -> dict[str, Any]: + # Only real returned identities enter the shared accounting reducer. Missing + # expected requests change coverage, never fabricate invocation identities. + summary: dict[str, Any] = dict( + summarize_usage( + results, + expected=[ + (result["runtime_id"], result["invocation_id"]) for result in results + ], + ) + ) + summary["unit"] = "request" + summary["accounting_evidence_count"] = len(results) + summary["normalized_accounting_count"] = sum( + result.get("model_usage") is not None for result in results + ) + summary["inventory_complete"] = inventory_complete and attempt_inventory_complete + missing = listed_requests - len(results) + for field in summary["dimensions"].values(): + field["expected_count"] = listed_requests + field["missing_result_count"] = missing + if missing: + field["complete_total"] = None + field["reasons"]["missing_result"] = missing + if not inventory_complete: + field["complete_total"] = None + field["reasons"]["request_inventory_unverified"] = 1 + if not attempt_inventory_complete: + field["complete_total"] = None + field["reasons"]["attempt_inventory_unverified"] = 1 + if not listed_requests: + field["complete_total"] = None + field["reasons"]["no_request_evidence"] = 1 + + # Comparison requests can be represented by a result or a typed record. + # Preserve the reducer's quantities while naming this coverage boundary. + def name_accounting_coverage(value: Any) -> Any: + if isinstance(value, dict): + return { + "missing_accounting_evidence_count" + if key == "missing_result_count" + else key: name_accounting_coverage(item) + for key, item in value.items() + } + if isinstance(value, list): + return [name_accounting_coverage(item) for item in value] + return value + + return name_accounting_coverage(summary) + + +def _task_counts(rows: list[dict[str, Any]]) -> Counter[tuple[str, str]]: + return Counter((row["task_id"], row["task_revision"]) for row in rows) + + +def _group( + rows: list[dict[str, Any]], results: list[dict[str, Any]], inventory_complete: bool +) -> dict[str, Any]: + evaluation = Counter(row["evaluation"]["status"] for row in rows) + evaluators = sorted( + { + (row["evaluation"]["evaluator"], row["evaluation"]["version"]) + for row in rows + if row["evaluation"]["status"] != "not_evaluated" + } + ) + evaluated = evaluation["passed"] + evaluation["failed"] + requests = [request for row in rows for request in row["requests"]] + available = [request for request in requests if request["state"] == "available"] + return { + "listed_attempt_count": len(rows), + "request_inventory_complete_attempt_count": sum( + row["requests_complete"] for row in rows + ), + "attempts_without_listed_requests": sum(not row["requests"] for row in rows), + "task_inventory": [ + {"task_id": task, "task_revision": revision, "listed_attempt_count": count} + for (task, revision), count in sorted(_task_counts(rows).items()) + ], + "accounting_sources": dict( + Counter( + request["accounting_source"] + for request in requests + if "accounting_source" in request + ) + ), + "execution": { + "listed_request_count": len(requests), + "available_result_count": len(available), + "missing_result_count": len(requests) - len(available), + "statuses": dict( + Counter(request["summary"]["status"] for request in available) + ), + }, + "evaluation": { + "passed": evaluation["passed"], + "failed": evaluation["failed"], + "not_evaluated": evaluation["not_evaluated"], + "pass_fraction_evaluated": evaluation["passed"] / evaluated + if evaluated and len(evaluators) == 1 + else None, + "evaluators": [ + {"name": name, "version": version} for name, version in evaluators + ], + }, + "timing": { + "available_attempt_count": sum( + row["elapsed_ms"] is not None for row in rows + ), + "observations": [ + { + "attempt_id": row["attempt_id"], + "scope": row["elapsed_scope"], + "elapsed_ms": row["elapsed_ms"], + } + for row in rows + if row["elapsed_ms"] is not None + ], + }, + "usage": _usage( + results, + len(requests), + inventory_complete=all( + row["requests_complete"] and row["requests"] for row in rows + ), + attempt_inventory_complete=inventory_complete, + ), + } + + +def _pricing_alignment(configurations: Mapping[str, Any]) -> dict[str, Any]: + # The shared reducer has already qualified and grouped contributing bases. + # This comparison labels that evidence without defining another price policy. + configuration_bases: dict[str, list[dict[str, Any]]] = {} + distinct_bases: list[dict[str, Any]] = [] + for name, group in configurations.items(): + usage = group["usage"] + priced = [usage] if "pricing" in usage else usage.get("pricing_groups", []) + bases = [part["pricing"] for part in priced] + configuration_bases[name] = bases + for basis in bases: + # Match the reducer's absent/null optional-field equivalence. + comparable = { + key: value for key, value in basis.items() if value is not None + } + if comparable not in distinct_bases: + distinct_bases.append(comparable) + unpriced = sorted(name for name, bases in configuration_bases.items() if not bases) + if len(distinct_bases) > 1: + status = "different_bases" + elif not distinct_bases: + status = "unavailable" + elif unpriced: + status = "unverified" + else: + status = "aligned" + return { + "status": status, + "configuration_bases": configuration_bases, + "unpriced_configurations": unpriced, + } + + +def summarize_attempts( + attempts: Iterable[Mapping[str, Any]], + *, + root: Path, + inventory_complete: bool, +) -> dict[str, Any]: + """Summarize an explicit inventory of scheduled custom-task attempts. + + Each attempt names its task/revision/configuration and expected requests. + ``inventory_complete`` asserts all scheduled attempts, including retries, + are listed; ``requests_complete`` makes the corresponding per-attempt claim. + Unknown inventory never establishes a complete cohort denominator. Evaluator + verdicts and caller elapsed boundaries remain independent of execution. + Resources retain per-invocation scopes. Logs and traces are references only. + ``pricing_alignment`` compares contributing shared-reducer bases, not cohort + eligibility or accounting completeness; per-configuration totals are retained. + """ + if type(inventory_complete) is not bool: + raise ValueError("inventory_complete must be a boolean") + root = Path(root).resolve() + rows: list[dict[str, Any]] = [] + groups: dict[str, list[dict[str, Any]]] = {} + records: dict[str, list[dict[str, Any]]] = {} + attempts_seen: set[str] = set() + requests_seen: set[str] = set() + invocations_seen: set[tuple[str, str]] = set() + for spec in attempts: + if not isinstance(spec, Mapping): + raise ValueError("attempt must be an object") + identity = { + key: _text(spec.get(key), key) + for key in ("attempt_id", "task_id", "task_revision", "configuration_id") + } + if identity["attempt_id"] in attempts_seen: + raise ValueError("duplicate attempt_id") + attempts_seen.add(identity["attempt_id"]) + requests = spec.get("requests") + if not isinstance(requests, list) or any( + not isinstance(request, Mapping) for request in requests + ): + raise ValueError("requests must be a list of objects") + complete = spec.get("requests_complete", False) + if type(complete) is not bool: + raise ValueError("requests_complete must be a boolean") + elapsed = spec.get("elapsed_ms") + if elapsed is not None and ( + type(elapsed) is not int or not 0 <= elapsed < 1 << 64 + ): + raise ValueError("elapsed_ms must be a nonnegative uint64 integer or null") + scope = ( + _text(spec.get("elapsed_scope"), "elapsed_scope") + if elapsed is not None + else None + ) + views = [] + results = [] + for request in requests: + view, result = _request(root, request) + if view["request_id"] in requests_seen: + raise ValueError("duplicate expected request_id") + requests_seen.add(view["request_id"]) + views.append(view) + if result is not None: + key = (result["runtime_id"], result["invocation_id"]) + if key in invocations_seen: + raise ValueError( + "one invocation cannot belong to multiple requests or attempts" + ) + invocations_seen.add(key) + results.append(result) + row = { + **identity, + "evaluation": _evaluation(spec.get("evaluation")), + "elapsed_ms": elapsed, + "elapsed_scope": scope, + "requests_complete": complete, + "requests": views, + "usage": _usage( + results, len(requests), inventory_complete=complete and bool(requests) + ), + } + rows.append(row) + configuration = identity["configuration_id"] + groups.setdefault(configuration, []).append(row) + records.setdefault(configuration, []).extend(results) + tasks = {name: set(_task_counts(group)) for name, group in groups.items()} + shared = set.intersection(*tasks.values()) if tasks else set() + task_counts = [_task_counts(group) for group in groups.values()] + configurations = { + name: _group(group, records[name], inventory_complete) + for name, group in groups.items() + } + return { + "schema": "nemo.fabric.comparison_summary", + "version": "v1alpha1", + "inventory_complete": inventory_complete, + "attempts": rows, + "task_alignment": { + "shared_listed_tasks": [ + {"task_id": task, "task_revision": revision} + for task, revision in sorted(shared) + ], + "unmatched_by_configuration": { + name: [ + {"task_id": task, "task_revision": revision} + for task, revision in sorted(listed - shared) + ] + for name, listed in tasks.items() + }, + "listed_attempt_counts_match": bool(task_counts) + and all(counts == task_counts[0] for counts in task_counts), + }, + "configurations": configurations, + "pricing_alignment": _pricing_alignment(configurations), + } + + +def _invalid_constant(value: str) -> Any: + raise ValueError(f"non-finite manifest number: {value}") + + +def _finite_float(value: str) -> float: + number = float(value) + if not math.isfinite(number): + raise ValueError("non-finite manifest number") + return number + + +def read_comparison_manifest(path: Path, *, root: Path | None = None) -> dict[str, Any]: + """Read a bounded manifest; evidence paths are relative to its explicit root.""" + path = Path(path) + with path.open("rb") as stream: + payload = stream.read(MAX_RESULT_BYTES + 1) + if len(payload) > MAX_RESULT_BYTES: + raise ValueError("comparison manifest exceeds 16 MiB") + try: + manifest = json.loads( + payload, parse_constant=_invalid_constant, parse_float=_finite_float + ) + except RecursionError as error: + raise ValueError("comparison manifest is nested too deeply") from error + if ( + not isinstance(manifest, dict) + or (manifest.get("schema"), manifest.get("version")) + != ("nemo.fabric.comparison_manifest", "v1alpha1") + or not isinstance(manifest.get("attempts"), list) + ): + raise ValueError("unsupported comparison manifest schema or version") + return summarize_attempts( + manifest["attempts"], + root=root or path.resolve().parent, + inventory_complete=manifest.get("inventory_complete", False), + ) + + +def _value(field: Mapping[str, Any], *, money: bool = False) -> str: + complete = field["complete_total"] + observed = field["observed_sum"] + if complete is not None: + value = format_money(complete) if money else f"{complete:,}" + elif observed is not None: + value = f"{format_money(observed) if money else f'{observed:,}'} observed, incomplete" + elif field.get("aggregation_error") == "incompatible_pricing": + value = "incompatible pricing groups" + else: + value = "unavailable" + return f"{value} ({field['complete_count']}/{field['expected_count']} complete)" + + +def _label(value: str) -> str: + # JSON quoting escapes terminal controls, embedded newlines and pipes. + return json.dumps(value, ensure_ascii=True).replace("|", "\\u007c") + + +def _catalog_identifier(basis: Mapping[str, Any]) -> str: + return ", ".join( + f"{label} {_label(basis[key].removeprefix('sha256:')[:12])}" + for key, label in ( + ("bundled_catalog_sha256", "bundled"), + ("user_catalog_sha256", "user"), + ) + if basis.get(key) + ) or _label(basis["basis"]) + + +def _timing(group: Mapping[str, Any], scopes: set[str]) -> str: + timing = group["timing"] + coverage = ( + f"({timing['available_attempt_count']}/{group['listed_attempt_count']} timed)" + ) + observations = timing["observations"] + if not observations: + return f"unavailable {coverage}" + if len(scopes) != 1: + return f"mixed scopes {coverage}" + duration = format_duration_ms(median(row["elapsed_ms"] for row in observations)) + return f"{duration} median {coverage}; {_label(observations[0]['scope'])}" + + +def render_comparison_summary( + report: Mapping[str, Any], *, details: bool = False +) -> str: + """Render per-metric coverage; details retain identities and pricing evidence. + + Timing describes observed caller attempts with one common scope, never a sum + of invocation durations. Neither partial spend nor task quality is ranked. + """ + alignment = report["pricing_alignment"] + configurations = report["configurations"] + lines = [] + if not report["inventory_complete"]: + lines.append( + "Attempt inventory is unverified; the full cohort denominator and totals are unknown." + ) + if any( + group["request_inventory_complete_attempt_count"] + != group["listed_attempt_count"] + or group["attempts_without_listed_requests"] + for group in configurations.values() + ): + lines.append( + "Request inventories are unverified or empty for some attempts; full accounting totals are unknown." + ) + if not report["task_alignment"]["listed_attempt_counts_match"]: + lines.append( + "Listed task/revision attempt inventories differ; no cohort was silently selected." + ) + evaluators = { + (item["name"], item["version"]) + for group in configurations.values() + for item in group["evaluation"]["evaluators"] + } + if len(evaluators) > 1: + lines.append( + "Evaluator identities or versions differ; outcome fractions are not comparable." + ) + scopes = { + row["scope"] + for group in configurations.values() + for row in group["timing"]["observations"] + } + if len(scopes) > 1: + lines.append("Timing scopes differ; no combined timing median is shown.") + if alignment["status"] == "different_bases": + lines.append( + "Catalog pricing bases differ; do not compare estimates across different bases." + ) + elif alignment["status"] == "unverified": + lines.append( + "Catalog pricing evidence is missing for some configurations; a shared basis is unverified." + ) + lines.append( + "Configuration | Task outcomes | Input tokens | Output tokens | Catalog estimate" + ) + for name, group in configurations.items(): + evaluation = group["evaluation"] + evaluated = evaluation["passed"] + evaluation["failed"] + attempts = group["listed_attempt_count"] + if not evaluated: + verdict = f"not evaluated (0/{attempts} evaluated)" + elif evaluation["pass_fraction_evaluated"] is None: + verdict = ( + f"{evaluation['passed']} passed; {evaluation['failed']} failed; " + f"{evaluated}/{attempts} evaluated; evaluator mismatch" + ) + else: + verdict = f"{evaluation['passed']}/{evaluated} passed; {evaluated}/{attempts} evaluated" + if len(evaluators) > 1: + verdict += "; evaluator mismatch" + dimensions = group["usage"]["dimensions"] + lines.append( + f"{_label(name)} | {verdict} | {_value(dimensions['input_tokens'])} | {_value(dimensions['output_tokens'])} | {_value(dimensions['estimated_cost'], money=True)}" + ) + execution = group["execution"] + lines.append( + f" Results: {execution['available_result_count']}/{execution['listed_request_count']} returned; {execution['missing_result_count']} missing. Attempts listed: {attempts}." + ) + if execution["statuses"]: + lines.append( + " Execution: " + + ", ".join( + f"{count} {status}" + for status, count in sorted(execution["statuses"].items()) + ) + ) + lines.append(f" Caller attempt time: {_timing(group, scopes)}.") + other_tokens = { + "total_tokens": "Total tokens", + "cached_input_tokens": "Cache reads", + "cache_write_input_tokens": "Cache writes", + } + other_gaps = [ + f"{label}: {_value(dimensions[key])}" + for key, label in other_tokens.items() + if dimensions[key]["complete_total"] is None + ] + if other_gaps: + lines.append(" Other token gaps: " + "; ".join(other_gaps)) + bases = alignment["configuration_bases"][name] + if bases: + label = ( + "Catalog basis" if len(bases) == 1 else f"Catalog bases ({len(bases)})" + ) + lines.append( + f" {label}: " + + "; ".join(_catalog_identifier(basis) for basis in bases) + ) + else: + lines.append(" No priced catalog evidence for this configuration.") + reported = dimensions["reported_cost"] + if ( + details + or reported["complete_total"] is not None + or reported["observed_sum"] is not None + ): + lines.append(f" Provider-reported cost: {_value(reported, money=True)}.") + reasons = sorted( + { + reason + for key, field in dimensions.items() + if details or key != "reported_cost" + for reason in field["reasons"] + } + ) + if reasons: + lines.append( + " Accounting gaps: " + + ", ".join(reason.replace("_", " ") for reason in reasons) + ) + if details: + lines.append( + " Other tokens: " + + "; ".join( + f"{label}: {_value(dimensions[key])}" + for key, label in other_tokens.items() + ) + + ". Cache reads and writes are included in input." + ) + if bases: + lines.append( + " Catalog pricing bases: " + + json.dumps(bases, ensure_ascii=True, sort_keys=True) + ) + lines.append( + " Execution statuses: " + + json.dumps(execution["statuses"], ensure_ascii=True) + + "; evaluators: " + + json.dumps(evaluation["evaluators"], ensure_ascii=True) + + f"; {evaluation['not_evaluated']} attempts not evaluated." + ) + lines.append( + f" Accounting sources: {json.dumps(group['accounting_sources'], ensure_ascii=True)}; result and record are never counted twice." + ) + if group["timing"]["observations"]: + lines.append( + " Caller timing boundaries: " + + json.dumps(group["timing"]["observations"], ensure_ascii=True) + ) + lines.append( + " Listed task revisions: " + + json.dumps(group["task_inventory"], ensure_ascii=True) + ) + lines.append( + "Token and cost coverage counts complete / listed requests; timing covers observed caller attempts." + ) + lines.append( + "Evaluator verdicts are caller-supplied; execution success is not task correctness." + ) + lines.append( + "Observed spend is incomplete, not a bill or a ranking. Details and JSON retain evidence and pricing groups." + ) + return "\n".join(lines) + + +def main() -> None: + """Print an explicit comparison manifest as a table or JSON without running agents.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path) + parser.add_argument( + "--root", type=Path, help="Evidence root; defaults to manifest directory" + ) + parser.add_argument("--json", action="store_true", dest="as_json") + parser.add_argument( + "--details", action="store_true", help="Include identities and pricing evidence" + ) + args = parser.parse_args() + try: + report = read_comparison_manifest(args.manifest, root=args.root) + except (OSError, TypeError, ValueError) as error: + parser.error(str(error)) + print( + json.dumps(report, indent=2, allow_nan=False) + if args.as_json + else render_comparison_summary(report, details=args.details) + ) + + +if __name__ == "__main__": + main() diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index 66a7443da..56c273660 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -39,6 +39,7 @@ MODULE_SLUGS = { "nemo_fabric.accounting": "/reference/api/python-library-reference/accounting", "nemo_fabric.reporting": "/reference/api/python-library-reference/reporting", + "nemo_fabric.comparisons": "/reference/api/python-library-reference/comparisons", "nemo_fabric.client": "/reference/api/python-library-reference/client", "nemo_fabric.capabilities": "/reference/api/python-library-reference/capabilities", "nemo_fabric.runtime": "/reference/api/python-library-reference/runtime", @@ -51,7 +52,7 @@ "nemo_fabric.types": "/reference/api/python-library-reference/types", "nemo_fabric.errors": "/reference/api/python-library-reference/errors", } -PUBLIC_SUBMODULES = ("nemo_fabric.reporting",) +PUBLIC_SUBMODULES = ("nemo_fabric.reporting", "nemo_fabric.comparisons") MEMBER_SIGNATURE = re.compile( r"^### (?:method|classmethod) `([^`]+)`\n\n" r"```python\n(.*?)\n```", diff --git a/tests/python/test_comparison_capture.py b/tests/python/test_comparison_capture.py new file mode 100644 index 000000000..575b3745f --- /dev/null +++ b/tests/python/test_comparison_capture.py @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Actual SDK/core publication to caller inventory, without a model or network.""" + +import asyncio +import json +import os +from pathlib import Path +import sys +from time import monotonic_ns + +import pytest + +from nemo_fabric import Fabric, FabricConfig, RunRequest +from nemo_fabric.comparisons import read_comparison_manifest, render_comparison_summary +from nemo_fabric_adapter_contract.models import GapReason, ReportedUsage +from nemo_fabric_adapter_contract.usage import UsageObservation, usage_from_observations + + +@pytest.mark.integration +def test_real_sdk_capture_preserves_normal_partial_and_missing_attempts(tmp_path): + repository = Path( + os.environ.get("NEMO_FABRIC_TEST_REPO", Path(__file__).resolve().parents[2]) + ) + schema = json.loads( + (repository / "schemas/extensions/usage-accounting.schema.json").read_text() + ) + definitions = schema.pop("$defs") + descriptor = { + "contract_version": "fabric.adapter/v1alpha2", + "adapter_id": "test.comparison-capture", + "adapter_kind": "process", + "runner": {"command": sys.executable, "script": "adapter.py"}, + "settings_schema": {"type": "object", "additionalProperties": False}, + "extension_schemas": { + "usage": { + "type": "object", + "$defs": definitions, + "properties": { + "nemo_fabric.usage_accounting.v1": schema, + "nemo_fabric.detailed_token_usage.v1": { + "$ref": "#/$defs/ReportedUsage" + }, + }, + "additionalProperties": False, + } + }, + } + (tmp_path / "comparison.fabric-adapter.json").write_text(json.dumps(descriptor)) + for mode, input_tokens, gap in [ + ("normal", 10, None), + ("partial", 7, GapReason.UNFINISHED_WORK), + ]: + usage = usage_from_observations( + [ + UsageObservation( + model="controlled-fixture", + is_model_call=True, + tokens_aligned=True, + gap=gap, + reported=ReportedUsage( + input_tokens=input_tokens, + input_tokens_include_cache=True, + cached_input_tokens=0, + cache_write_input_tokens=0, + output_tokens=2, + output_tokens_include_reasoning=True, + reasoning_tokens=0, + total_tokens=input_tokens + 2, + ), + ) + ] + ) + (tmp_path / f"{mode}.json").write_text(json.dumps(usage.to_mapping())) + (tmp_path / "adapter.py").write_text("""import json,sys +from pathlib import Path +for line in sys.stdin: + message=json.loads(line) + operation=message["operation"] + output=None + if operation=="invoke": + mode=message["payload"]["request"]["input"] + output={"status":"failed" if mode=="partial" else "succeeded", + "output":{"response":"controlled fixture"}, + "usage":json.loads(Path(mode+".json").read_text())} + if mode=="partial": + output["error"]={"code":"controlled_timeout","message":"controlled unfinished work","retryable":False} + print(json.dumps({"operation":operation,"outcome":{"status":"succeeded","output":output}}),flush=True) + if operation=="stop":break +""") + config = FabricConfig.from_mapping( + { + "metadata": {"name": "comparison-capture"}, + "harness": {"adapter_id": "test.comparison-capture"}, + "discovery": {"local_paths": ["comparison.fabric-adapter.json"]}, + "environment": {"provider": "local"}, + "runtime": {"artifacts": "./artifacts", "record": {"enabled": True}}, + } + ) + + async def capture(): + attempts = [] + for mode in ("normal", "partial"): + started = monotonic_ns() + result = await Fabric().run( + config, + base_dir=tmp_path, + request=RunRequest(request_id=mode, input=mode), + ) + elapsed = (monotonic_ns() - started) // 1_000_000 + saved = next( + artifact.path + for artifact in result.artifacts.artifacts + if artifact.kind == "run_result" + ) + attempts.append( + { + "attempt_id": mode, + "task_id": "custom-change-review", + "task_revision": "v1", + "configuration_id": mode, + "requests_complete": True, + "requests": [ + { + "request_id": mode, + "evidence": str(Path(saved).relative_to(tmp_path)), + } + ], + "elapsed_ms": elapsed, + "elapsed_scope": "caller_one_shot", + } + ) + attempts.append( + { + "attempt_id": "scheduled", + "task_id": "custom-change-review", + "task_revision": "v1", + "configuration_id": "partial", + "requests_complete": True, + "requests": [ + {"request_id": "scheduled", "evidence": "never-returned.json"} + ], + } + ) + return attempts + + attempts = asyncio.run(capture()) + path = tmp_path / "manifest.json" + path.write_text( + json.dumps( + { + "schema": "nemo.fabric.comparison_manifest", + "version": "v1alpha1", + "inventory_complete": True, + "attempts": attempts, + } + ) + ) + summary = read_comparison_manifest(path) + normal = summary["configurations"]["normal"]["usage"]["dimensions"]["input_tokens"] + partial = summary["configurations"]["partial"]["usage"]["dimensions"][ + "input_tokens" + ] + assert normal["complete_total"] == 10 + assert partial["complete_total"] is None + assert partial["observed_sum"] == 7 + assert partial["missing_accounting_evidence_count"] == 1 + assert summary["configurations"]["partial"]["listed_attempt_count"] == 2 + assert summary["configurations"]["partial"]["execution"]["statuses"] == { + "failed": 1 + } + assert summary["configurations"]["normal"]["evaluation"]["not_evaluated"] == 1 + assert "7 observed, incomplete" in render_comparison_summary(summary) diff --git a/tests/python/test_comparisons.py b/tests/python/test_comparisons.py new file mode 100644 index 000000000..0739bd429 --- /dev/null +++ b/tests/python/test_comparisons.py @@ -0,0 +1,768 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from copy import deepcopy +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from nemo_fabric.comparisons import ( + read_comparison_manifest, + render_comparison_summary, + summarize_attempts, +) + + +@pytest.fixture(name="result") +def result_fixture(): + return { + "runtime_id": "runtime", + "invocation_id": "invocation", + "request_id": "request", + "status": "succeeded", + "model_usage": { + "input_tokens": 100, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + "total_tokens": 102, + "observed": {}, + "gaps": [{"dimensions": ["estimated_cost"], "reason": "missing_usage"}], + "models_exhaustive": False, + "models": [], + "cost": { + "reported": {"amount_nano_usd": 10, "observed_nano_usd": None}, + "estimated": {"amount_nano_usd": None, "observed_nano_usd": None}, + }, + }, + "artifacts": {"artifacts": []}, + } + + +@pytest.fixture(name="attempt") +def attempt_fixture(tmp_path, result): + (tmp_path / "result.json").write_text(json.dumps(result)) + return { + "attempt_id": "attempt", + "task_id": "custom-review", + "task_revision": "v1", + "configuration_id": "baseline", + "requests_complete": True, + "requests": [{"request_id": "request", "evidence": "result.json"}], + "elapsed_ms": 6000, + "elapsed_scope": "caller_attempt", + } + + +def report(tmp_path, attempts, complete=True): + return summarize_attempts(attempts, root=tmp_path, inventory_complete=complete) + + +def test_custom_task_needs_no_evaluator_or_harbor(tmp_path, attempt): + summary = report(tmp_path, [attempt]) + group = summary["configurations"]["baseline"] + assert group["evaluation"]["not_evaluated"] == 1 + assert group["evaluation"]["pass_fraction_evaluated"] is None + assert group["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + assert summary["attempts"][0]["requests"][0]["summary"]["status"] == "succeeded" + + +def test_missing_attempt_keeps_expected_requests_and_available_observations( + tmp_path, attempt +): + missing = { + **attempt, + "attempt_id": "missing", + "requests": [{"request_id": "missing", "evidence": "missing.json"}], + } + group = report(tmp_path, [attempt, missing])["configurations"]["baseline"] + assert group["listed_attempt_count"] == 2 + dimension = group["usage"]["dimensions"]["input_tokens"] + assert dimension["complete_total"] is None + assert dimension["observed_sum"] == 100 + assert dimension["expected_count"] == 2 + assert dimension["missing_accounting_evidence_count"] == 1 + + +def test_parallel_invocations_keep_caller_time_and_separate_resource_scopes( + tmp_path, attempt, result +): + second = deepcopy(result) + second.update( + request_id="parallel", invocation_id="parallel", runtime_id="parallel-runtime" + ) + result["resource_snapshot"] = { + "runtime_id": "runtime", + "cpu_time_seconds": 10.0, + "resident_memory_bytes": 100, + "process_count": 1, + "started_at": "2026-10-21T00:00:00Z", + "observed_at": "2026-10-21T00:00:05Z", + } + second["resource_snapshot"] = { + **result["resource_snapshot"], + "runtime_id": "parallel-runtime", + "cpu_time_seconds": 20.0, + "resident_memory_bytes": 200, + } + (tmp_path / "result.json").write_text(json.dumps(result)) + (tmp_path / "parallel.json").write_text(json.dumps(second)) + attempt["requests"].append({"request_id": "parallel", "evidence": "parallel.json"}) + row = report(tmp_path, [attempt])["attempts"][0] + assert row["elapsed_ms"] == 6000 + assert row["usage"]["dimensions"]["input_tokens"]["complete_total"] == 200 + resources = [request["summary"]["resource_snapshot"] for request in row["requests"]] + assert [resource["cpu_time_seconds"] for resource in resources] == [10.0, 20.0] + assert [resource["runtime_id"] for resource in resources] == [ + "runtime", + "parallel-runtime", + ] + assert "resource_snapshot" not in row + assert "cpu_time_seconds" not in row + + +def test_partial_accounting_does_not_erase_quality_or_independent_dimensions( + tmp_path, attempt, result +): + usage = result["model_usage"] + usage["input_tokens"] = usage["total_tokens"] = None + usage["observed"] = {"input_tokens": 50} + usage["gaps"].append( + {"dimensions": ["input_tokens", "total_tokens"], "reason": "unfinished_work"} + ) + result["status"] = "failed" + (tmp_path / "result.json").write_text(json.dumps(result)) + attempt["evaluation"] = { + "status": "passed", + "evaluator": "external-tests", + "version": "v1", + } + group = report(tmp_path, [attempt])["configurations"]["baseline"] + assert group["evaluation"]["passed"] == 1 + assert group["execution"]["statuses"] == {"failed": 1} + assert group["usage"]["dimensions"]["input_tokens"]["observed_sum"] == 50 + assert group["usage"]["dimensions"]["output_tokens"]["complete_total"] == 2 + + +@pytest.mark.parametrize("scope", ["attempt", "requests"]) +def test_unknown_inventory_never_certifies_complete_cohort_total( + tmp_path, attempt, scope +): + if scope == "requests": + attempt["requests_complete"] = False + group = report(tmp_path, [attempt], complete=scope != "attempt")["configurations"][ + "baseline" + ] + dimension = group["usage"]["dimensions"]["input_tokens"] + assert dimension["complete_total"] is None + assert dimension["observed_sum"] == 100 + assert group["usage"]["inventory_complete"] is False + + +@pytest.mark.parametrize("path", ["../escape.json", "/tmp/escape.json"]) +def test_evidence_path_cannot_escape_root(tmp_path, attempt, path): + attempt["requests"][0]["evidence"] = path + with pytest.raises(ValueError, match="within"): + report(tmp_path, [attempt]) + + +def test_symlink_evidence_cannot_escape_root(tmp_path, attempt): + outside = tmp_path.parent / (tmp_path.name + "-outside.json") + outside.write_text("{}") + (tmp_path / "link.json").symlink_to(outside) + attempt["requests"][0]["evidence"] = "link.json" + with pytest.raises(ValueError, match="within"): + report(tmp_path, [attempt]) + + +def test_reused_invocation_identity_is_rejected_across_distinct_requests( + tmp_path, attempt, result +): + result["request_id"] = "second" + (tmp_path / "second.json").write_text(json.dumps(result)) + other = { + **attempt, + "attempt_id": "other", + "requests": [{"request_id": "second", "evidence": "second.json"}], + } + with pytest.raises(ValueError, match="one invocation"): + report(tmp_path, [attempt, other]) + + +@pytest.mark.parametrize("mutation", ["wrong_id", "truncated", "invalid_status"]) +def test_bad_result_is_a_visible_gap_without_losing_evaluator( + tmp_path, attempt, result, mutation +): + attempt["evaluation"] = {"status": "failed", "evaluator": "tests", "version": "v1"} + if mutation == "wrong_id": + result["request_id"] = "another" + if mutation == "invalid_status": + result["status"] = "unknown" + (tmp_path / "result.json").write_text( + "{" if mutation == "truncated" else json.dumps(result) + ) + row = report(tmp_path, [attempt])["attempts"][0] + assert row["requests"][0]["state"] != "available" + assert row["evaluation"]["status"] == "failed" + assert ( + row["usage"]["dimensions"]["input_tokens"]["missing_accounting_evidence_count"] + == 1 + ) + + +def test_invalid_accounting_is_quarantined_by_canonical_reader( + tmp_path, attempt, result +): + result["model_usage"]["input_tokens"] = -1 + result["telemetry"] = [{"provider": "relay", "uri": "do-not-open.jsonl"}] + (tmp_path / "result.json").write_text(json.dumps(result)) + view = report(tmp_path, [attempt])["attempts"][0]["requests"][0] + assert view["state"] == "available" + assert view["summary"]["status"] == "succeeded" + assert view["summary"]["accounting_error"] == "invalid_model_usage" + assert view["summary"]["telemetry"] == result["telemetry"] + + +def test_legacy_usage_stays_diagnostics_without_false_completeness( + tmp_path, attempt, result +): + result.pop("model_usage") + result["usage"] = {"input_tokens": 100} + (tmp_path / "result.json").write_text(json.dumps(result)) + row = report(tmp_path, [attempt])["attempts"][0] + assert row["requests"][0]["summary"]["legacy_usage"] == {"input_tokens": 100} + assert row["usage"]["dimensions"]["input_tokens"]["complete_total"] is None + + +def test_empty_scheduled_attempt_stays_in_inventory(tmp_path, attempt): + attempt["requests"] = [] + group = report(tmp_path, [attempt])["configurations"]["baseline"] + assert group["listed_attempt_count"] == 1 + assert group["attempts_without_listed_requests"] == 1 + assert group["usage"]["dimensions"]["input_tokens"]["complete_total"] is None + + +def test_mixed_evaluator_identities_have_no_combined_fraction(tmp_path, attempt): + attempt["evaluation"] = {"status": "passed", "evaluator": "A", "version": "v1"} + other = { + **attempt, + "attempt_id": "other", + "requests": [], + "evaluation": {"status": "failed", "evaluator": "B", "version": "v1"}, + } + summary = report(tmp_path, [attempt, other]) + evaluation = summary["configurations"]["baseline"]["evaluation"] + assert evaluation["passed"] == evaluation["failed"] == 1 + assert evaluation["pass_fraction_evaluated"] is None + rendered = render_comparison_summary(summary) + assert "1/2 passed" not in rendered + assert "1 passed; 1 failed; 2/2 evaluated; evaluator mismatch" in rendered + + +def test_unmatched_tasks_and_repetitions_are_visible_without_selection( + tmp_path, attempt +): + other = { + **attempt, + "attempt_id": "other", + "configuration_id": "candidate", + "task_revision": "v2", + "requests": [], + } + summary = report(tmp_path, [attempt, other]) + assert summary["task_alignment"]["shared_listed_tasks"] == [] + assert ( + len(summary["task_alignment"]["unmatched_by_configuration"]["candidate"]) == 1 + ) + assert summary["task_alignment"]["listed_attempt_counts_match"] is False + assert "no cohort was silently selected" in render_comparison_summary(summary) + + +def test_labels_cannot_inject_terminal_controls_or_new_rows(tmp_path, attempt): + attempt["configuration_id"] = "bad\x1b[31m\nlabel|extra" + text = render_comparison_summary(report(tmp_path, [attempt])) + assert "\x1b" not in text + assert "bad\\u001b[31m\\nlabel\\u007cextra" in text + + +def test_outer_request_evidence_remains_no_result_not_zero(tmp_path, attempt): + document = { + "schema": "nemo.fabric.request_record", + "version": "v1alpha1", + "kind": "end", + "state": "no_result", + "stage": "start", + "request_id": "request", + "time": "2026-10-21T00:00:00Z", + } + (tmp_path / "result.json").write_text(json.dumps(document)) + row = report(tmp_path, [attempt])["attempts"][0] + assert row["requests"][0]["state"] == "no_result" + assert row["usage"]["dimensions"]["input_tokens"]["observed_sum"] is None + + +def test_cli_manifest_reader_uses_same_contract(tmp_path, attempt): + manifest = { + "schema": "nemo.fabric.comparison_manifest", + "version": "v1alpha1", + "inventory_complete": True, + "attempts": [attempt], + } + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest)) + expected = report(tmp_path, [attempt]) + assert read_comparison_manifest(path) == expected + command = [sys.executable, "-m", "nemo_fabric.comparisons", str(path)] + output = subprocess.run( + [*command, "--json"], check=True, capture_output=True, text=True + ) + assert json.loads(output.stdout) == expected + table = subprocess.run(command, check=True, capture_output=True, text=True) + assert "100 (1/1 complete)" in table.stdout + assert "not evaluated" in table.stdout + detail = subprocess.run( + [*command, "--details"], check=True, capture_output=True, text=True + ) + assert "Listed task revisions:" in detail.stdout + assert "Listed task revisions:" not in table.stdout + structured_detail = subprocess.run( + [*command, "--json", "--details"], check=True, capture_output=True, text=True + ) + assert json.loads(structured_detail.stdout) == expected + + +@pytest.mark.parametrize( + "mutation", + [ + "duplicate_attempt", + "duplicate_request", + "missing_elapsed_scope", + "boolean_elapsed", + "malformed_evaluation", + ], +) +def test_inventory_validation_rejects_ambiguous_inputs(tmp_path, attempt, mutation): + attempts = [attempt] + if mutation == "duplicate_attempt": + attempts.append(deepcopy(attempt)) + elif mutation == "duplicate_request": + attempt["requests"].append(deepcopy(attempt["requests"][0])) + elif mutation == "missing_elapsed_scope": + attempt.pop("elapsed_scope") + elif mutation == "boolean_elapsed": + attempt["elapsed_ms"] = True + else: + attempt["evaluation"] = {"status": {"bad": "shape"}} + with pytest.raises(ValueError): + report(tmp_path, attempts) + + +def test_partial_pricing_groups_keep_their_coverage_and_never_become_one_cost( + tmp_path, attempt, result +): + requests = [] + for index, amount, catalog in [(0, 14000, "a"), (1, 20000, "b"), (2, None, "a")]: + value = deepcopy(result) + request_id = f"priced-{index}" + value.update(request_id=request_id, invocation_id=request_id) + usage = value["model_usage"] + usage["cost"]["estimated"] = { + "amount_nano_usd": amount, + "observed_nano_usd": amount, + } + usage["cost"]["pricing"] = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + catalog * 64, + } + usage["gaps"] = ( + [] + if amount is not None + else [{"dimensions": ["estimated_cost"], "reason": "missing_usage"}] + ) + filename = f"{request_id}.json" + (tmp_path / filename).write_text(json.dumps(value)) + requests.append({"request_id": request_id, "evidence": filename}) + requests.append({"request_id": "missing-price", "evidence": "missing-price.json"}) + attempt["requests"] = requests + usage = report(tmp_path, [attempt])["configurations"]["baseline"]["usage"] + assert usage["dimensions"]["estimated_cost"]["observed_sum"] is None + assert ( + usage["dimensions"]["estimated_cost"]["aggregation_error"] + == "incompatible_pricing" + ) + assert usage["dimensions"]["estimated_cost"]["expected_count"] == 4 + groups = { + group["pricing"]["bundled_catalog_sha256"]: group["dimensions"][ + "estimated_cost" + ] + for group in usage["pricing_groups"] + } + first = groups["sha256:" + "a" * 64] + assert first["expected_count"] == 2 + assert first["missing_accounting_evidence_count"] == 0 + assert "missing_result_count" not in first + assert first["incomplete_count"] == 1 + assert first["complete_total"] is None + assert first["observed_sum"] == 14000 + assert groups["sha256:" + "b" * 64]["complete_total"] == 20000 + assert usage["dimensions"]["input_tokens"]["observed_sum"] == 300 + + +@pytest.fixture(name="record") +def record_fixture(tmp_path, result): + identity = { + key: result[key] for key in ("request_id", "runtime_id", "invocation_id") + } + start = { + "schema": "nemo.fabric.run_record", + "version": "v1alpha1", + "kind": "start", + **identity, + } + end = { + **start, + "kind": "end", + "status": "succeeded", + "elapsed_ms": 1000, + "outcome_scope": "invocation", + "model_usage": result["model_usage"], + } + path = tmp_path / "fallback.jsonl" + path.write_text(json.dumps(start) + "\n" + json.dumps(end) + "\n") + return path + + +@pytest.mark.parametrize("condition", ["missing", "invalid", "oversized", "direct"]) +def test_record_accounting_survives_missing_full_result_without_creating_outer_success( + tmp_path, attempt, record, condition +): + from nemo_fabric.reporting import MAX_RESULT_BYTES + + request = attempt["requests"][0] + if condition == "direct": + request["evidence"] = "fallback.jsonl" + else: + request["record"] = "fallback.jsonl" + request["evidence"] = "unavailable.json" + if condition == "invalid": + (tmp_path / "unavailable.json").write_text("{") + elif condition == "oversized": + with (tmp_path / "unavailable.json").open("wb") as stream: + stream.seek(MAX_RESULT_BYTES) + stream.write(b" ") + summary = report(tmp_path, [attempt]) + row = summary["attempts"][0] + view = row["requests"][0] + assert view["state"] != "available" + assert view["accounting_source"] == "record" + assert view["accounting_boundary"] == "core_invocation" + assert row["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + execution = summary["configurations"]["baseline"]["execution"] + assert execution["available_result_count"] == 0 + assert execution["missing_result_count"] == 1 + assert execution["statuses"] == {} + assert row["evaluation"]["status"] == "not_evaluated" + + +@pytest.mark.parametrize("condition", ["omitted_counts", "invalid_accounting"]) +def test_valid_full_result_is_authority_even_when_record_has_more_accounting( + tmp_path, attempt, result, record, condition +): + attempt["requests"][0]["record"] = "fallback.jsonl" + if condition == "omitted_counts": + result["model_usage"]["input_tokens"] = result["model_usage"][ + "total_tokens" + ] = None + result["model_usage"]["observed"]["input_tokens"] = 50 + result["model_usage"]["gaps"].append( + {"dimensions": ["input_tokens", "total_tokens"], "reason": "missing_usage"} + ) + else: + result["model_usage"]["input_tokens"] = -1 + (tmp_path / "result.json").write_text(json.dumps(result)) + row = report(tmp_path, [attempt])["attempts"][0] + assert row["requests"][0]["accounting_source"] == "result" + assert "record_summary" not in row["requests"][0] + assert row["usage"]["dimensions"]["input_tokens"]["complete_total"] is None + assert row["usage"]["dimensions"]["input_tokens"]["observed_sum"] == ( + 50 if condition == "omitted_counts" else None + ) + + +@pytest.mark.parametrize( + "condition", ["wrong_request", "wrong_runtime", "startup_conflict", "legacy"] +) +def test_record_fallback_checks_canonical_identity_and_never_promotes_legacy_counts( + tmp_path, attempt, result, record, condition +): + request = attempt["requests"][0] + request.update(evidence="request-end.json", record="fallback.jsonl") + if condition == "wrong_runtime": + outer = { + "schema": "nemo.fabric.request_record", + "version": "v1alpha1", + "kind": "end", + "state": "returned", + "time": "2026-10-21T00:00:00Z", + "request_id": "request", + "runtime_id": "different", + "invocation_id": "invocation", + "result": "result.json", + } + (tmp_path / "request-end.json").write_text(json.dumps(outer)) + elif condition == "startup_conflict": + outer = { + "schema": "nemo.fabric.request_record", + "version": "v1alpha1", + "kind": "end", + "state": "no_result", + "stage": "start", + "time": "2026-10-21T00:00:00Z", + "request_id": "request", + } + (tmp_path / "request-end.json").write_text(json.dumps(outer)) + else: + lines = [json.loads(line) for line in record.read_text().splitlines()] + if condition == "wrong_request": + for line in lines: + line["request_id"] = "another" + else: + lines[1].pop("model_usage") + lines[1]["usage"] = {"input_tokens": 100} + record.write_text("".join(json.dumps(line) + "\n" for line in lines)) + row = report(tmp_path, [attempt])["attempts"][0] + assert "accounting_source" not in row["requests"][0] + assert row["usage"]["dimensions"]["input_tokens"]["complete_total"] is None + assert row["usage"]["dimensions"]["input_tokens"]["observed_sum"] is None + + +def test_optional_record_path_has_same_containment_boundary(tmp_path, attempt): + attempt["requests"][0]["record"] = "../outside.jsonl" + with pytest.raises(ValueError, match="within"): + report(tmp_path, [attempt]) + + +def test_unknown_sibling_fields_do_not_abort_comparison( + tmp_path, attempt, result, record +): + lines = [json.loads(line) for line in record.read_text().splitlines()] + lines[0]["resources"] = {"error": {"invalid": "shape"}} + lines[0]["resource_scope"] = "runtime_adapter_host_process_tree" + lines[1]["resource_scope"] = "runtime_adapter_host_process_tree" + (tmp_path / "run-record.jsonl").write_text( + "".join(json.dumps(line) + "\n" for line in lines) + ) + summary = report(tmp_path, [attempt]) + group = summary["configurations"]["baseline"] + assert group["execution"]["statuses"] == {"succeeded": 1} + assert group["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + view = summary["attempts"][0]["requests"][0] + assert view["accounting_source"] == "result" + + +@pytest.mark.parametrize( + ("sources", "status"), + [ + ([("baseline", "a")], "aligned"), + ([("baseline", "a"), ("candidate", "a")], "aligned"), + ([("baseline", "bundled"), ("candidate", "bundled-null")], "aligned"), + ([("baseline", "a"), ("candidate", "b")], "different_bases"), + ([("baseline", "a"), ("baseline", "b")], "different_bases"), + ([("baseline", None)], "unavailable"), + ([("baseline", "a"), ("candidate", None)], "unverified"), + ], + ids=[ + "one", + "same", + "omitted-versus-null-override", + "different", + "mixed-within-config", + "unpriced", + "partly-priced", + ], +) +def test_catalog_comparison_labels_bases_without_changing_configuration_totals( + tmp_path, result, sources, status +): + attempts = [] + for index, (configuration, catalog) in enumerate(sources): + value = deepcopy(result) + request_id = f"request-{index}" + value.update(request_id=request_id, invocation_id=request_id) + if catalog is not None: + multiplier = 2 if catalog == "b" else 1 + usage = value["model_usage"] + usage["gaps"] = [] + usage["cost"] = { + "reported": {}, + "estimated": { + "amount_nano_usd": 104000 * multiplier, + "observed_nano_usd": 104000 * multiplier, + }, + "pricing": { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "c" * 64, + "user_catalog_sha256": "sha256:" + catalog * 64, + }, + } + if catalog == "bundled": + usage["cost"]["pricing"].pop("user_catalog_sha256") + elif catalog == "bundled-null": + usage["cost"]["pricing"]["user_catalog_sha256"] = None + usage["models_exhaustive"] = True + usage["models"] = [ + { + key: deepcopy(item) + for key, item in usage.items() + if key not in {"models_exhaustive", "models"} + } + | {"provider": "openai", "model": "model"} + ] + filename = request_id + ".json" + (tmp_path / filename).write_text(json.dumps(value)) + attempts.append( + { + "attempt_id": str(index), + "task_id": "same-task", + "task_revision": "same-revision", + "configuration_id": configuration, + "requests_complete": True, + "requests": [{"request_id": request_id, "evidence": filename}], + } + ) + summary = report(tmp_path, attempts) + alignment = summary["pricing_alignment"] + assert alignment["status"] == status + assert alignment["unpriced_configurations"] == sorted( + {configuration for configuration, catalog in sources if catalog is None} + ) + text = render_comparison_summary(summary, details=True) + compact = render_comparison_summary(summary) + for configuration in {name for name, _ in sources}: + catalogs = {catalog for name, catalog in sources if name == configuration} + bases = alignment["configuration_bases"][configuration] + assert {basis.get("user_catalog_sha256") for basis in bases} == { + None if catalog.startswith("bundled") else "sha256:" + catalog * 64 + for catalog in catalogs + if catalog is not None + } + dimension = summary["configurations"][configuration]["usage"]["dimensions"] + count = sum(name == configuration for name, _ in sources) + assert dimension["input_tokens"]["complete_total"] == 100 * count + if len(catalogs) == 1 and None not in catalogs: + catalog = next(iter(catalogs)) + expected = 104000 * count * (2 if catalog == "b" else 1) + assert dimension["estimated_cost"]["complete_total"] == expected + else: + assert dimension["estimated_cost"]["complete_total"] is None + if bases: + assert "Catalog basis:" in compact or "Catalog bases (" in compact + if len(bases) > 1: + assert f"Catalog bases ({len(bases)}):" in compact + for basis in bases: + bundled = basis["bundled_catalog_sha256"] + assert bundled.removeprefix("sha256:")[:12] in compact + assert bundled not in compact + assert basis["bundled_catalog_sha256"] in text + if basis.get("user_catalog_sha256") is not None: + assert basis["user_catalog_sha256"] in text + assert ("Catalog pricing bases differ" in text) == (status == "different_bases") + if alignment["unpriced_configurations"]: + assert "No priced catalog evidence" in text + else: + assert "No priced catalog evidence" not in text + + +def test_report_shows_independent_metric_and_evaluation_coverage( + tmp_path, attempt, result +): + result["model_usage"]["input_tokens"] = None + result["model_usage"]["total_tokens"] = None + result["model_usage"]["observed"]["input_tokens"] = 50 + result["model_usage"]["gaps"].append( + {"dimensions": ["input_tokens", "total_tokens"], "reason": "unfinished_work"} + ) + (tmp_path / "result.json").write_text(json.dumps(result)) + attempt["evaluation"] = {"status": "passed", "evaluator": "tests", "version": "v1"} + text = render_comparison_summary(report(tmp_path, [attempt])) + assert "50 observed, incomplete (0/1 complete)" in text + assert "2 (1/1 complete)" in text + assert "unavailable (0/1 complete)" in text + assert "1/1 passed; 1/1 evaluated" in text + assert "Total tokens" in text + + +def test_inventory_warnings_precede_table_and_missing_results_remain_visible( + tmp_path, attempt +): + attempt["requests_complete"] = False + other = { + **attempt, + "attempt_id": "missing", + "configuration_id": "other", + "task_revision": "v2", + "requests": [{"request_id": "missing", "evidence": "missing.json"}], + } + text = render_comparison_summary(report(tmp_path, [attempt, other], complete=False)) + header = text.index("Configuration |") + assert text.index("Attempt inventory is unverified") < header + assert text.index("Request inventories are unverified") < header + assert text.index("Listed task/revision attempt inventories differ") < header + assert "0/1 returned; 1 missing" in text + + +def test_timing_uses_caller_attempt_median_and_rejects_mixed_scopes( + tmp_path, attempt, result +): + other = {**attempt, "attempt_id": "other", "elapsed_ms": 10000, "requests": []} + summary = report(tmp_path, [attempt, other]) + text = render_comparison_summary(summary) + assert "8 s median (2/2 timed)" in text + assert "caller_attempt" in text + assert "16 s" not in text + other["elapsed_scope"] = "adapter_exchange" + text = render_comparison_summary(report(tmp_path, [attempt, other])) + assert "Timing scopes differ" in text + assert "mixed scopes (2/2 timed)" in text + assert " s median (" not in text + + +def test_comparison_details_keep_verbose_evidence_out_of_default(tmp_path, attempt): + summary = report(tmp_path, [attempt]) + text = render_comparison_summary(summary) + assert "Listed task revisions:" not in text + assert "Caller timing boundaries:" not in text + assert "Accounting sources:" not in text + detail = render_comparison_summary(summary, details=True) + assert "Listed task revisions:" in detail + assert "Caller timing boundaries:" in detail + assert "Accounting sources:" in detail + + +@pytest.mark.parametrize("mode", ["complete", "observed", "absent"]) +def test_provider_cost_remains_visible_when_available(tmp_path, attempt, result, mode): + cost = result["model_usage"]["cost"]["reported"] + if mode != "complete": + cost["amount_nano_usd"] = None + cost["observed_nano_usd"] = 10 if mode == "observed" else None + result["model_usage"]["gaps"].append( + {"dimensions": ["reported_cost"], "reason": "missing_usage"} + ) + (tmp_path / "result.json").write_text(json.dumps(result)) + summary = report(tmp_path, [attempt]) + text = render_comparison_summary(summary) + if mode == "absent": + assert "Provider-reported cost:" not in text + else: + assert "Provider-reported cost: $0.000000010" in text + assert ("$0.000000010 observed, incomplete" in text) == (mode == "observed") + assert "Provider-reported cost:" in render_comparison_summary(summary, details=True) From a6318d4dede15289bbf31485296c6bec8bc524b0 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:24:17 -0700 Subject: [PATCH 076/102] chore: check packaged usage schema declarations Keep packaged adapter declarations in the canonical usage-schema synchronization and drift check. Prevent the check from passing with a stale packaged schema. Signed-off-by: mnajafian-nv --- scripts/contracts/generate_usage_accounting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/contracts/generate_usage_accounting.py b/scripts/contracts/generate_usage_accounting.py index 49bb52bef..3b5cbd19c 100644 --- a/scripts/contracts/generate_usage_accounting.py +++ b/scripts/contracts/generate_usage_accounting.py @@ -22,6 +22,7 @@ def synchronize(*, check: bool) -> None: descriptors = [ *ROOT.glob("adapters/python/*/*.fabric-adapter.json"), *ROOT.glob("adapters/typescript/*/*.fabric-adapter.json"), + *ROOT.glob("crates/fabric-cli/assets/adapters/*/*.fabric-adapter.json"), ] for path in descriptors: original = path.read_text() From d35ebb3fa8b055d80d69375ccf5b42e896401092 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:38:47 -0700 Subject: [PATCH 077/102] fix: qualify child model pricing by provider endpoint Check the source-qualified child provider against its own inherited endpoint before applying bundled prices. Keep native served aliases and missing-provider behavior unchanged. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/pricing.rs | 56 ++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/pricing.rs b/crates/fabric-core/src/pricing.rs index 191c1f965..008c576d6 100644 --- a/crates/fabric-core/src/pricing.rs +++ b/crates/fabric-core/src/pricing.rs @@ -27,7 +27,21 @@ pub(crate) fn native_catalog_route( forwarded: &std::collections::BTreeMap, inherited: impl Fn(&str) -> Option, ) -> bool { - let (keys, urls): (&[&str], &[&str]) = match model.provider.as_str() { + native_catalog_endpoint( + &model.provider, + model.base_url.as_deref(), + forwarded, + inherited, + ) +} + +fn native_catalog_endpoint( + provider: &str, + base_url: Option<&str>, + forwarded: &std::collections::BTreeMap, + inherited: impl Fn(&str) -> Option, +) -> bool { + let (keys, urls): (&[&str], &[&str]) = match provider { "openai" => (&OPENAI_ENDPOINT_ENV_KEYS, &["https://api.openai.com/v1"]), "anthropic" => ( &ANTHROPIC_ENDPOINT_ENV_KEYS, @@ -36,7 +50,7 @@ pub(crate) fn native_catalog_route( _ => return false, }; let native = |url: &str| urls.contains(&url.trim_end_matches('/')); - if let Some(url) = &model.base_url { + if let Some(url) = base_url { return native(url); } keys.iter().all(|key| { @@ -53,6 +67,7 @@ pub(crate) fn native_catalog_route( #[derive(Debug, Clone)] pub(crate) struct NativePricingRoutes { models: Vec, + endpoints: std::collections::BTreeMap, } #[derive(Debug, Clone)] @@ -76,6 +91,7 @@ impl NativePricingRoutes { native: native_catalog_route(model, endpoints, |_| None), }) .collect(), + endpoints: endpoints.clone(), } } @@ -98,8 +114,13 @@ impl NativePricingRoutes { if provider_routes.peek().is_some() { return provider_routes.all(|route| route.native); } - // Preserve the prior all-native case for source-qualified child models. - !self.models.is_empty() && self.models.iter().all(|route| route.native) + // A child provider's inherited endpoint can differ from configured roles. + let accounted_endpoint_native = provider.is_none_or(|provider| { + native_catalog_endpoint(provider, None, &self.endpoints, |_| None) + }); + accounted_endpoint_native + && !self.models.is_empty() + && self.models.iter().all(|route| route.native) } } @@ -712,6 +733,33 @@ mod tests { assert!(!empty.allows(Some("openai"), "gpt-5.4")); } + #[test] + fn child_provider_pricing_checks_its_own_endpoint() { + let models = serde_json::from_value(json!({ + "default": {"provider": "openai", "model": "gpt-5.4"} + })) + .unwrap(); + for (endpoint, expected) in [ + (None, true), + (Some("https://api.anthropic.com/"), true), + (Some("https://proxy.invalid"), false), + ] { + let endpoints = endpoint + .map(|url| BTreeMap::from([("ANTHROPIC_BASE_URL".into(), url.into())])) + .unwrap_or_default(); + let routes = super::NativePricingRoutes::from_models(&models, &endpoints); + assert_eq!( + routes.allows(Some("anthropic"), "source-qualified-child"), + expected, + "child endpoint: {endpoint:?}" + ); + assert!(routes.allows(Some("openai"), "served-alias")); + assert!(routes.allows(None, "source-qualified-child")); + } + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(!routes.allows(Some("unknown-provider"), "source-qualified-child")); + } + #[test] fn inherited_and_forwarded_custom_routes_require_explicit_prices() { for (configured, inherited, forwarded, expected) in [ From c198472ce1ab461c90ccfc20402ba2252942e614 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:40:15 -0700 Subject: [PATCH 078/102] fix: return detached applied pricing rates in Python Return the documented JSON-compatible rate snapshot instead of exposing a frozen internal mapping. Preserve optional cache rates and snapshot isolation. Signed-off-by: mnajafian-nv --- sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py | 1 + tests/python/test_model_usage.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index bea167f4a..f5a52cc4c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2577,6 +2577,7 @@ class AppliedPricing(FabricMapping): "rates", } ) + _json_fields = frozenset({"rates"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index fb018ffaa..805263033 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -486,9 +486,13 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): "rates": {"input": 10, "output": 20, **cache_rates}, } ) - assert parsed.rates == { + expected = { "input": 10, "output": 20, "cache_read": cache_rates.get("cache_read"), "cache_write": cache_rates.get("cache_write"), } + rates = parsed.rates + assert json.loads(json.dumps(rates)) == expected + rates["input"] = 999 + assert parsed.rates == expected From 06e2396de292a90c375e25a9d0dd623c9a5688f6 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:41:47 -0700 Subject: [PATCH 079/102] fix: preserve frozen catalog pricing for OpenShell results Keep host catalog paths out of task planning and apply the frozen explicit catalog at the host boundary. Preserve validated task-native bundled estimates when no override is configured. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/pricing.rs | 2 +- crates/fabric-core/src/runtime/openshell.rs | 266 ++++++++++++++++++++ docs/sdk/python.mdx | 4 +- 3 files changed, 269 insertions(+), 3 deletions(-) diff --git a/crates/fabric-core/src/pricing.rs b/crates/fabric-core/src/pricing.rs index 008c576d6..f3eb8fe12 100644 --- a/crates/fabric-core/src/pricing.rs +++ b/crates/fabric-core/src/pricing.rs @@ -180,7 +180,7 @@ pub struct AppliedPricing { } impl RateSnapshot { - fn from_rates(rates: TokenPricingRates) -> Result { + pub(crate) fn from_rates(rates: TokenPricingRates) -> Result { let convert = |rate| { nano_usd_per_million_tokens(rate) .and_then(|value| u64::try_from(value).ok()) diff --git a/crates/fabric-core/src/runtime/openshell.rs b/crates/fabric-core/src/runtime/openshell.rs index 75dc78bc3..38e3403c1 100644 --- a/crates/fabric-core/src/runtime/openshell.rs +++ b/crates/fabric-core/src/runtime/openshell.rs @@ -129,6 +129,11 @@ fn task_config(plan: &RunPlan, environment: &EnvironmentHandle) -> Result Result, + provider: Option<&str>, + model: &str, + prices: &crate::price_catalog::PriceCatalogs<'_>, +) -> bool { + let Some(task) = task_usage else { + return false; + }; + let Some((bundled, _)) = prices.find(provider, model, false) else { + return false; + }; + let rates = match &bundled.rate_schedule { + Some(crate::price_catalog::TokenRateSchedule::PromptTokenThreshold { tiers, .. }) => { + tiers.iter().map(|tier| tier.rates).collect::>() + } + None => bundled.rates.into_iter().collect(), + }; + let canonical_rates = rates + .into_iter() + .filter_map(|rates| crate::pricing::RateSnapshot::from_rates(rates).ok()) + .collect::>(); + // A task-core tariff decision supplies sandbox-local route evidence. Host + // endpoint inheritance cannot establish which route the sandbox used. + task.models.iter().any(|entry| { + entry.provider.as_deref() == provider + && entry.model == model + && entry.cost.pricing.as_ref().is_some_and(|basis| { + basis.user_catalog_sha256.is_none() + && basis.bundled_catalog_sha256 == prices.bundled_sha256() + }) + && !entry.cost.applied_rates.is_empty() + && entry.cost.applied_rates.iter().all(|applied| { + applied.catalog_sha256 == prices.bundled_sha256() + && applied.provider == bundled.provider + && applied.model_id == bundled.model_id + && applied.pricing_as_of == bundled.pricing_as_of + && applied.pricing_source == bundled.pricing_source + && canonical_rates.contains(&applied.rates) + }) + }) +} +fn apply_host_catalog( + result: &mut RunResult, + catalog: Option<&crate::price_catalog::UserPriceCatalog>, +) { + let Some(catalog) = catalog else { + // The task core observes the sandbox's actual provider endpoints. Keep + // its independently fingerprinted bundled estimates when no override + // was supplied, rather than inferring routes from host environment. + return; + }; + let raw = result.usage.as_ref().map(|usage| AgentUsage { + input_tokens: usage.input_tokens, + cached_input_tokens: usage.cached_input_tokens, + input_tokens_include_cache: usage.input_tokens_include_cache, + output_tokens: usage.output_tokens, + total_tokens: usage.total_tokens, + cost_usd: usage.cost_usd, + extensions: usage.metadata.clone(), + }); + let task_usage = result.model_usage.as_ref(); + let prices = crate::price_catalog::PriceCatalogs::with_user_catalog(None); + result.model_usage = match raw + .as_ref() + .map(|usage| { + crate::model_usage::normalize_model_usage_for_routes( + usage, + Some(catalog), + |provider, model| task_has_bundled_tariff(task_usage, provider, model, &prices), + ) + }) + .transpose() + { + Ok(usage) => usage.flatten(), + Err(_) => { + if !result + .events + .iter() + .any(|event| event.kind == "model_usage_invalid") + { + result.events.push(event_with_metadata( + "model_usage_invalid", + "model accounting could not be normalized; execution outcome and raw usage were retained".to_string(), + BTreeMap::new(), + )); + } + None + } + }; +} fn set_state(key: &str, runtime_id: &str, state: State) { BINDINGS .lock() @@ -433,6 +529,7 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { "", )); } + apply_host_catalog(&mut result, plan.user_price_catalog.as_ref()); for telemetry in &mut result.telemetry { telemetry.relay_enabled = telemetry .metadata @@ -847,6 +944,175 @@ mod tests { } } + fn catalog_for_task_usage() -> crate::price_catalog::UserPriceCatalog { + crate::price_catalog::UserPriceCatalog::from_bytes( + PathBuf::from("host-only-catalog.json"), + br#"{"version":1,"entries":[{"provider":"fixture","model_id":"model","currency":"USD","unit":"per_token","rates":{"input_per_million":1.0,"output_per_million":2.0,"cache_read_per_million":0.1,"cache_write_per_million":1.25},"prompt_cache":{"read_accounting":"included_in_prompt_tokens"},"pricing_as_of":"2026-10-22","pricing_source":"test fixture"}]}"#, + ).expect("frozen test catalog") + } + + fn task_usage_result() -> RunResult { + let raw: AgentUsage = serde_json::from_value(serde_json::json!({ + "extensions": {"nemo_fabric.usage_accounting.v1": { + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "provider": "fixture", "gaps": [], + "reported": {"input_tokens": 100, "input_tokens_include_cache": true, + "cached_input_tokens": 0, "cache_write_input_tokens": 0, + "output_tokens": 5, "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 100} + }] + }} + })) + .expect("raw task accounting"); + let mut result: RunResult = serde_json::from_value(serde_json::json!({ + "agent_name": "agent", "harness": "fixture", "adapter_kind": "process", + "runtime_id": "runtime-task", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "failed", "output": "partial output", + "error": {"stage": "invoke", "code": "fixture_failure", "message": "fixture failure", "retryable": false} + })).expect("task result"); + result.usage = Some(run_usage(&raw)); + result.model_usage = crate::model_usage::normalize_model_usage(&raw, None, false) + .expect("normalized task accounting"); + result + } + + #[test] + fn task_projection_does_not_send_a_host_catalog_path_to_the_sandbox() { + let plan: RunPlan = serde_json::from_value(serde_json::json!({ + "agent_name": "agent", "base_dir": "/host", "agent_config": {}, "capabilities": {}, + "config": {"schema_version": "fabric.agent/v1alpha1", "metadata": {"name": "agent"}, + "runtime": {}, "pricing": {"catalog_path": "/host-only/catalog.json"}} + })) + .expect("projection plan"); + let projected = task_config(&plan, &environment()).expect("task configuration"); + assert!(projected.pricing.unwrap().catalog_path.is_none()); + assert_eq!( + plan.config.pricing.unwrap().catalog_path, + Some(PathBuf::from("/host-only/catalog.json")) + ); + } + + #[test] + fn task_usage_uses_the_host_frozen_catalog_without_changing_the_outcome() { + let mut result = task_usage_result(); + let original = result.clone(); + assert!( + result + .model_usage + .as_ref() + .unwrap() + .cost + .estimated + .amount_nano_usd + .is_none() + ); + let catalog = catalog_for_task_usage(); + apply_host_catalog(&mut result, Some(&catalog)); + let usage = result + .model_usage + .as_ref() + .expect("host-normalized accounting"); + assert_eq!(usage.cost.estimated.amount_nano_usd, Some(110_000)); + assert_eq!( + usage + .cost + .pricing + .as_ref() + .unwrap() + .user_catalog_sha256 + .as_deref(), + Some(catalog.sha256.as_str()) + ); + assert_eq!(result.status, original.status); + assert_eq!(result.error, original.error); + assert_eq!(result.output, original.output); + assert_eq!(result.usage, original.usage); + } + + #[test] + fn task_usage_without_a_host_override_keeps_the_task_core_route_evidence() { + let mut result = task_usage_result(); + let mut raw = AgentUsage::default(); + raw.extensions = result.usage.as_ref().unwrap().metadata.clone(); + let model = &mut raw + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]; + model["provider"] = Value::from("openai"); + model["model"] = Value::from("gpt-5.4"); + result.usage = Some(run_usage(&raw)); + result.model_usage = crate::model_usage::normalize_model_usage(&raw, None, true).unwrap(); + assert!( + result + .model_usage + .as_ref() + .unwrap() + .cost + .estimated + .amount_nano_usd + .is_some() + ); + let original = result.clone(); + apply_host_catalog(&mut result, None); + assert_eq!(result, original); + apply_host_catalog(&mut result, Some(&catalog_for_task_usage())); + assert_eq!( + result.model_usage.as_ref().unwrap().cost.estimated, + original.model_usage.as_ref().unwrap().cost.estimated, + "a partial override keeps the task-qualified bundled tariff for other models" + ); + for invalid_proof in ["provider", "fingerprint", "rates", "user_catalog"] { + let mut unqualified = original.clone(); + let entry = &mut unqualified.model_usage.as_mut().unwrap().models[0]; + match invalid_proof { + "provider" => entry.provider = Some("another-provider".into()), + "fingerprint" => { + entry.cost.pricing.as_mut().unwrap().bundled_catalog_sha256 = + "sha256:another-catalog".into() + } + "rates" => entry.cost.applied_rates[0].rates.input += 1, + "user_catalog" => { + entry.cost.pricing.as_mut().unwrap().user_catalog_sha256 = + Some("sha256:user-catalog".into()) + } + _ => unreachable!(), + } + apply_host_catalog(&mut unqualified, Some(&catalog_for_task_usage())); + assert!( + unqualified + .model_usage + .as_ref() + .unwrap() + .cost + .estimated + .amount_nano_usd + .is_none(), + "{invalid_proof}" + ); + } + } + + #[test] + fn invalid_task_accounting_with_a_host_catalog_keeps_the_execution_and_raw_evidence() { + let mut result = task_usage_result(); + result.usage.as_mut().unwrap().metadata.insert( + "nemo_fabric.usage_accounting.v1".into(), + serde_json::json!({"models_exhaustive": "invalid"}), + ); + let original = result.clone(); + apply_host_catalog(&mut result, Some(&catalog_for_task_usage())); + assert!(result.model_usage.is_none()); + assert_eq!(result.status, original.status); + assert_eq!(result.error, original.error); + assert_eq!(result.usage, original.usage); + assert!( + result + .events + .iter() + .any(|event| event.kind == "model_usage_invalid") + ); + } + #[test] fn unknown_binding_survives_a_new_host_environment_handle() { let original = environment(); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index ae64c7989..c6cb2bfeb 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1231,8 +1231,8 @@ accounting remains available in `result.usage` and is not certified complete by `cost.reported` contains only provider-reported amounts; `cost.estimated` contains catalog estimates. Both distinguish `amount_nano_usd` from `observed_nano_usd`. Estimates retain their pricing basis and distinct selected rate snapshots. -Configure `PricingConfig(catalog_path="prices.json")` for a Relay-compatible -catalog override; planning freezes its bytes. An explicitly established provider must match the catalog entry's provider, including aliases and shorter model paths. A different provider's entry cannot price that source solely because the model name matches. When no provider identity is available, an explicit user catalog can match the model name alone. Bundled provider prices apply only +Set `FabricConfig(pricing=PricingConfig(catalog_path="prices.json"))` for a +Relay-compatible catalog override; planning freezes its bytes. An explicitly established provider must match the catalog entry's provider, including aliases and shorter model paths. A different provider's entry cannot price that source solely because the model name matches. When no provider identity is available, an explicit user catalog can match the model name alone. Bundled provider prices apply only when the configured route establishes the native provider. Unknown model, cache-price or request-tier evidence affects estimates without erasing tokens. Nonzero Anthropic cache writes currently lack expiry-price evidence, so their From 630820f93384bf5ce9729edd1e2c9c18d63f1999 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:44:48 -0700 Subject: [PATCH 080/102] fix: return detached applied pricing rates in Python Return the documented JSON-compatible rate snapshot instead of exposing a frozen internal mapping. Preserve optional cache rates and snapshot isolation. Signed-off-by: mnajafian-nv --- sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py | 1 + tests/python/test_model_usage.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index bea167f4a..f5a52cc4c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2577,6 +2577,7 @@ class AppliedPricing(FabricMapping): "rates", } ) + _json_fields = frozenset({"rates"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index fb018ffaa..805263033 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -486,9 +486,13 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): "rates": {"input": 10, "output": 20, **cache_rates}, } ) - assert parsed.rates == { + expected = { "input": 10, "output": 20, "cache_read": cache_rates.get("cache_read"), "cache_write": cache_rates.get("cache_write"), } + rates = parsed.rates + assert json.loads(json.dumps(rates)) == expected + rates["input"] = 999 + assert parsed.rates == expected From 93b00ecdbf82a618e8b6947692b9ea333f59f38b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 17:23:32 -0700 Subject: [PATCH 081/102] fix: expose omitted optional SDK usage attributes Signed-off-by: mnajafian-nv --- .../nemo_fabric.types.md | 16 +++++++- .../src/nemo_fabric/types.py | 12 +++++- tests/python/test_model_usage.py | 39 +++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 83cdfab2b..b930369db 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -2067,7 +2067,6 @@ The mapping exposes the following typed fields: | `normalization_version` | `str` | | `assumption_policy` | `str` | | `bundled_catalog_sha256` | `str` | -| `user_catalog_sha256` | `str \| None` | ### method `__init__` @@ -2088,6 +2087,13 @@ Return an immutable view of preserved extension fields. +--- + +### property user_catalog_sha256 + +Return the user catalog fingerprint when supplied, otherwise None. + + --- ### classmethod `from_mapping` @@ -2475,7 +2481,6 @@ The mapping exposes the following typed fields: | Field | Type | | --- | --- | | `model` | `str` | -| `provider` | `str \| None` | | `input_tokens` | `int \| None` | | `cached_input_tokens` | `int \| None` | | `cache_write_input_tokens` | `int \| None` | @@ -2504,6 +2509,13 @@ Return an immutable view of preserved extension fields. +--- + +### property provider + +Return the accounting provider when reported, otherwise None. + + --- ### classmethod `from_mapping` diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index f5a52cc4c..2dfcd259b 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2527,7 +2527,6 @@ class PricingBasis(FabricMapping): normalization_version: str assumption_policy: str bundled_catalog_sha256: str - user_catalog_sha256: str | None _fields = frozenset( { "basis", @@ -2539,6 +2538,11 @@ class PricingBasis(FabricMapping): } ) + @property + def user_catalog_sha256(self) -> str | None: + """Return the user catalog fingerprint when supplied, otherwise None.""" + return self._data.get("user_catalog_sha256") + @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if set(data).difference(cls._fields) or any( @@ -2796,7 +2800,6 @@ class PerModelUsage(FabricMapping): """ model: str - provider: str | None input_tokens: int | None cached_input_tokens: int | None cache_write_input_tokens: int | None @@ -2809,6 +2812,11 @@ class PerModelUsage(FabricMapping): {*_MODEL_USAGE_TOKEN_FIELDS, "model", "provider", "observed", "gaps", "cost"} ) + @property + def provider(self) -> str | None: + """Return the accounting provider when reported, otherwise None.""" + return self._data.get("provider") + @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: data["model"] = _required_text(data.get("model"), "model") diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 805263033..4c731b546 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -496,3 +496,42 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): assert json.loads(json.dumps(rates)) == expected rates["input"] = 999 assert parsed.rates == expected + + +@pytest.mark.parametrize( + "provider", [{}, {"provider": None}, {"provider": "fixture-provider"}] +) +def test_per_model_optional_provider_attribute_preserves_source_presence( + per_model_usage, provider +): + from nemo_fabric.types import PerModelUsage + + per_model_usage.update(provider) + parsed = PerModelUsage.from_mapping(per_model_usage) + assert parsed.provider == provider.get("provider") + serialized = parsed.to_mapping() + assert ("provider" in serialized) == ("provider" in provider) + if "provider" in provider: + assert serialized["provider"] == provider["provider"] + + +@pytest.mark.parametrize( + "user_catalog", + [{}, {"user_catalog_sha256": None}, {"user_catalog_sha256": "sha256:" + "b" * 64}], +) +def test_pricing_basis_optional_user_catalog_attribute_preserves_source_presence( + user_catalog, +): + from nemo_fabric.types import PricingBasis + + source = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + **user_catalog, + } + parsed = PricingBasis.from_mapping(source) + assert parsed.user_catalog_sha256 == user_catalog.get("user_catalog_sha256") + assert parsed.to_mapping() == source From f5aa5beaf69a793dde7c80fa36447a0b229cc0e0 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 18:15:27 -0700 Subject: [PATCH 082/102] docs: clarify run record crash durability Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/writer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs index a1ece00d5..b8240ba33 100644 --- a/crates/fabric-core/src/record/writer.rs +++ b/crates/fabric-core/src/record/writer.rs @@ -3,10 +3,10 @@ //! The run record file: created once, owner-only, appended one line per write. //! -//! Each line is one `write_all` on an unbuffered file, so a SIGKILL of Fabric -//! or the adapter host keeps every line whose write returned. Only the end -//! line is synced, once, so the record survives a crash of the machine only -//! up to its last synced line. +//! Each line uses `write_all` on an unbuffered file. If the Fabric process +//! terminates, lines whose writes completed remain in the file. The end line +//! is synced once. Durability after a machine crash or power loss is not +//! guaranteed because the containing directory is not synced. use std::fs::{File, OpenOptions}; use std::io::{self, Write}; From 101bce6206a4ce6663e1fc876f99c35404de6fe8 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 20:53:44 -0700 Subject: [PATCH 083/102] fix: reject Relay exit during successful health probe Signed-off-by: mnajafian-nv --- adapters/python/common/README.md | 6 ++++-- .../nemo_fabric_adapters/common/relay_gateway.py | 5 +++++ .../adapters/test_adapters_common_relay_gateway.py | 13 +++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index 919bdd233..df9e8aa93 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -126,8 +126,10 @@ runtime. The Claude and Codex adapters wait up to 30 seconds for the supervised Relay gateway's HTTP health endpoint. This budget includes configuration validation -and required worker initialization. Early process exit fails startup immediately; -a gateway that never becomes healthy fails when the budget expires and is stopped. +and required worker initialization. Process exit is checked before each health +probe and again before accepting a successful response. An exit during an +unsuccessful probe is detected at the next poll. A gateway that never becomes +healthy fails when the budget expires and is stopped. The adapters retain its log for diagnosis. Local Python hosts separately enforce a 90-second adapter startup limit. diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 959f54fa7..1a57c5fe4 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -132,6 +132,11 @@ def wait_for_relay_gateway( health_url, timeout=min(1.0, remaining) ) as response: if 200 <= response.status < 300: + returncode = process.poll() + if returncode is not None: + raise RelayGatewayError( + f"NeMo Relay gateway exited with status {returncode} before becoming ready" + ) return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index e478f85de..fbd9ca893 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -239,6 +239,19 @@ def test_wait_for_relay_gateway_reports_early_exit(): ) +def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch): + process = MagicMock() + process.poll.side_effect = [None, 17] + response = MagicMock() + response.__enter__.return_value.status = 200 + monkeypatch.setattr( + relay_gateway.urllib.request, "urlopen", MagicMock(return_value=response) + ) + + with pytest.raises(relay_gateway.RelayGatewayError, match="status 17"): + relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From 2014cbe68af1259c837ab3da256883ff3dcd4a74 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 21:28:53 -0700 Subject: [PATCH 084/102] fix: exclude unknown keys from request usage Signed-off-by: mnajafian-nv --- adapter-contract/typescript/src/usage.ts | 7 ++++++- adapter-contract/typescript/test/usage-reducer.test.js | 8 ++++++++ adapters/python/deepagents/README.md | 7 +++++++ docs/sdk/python.mdx | 2 +- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index cb8f0dd9f..16c8cc5ae 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -144,7 +144,12 @@ function aligned(records: readonly UsageObservation[]): boolean { function requestUsage(record: UsageObservation): ReportedUsage { // Retain uncertain literal counts in model observations, not complete calls. - const fields = { ...record.reported }; + const fields: ReportedUsage = {}; + const keys = [...dimensions, "input_tokens_include_cache", "output_tokens_include_reasoning"] as const; + for (const key of keys) { + const value = record.reported[key]; + if (value !== undefined) (fields as Record)[key] = value; + } for (const gap of record.gaps ?? []) for (const dimension of gap.dimensions) delete fields[dimension]; return fields; } diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js index e83f1ad8b..de7fbdfb2 100644 --- a/adapter-contract/typescript/test/usage-reducer.test.js +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -46,3 +46,11 @@ test("scoped request gaps retain independent fields without certifying cache", ( assert.equal(model.requests[0].gap, undefined); assert.equal(model.requests[0].tokens_aligned, true); }); + +test("request usage omits provider-specific keys outside the wire schema", () => { + const usage = usageFromObservations([{ model: "m", is_model_call: true, + reported: { input_tokens: 7, input_tokens_include_cache: true, prompt_tokens: 7 }, + }]); + const request = usage.extensions[USAGE_ACCOUNTING_KEY].models[0].requests[0].usage; + assert.deepEqual(request, { input_tokens: 7, input_tokens_include_cache: true }); +}); diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index fdfb09f2e..af2f9fc34 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -241,6 +241,13 @@ The descriptor declares `nemo_fabric.usage_accounting.v1` for complete dimension observations and gaps, and `nemo_fabric.detailed_token_usage.v1` for raw cache and reasoning conventions. An omitted count stays unknown; a reported zero stays zero. +**Upgrade note:** For providers whose usage is visible only in LangGraph +messages, `RunResult.usage.input_tokens` and `output_tokens` can now be `null`. +The previous graph-message subtotal remains in `RunResult.output.usage`, labeled +`usage_source: graph_state_messages`. It is a diagnostic subtotal, not a +complete invocation count; consumers that previously read the top-level fields +should handle unknown counts rather than substituting this subtotal. + For adapter-created OpenAI-compatible clients, the collector observes sync and async calls and their HTTP attempts without changing requests or retries. Normal reported calls stay known. An actual failed or unfinished attempt without usable diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index f55fe8ba9..d47a7bdb1 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1229,7 +1229,7 @@ accounting remains available in `result.usage` and is not certified complete by `summarize_usage`. `cost.reported` contains only provider-reported amounts. Complete and observed -amounts use integer nano-USD. Catalog estimates remain unavailable in this slice; +amounts use integer nano-USD. Catalog estimates are not available; `cost.estimated` has no amounts and no typed pricing setting is exposed. The stable result shape retains optional pricing-basis fields for persisted evidence. From 3fd1566c08cb7307af1005597fc42dc017f3f57e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:10:52 -0700 Subject: [PATCH 085/102] fix: skip execution-host pricing during inspection Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 19fc846d8..3d975e04f 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -2465,6 +2465,8 @@ pub fn inspect_adapter_config( ); } config.discovery = None; + // Inspection returns metadata only; catalog paths belong to the execution host. + config.pricing = None; validate_config(&config)?; let path = descriptor_path .canonicalize() @@ -4368,6 +4370,10 @@ mod tests { local_paths: vec![PathBuf::from("/task-only/nonexistent-descriptor")], ..Default::default() }); + config.pricing = Some(PricingConfig { + catalog_path: Some(PathBuf::from("/task-only/nonexistent-pricing-catalog.json")), + ..Default::default() + }); let inspection = inspect_adapter_config(config.clone(), &path).expect("host inspection"); assert_eq!( inspection.descriptor, From 6efff4d4e0345c9e2c5373d9874ffd48b995ffa2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:11:34 -0700 Subject: [PATCH 086/102] docs: use full product name in pricing references Signed-off-by: mnajafian-nv --- crates/fabric-core/src/price_catalog.rs | 4 ++-- .../nemo-fabric-core/price-catalog/index.mdx | 2 +- .../nemo-fabric-core/price-catalog/struct-modelpricing.mdx | 2 +- schemas/sdk/run-plan.schema.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs index f457829f1..108bb3563 100644 --- a/crates/fabric-core/src/price_catalog.rs +++ b/crates/fabric-core/src/price_catalog.rs @@ -4,7 +4,7 @@ //! Model price catalogs for estimating token cost. //! //! A catalog uses the NeMo Relay model pricing catalog format, version 1, so -//! one file can configure both tools. NeMo Fabric parses the format itself and +//! one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and //! does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. //! An explicit provider restricts aliases and model-path suffixes to that provider. //! Without a provider, model-only lookup uses the first matching entry in listed order. @@ -74,7 +74,7 @@ pub struct ModelPricing { /// Other identifiers of the same model, such as dated snapshots. #[serde(default)] pub aliases: Vec, - /// ISO 4217 currency of the rates. NeMo Fabric estimates cost only from + /// ISO 4217 currency of the rates. NVIDIA NeMo Fabric estimates cost only from /// `USD` entries. #[serde(default = "default_currency")] pub currency: String, diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx index c9aa13b29..07a60a689 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx @@ -12,7 +12,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Model price catalogs for estimating token cost. -A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, model-only lookup uses the first matching entry in listed order. +A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, model-only lookup uses the first matching entry in listed order. NeMo Fabric ships one catalog, embedded at build time, with the list prices of models that its presets, docs, and examples use. Each entry cites the provider pricing page it was taken from and the date it was checked. A caller can add a catalog of their own, whose entries take precedence. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx index 814233c4e..4b9be5cf7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx @@ -30,7 +30,7 @@ Other identifiers of the same model, such as dated snapshots. ### `currency: String` -ISO 4217 currency of the rates. NeMo Fabric estimates cost only from `USD` entries. +ISO 4217 currency of the rates. NVIDIA NeMo Fabric estimates cost only from `USD` entries. ### `unit: PricingUnit` diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index ec8bcc2c7..aa7c4c4c7 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -1965,7 +1965,7 @@ }, "currency": { "default": "USD", - "description": "ISO 4217 currency of the rates. NeMo Fabric estimates cost only from\n`USD` entries.", + "description": "ISO 4217 currency of the rates. NVIDIA NeMo Fabric estimates cost only from\n`USD` entries.", "type": "string" }, "model_id": { From aef62ddcf6f23626f4220ff980bf9ec8e14c96a7 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:16:19 -0700 Subject: [PATCH 087/102] test: clean up recorded invocation fixtures Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 7dc0e379d..81225e0f2 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -7110,19 +7110,20 @@ for line in sys.stdin: } let (record, _) = only_record(&root); assert!(!record.contains("peak_request_input_tokens")); - let record_ref = result - .artifacts - .artifacts - .iter() - .find(|a| a.kind == "run_record") - .expect("record artifact"); - println!("ACCOUNTING_RECORD_RECEIPT={}", record_ref.path.display()); + assert!( + result + .artifacts + .artifacts + .iter() + .any(|a| a.kind == "run_record") + ); + let _ = fs::remove_dir_all(root); } } #[test] fn recorded_native_stream_publishes_the_returned_result() { - let (_, mut plan) = record_plan("success"); + let (root, mut plan) = record_plan("success"); plan.capabilities.streaming = true; plan.adapter_descriptor .as_mut() @@ -7147,5 +7148,6 @@ for line in sys.stdin: .expect("typed saved result"); assert_eq!(decoded, result); assert!(saved.path.with_file_name("run-record.jsonl").is_file()); + let _ = fs::remove_dir_all(root); } } From e7c495b55e295f1c9d34523deda4e430c0ef5e84 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:18:52 -0700 Subject: [PATCH 088/102] test: require exact Python API reference coverage Signed-off-by: mnajafian-nv --- tests/docs/test_python_api_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index dbedf2b94..a71839aa9 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -192,7 +192,7 @@ def test_python_reference_exactly_covers_exported_sdk_classes() -> None: documented = { match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } - assert names <= documented + assert names == documented assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") From fc265da047b1f27dc4e8325f0c6534f805f61e1c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 23:09:37 -0700 Subject: [PATCH 089/102] docs: name NeMo Fabric in attempt comparisons Signed-off-by: mnajafian-nv --- docs/reference/api/python-library-reference/index.md | 2 +- .../nemo_fabric.comparisons.md | 2 +- examples/attempt_comparison/README.md | 12 ++++++------ .../src/nemo_fabric/comparisons.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index fcda2e405..bf3c1c466 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -12,7 +12,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`nemo_fabric.accounting`](./nemo_fabric.accounting.md#module-nemo_fabricaccounting): Typed model accounting and invocation summaries with explicit coverage. - [`nemo_fabric.reporting`](./nemo_fabric.reporting.md#module-nemo_fabricreporting): Inspect saved invocation evidence without a benchmark manifest or trace import. -- [`nemo_fabric.comparisons`](./nemo_fabric.comparisons.md#module-nemo_fabriccomparisons): Compare caller-defined task attempts using saved Fabric invocation evidence. +- [`nemo_fabric.comparisons`](./nemo_fabric.comparisons.md#module-nemo_fabriccomparisons): Compare caller-defined task attempts using saved NeMo Fabric invocation evidence. - [`nemo_fabric.client`](./nemo_fabric.client.md#module-nemo_fabricclient): Native Python client for resolving and running NVIDIA NeMo Fabric agents. - [`nemo_fabric.capabilities`](./nemo_fabric.capabilities.md#module-nemo_fabriccapabilities): Metadata-only adapter inspection for host-side admission. - [`nemo_fabric.runtime`](./nemo_fabric.runtime.md#module-nemo_fabricruntime): Runtime lifecycle support for the NVIDIA NeMo Fabric Python SDK. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.comparisons.md b/docs/reference/api/python-library-reference/nemo_fabric.comparisons.md index a73dfd699..b167e856d 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.comparisons.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.comparisons.md @@ -8,7 +8,7 @@ SPDX-License-Identifier: Apache-2.0 --> # module `nemo_fabric.comparisons` -Compare caller-defined task attempts using saved Fabric invocation evidence. +Compare caller-defined task attempts using saved NeMo Fabric invocation evidence. **Global Variables** --------------- diff --git a/examples/attempt_comparison/README.md b/examples/attempt_comparison/README.md index 7d498e405..ded09cbe5 100644 --- a/examples/attempt_comparison/README.md +++ b/examples/attempt_comparison/README.md @@ -5,10 +5,10 @@ SPDX-License-Identifier: Apache-2.0 # Compare Your Own Task Attempts -Use saved Fabric evidence to compare your tasks across harnesses or model -configurations. Task IDs, revisions and configuration labels are caller-defined. +Use saved NVIDIA NeMo Fabric evidence to compare your tasks across harnesses or +model configurations. Task IDs, revisions and configuration labels are caller-defined. A task can be a code review, document review, investigation or benchmark problem. -An ordinary Fabric run does not require these labels. +An ordinary NeMo Fabric run does not require these labels. ## Try the Example @@ -29,7 +29,7 @@ The report does not choose a matched subset or rank partial spend. ## Capture a Real Task List every scheduled attempt, including retries, before dispatch. Give each -expected Fabric request a distinct `request_id`. Recording saves full results +expected NeMo Fabric request a distinct `request_id`. Recording saves full results and request evidence under the configured artifact directory. Keep missing requests in the inventory when startup, timeout or recording prevents a result. @@ -82,7 +82,7 @@ selects another root. Paths cannot escape that root, including through symlinks. ## Interpret the Report -- Execution status describes Fabric execution. It does not establish that your +- Execution status describes NeMo Fabric execution. It does not establish that your task was solved. Add an optional caller verdict as `"evaluation": {"status": "passed", "evaluator": "my-tests", "version": "v1"}` or use `failed`. Otherwise the task remains `not_evaluated`. Mixed evaluator @@ -99,7 +99,7 @@ selects another root. Paths cannot escape that root, including through symlinks. available configuration and artifact evidence when checking conditions. Logs, telemetry and Relay references remain links to supporting evidence. Their -payloads are not read or used to fill missing Fabric accounting. Harbor, +payloads are not read or used to fill missing NeMo Fabric accounting. Harbor, OpenShell, a benchmark runner and a telemetry backend are not required. ## Use a Bounded Invocation Record When the Full Result Is Missing diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/comparisons.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/comparisons.py index 5e1033e2a..e2a2f0fb3 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/comparisons.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/comparisons.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Compare caller-defined task attempts using saved Fabric invocation evidence.""" +"""Compare caller-defined task attempts using saved NeMo Fabric invocation evidence.""" from __future__ import annotations From a71c3fc8c36d9aa79cf855b3a2d1369034c75bc5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 23:23:09 -0700 Subject: [PATCH 090/102] test: retain exchange timing in comparison fallback Signed-off-by: mnajafian-nv --- tests/python/test_comparisons.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/python/test_comparisons.py b/tests/python/test_comparisons.py index 0739bd429..892cee5a9 100644 --- a/tests/python/test_comparisons.py +++ b/tests/python/test_comparisons.py @@ -430,6 +430,7 @@ def record_fixture(tmp_path, result): "kind": "end", "status": "succeeded", "elapsed_ms": 1000, + "exchange_elapsed_ms": 10, "outcome_scope": "invocation", "model_usage": result["model_usage"], } @@ -462,6 +463,8 @@ def test_record_accounting_survives_missing_full_result_without_creating_outer_s assert view["state"] != "available" assert view["accounting_source"] == "record" assert view["accounting_boundary"] == "core_invocation" + recorded = view["summary"] if condition == "direct" else view["record_summary"] + assert recorded["exchange_elapsed_ms"] == 10 assert row["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 execution = summary["configurations"]["baseline"]["execution"] assert execution["available_result_count"] == 0 From 4f7e551ca0845920074f81b88676b79f1bcf9685 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 00:27:40 -0700 Subject: [PATCH 091/102] fix: preserve lifecycle stage for recorded one-shot failures Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 13 +++++++++++-- crates/fabric-python/src/lib.rs | 29 +++++++++++++++++------------ tests/python/test_recording.py | 16 +++++++++++++++- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 81225e0f2..db9d90fed 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -898,12 +898,21 @@ enum LocalServiceState { /// Invoke a NeMo Fabric run plan. pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { - let runtime = start_runtime(plan)?; + run_plan_with_stage(plan, request).map_err(|(error, _)| error) +} + +/// Preserve the phase of a one-shot failure for language bindings. +#[doc(hidden)] +pub fn run_plan_with_stage( + plan: &RunPlan, + request: RunRequest, +) -> std::result::Result { + let runtime = start_runtime(plan).map_err(|error| (error, ErrorStage::Start))?; let mut result = match invoke_runtime_unpersisted(plan, &runtime, request) { Ok(result) => result, Err(error) => { let _ = stop_runtime(plan, &runtime); - return Err(error); + return Err((error, ErrorStage::Invoke)); } }; match stop_runtime(plan, &runtime) { diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index ede9a2eb8..a6e013d7f 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -10,9 +10,9 @@ use std::thread; use std::time::{Duration, Instant}; use nemo_fabric_core::{ - EnvironmentHandle, EnvironmentReference, FabricConfig, FabricError, OpenAiStreamTransport, - ResolveContext, RunPlan, RunRequest, RuntimeHandle, ServiceHandle, ServiceReference, - doctor_plan, inspect_adapter_config, + EnvironmentHandle, EnvironmentReference, ErrorStage, FabricConfig, FabricError, + OpenAiStreamTransport, ResolveContext, RunPlan, RunRequest, RuntimeHandle, ServiceHandle, + ServiceReference, doctor_plan, inspect_adapter_config, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config_with_adapter_directories, run_plan, }; @@ -157,18 +157,23 @@ fn run_config( fn run_resolved_plan(py: Python<'_>, plan_json: String, request_json: String) -> PyResult { let plan = parse_run_plan(plan_json)?; let request = parse_run_request(request_json)?; - let result = match py.detach(|| run_plan(&plan, request)) { + let result = match py.detach(|| nemo_fabric_core::runtime::run_plan_with_stage(&plan, request)) + { Ok(result) => result, - Err(error) => { - let stage = match &error { - FabricError::AdapterLifecycleOperation { operation, .. } => Some(*operation), - _ => None, + Err((error, stage)) => { + let stage = match stage { + ErrorStage::Config => "config", + ErrorStage::Plan => "plan", + ErrorStage::Prepare => "prepare", + ErrorStage::Start => "start", + ErrorStage::Invoke => "invoke", + ErrorStage::Stop => "stop", + ErrorStage::Release => "release", + ErrorStage::Artifact => "artifact", }; let failure = to_py_error(error); - if let Some(stage) = stage { - // Preserve lifecycle stage without parsing diagnostic text. - failure.value(py).setattr("fabric_stage", stage)?; - } + // Preserve lifecycle stage without parsing diagnostic text. + failure.value(py).setattr("fabric_stage", stage)?; return Err(failure); } }; diff --git a/tests/python/test_recording.py b/tests/python/test_recording.py index dab7170bc..471acb785 100644 --- a/tests/python/test_recording.py +++ b/tests/python/test_recording.py @@ -11,7 +11,7 @@ import pytest from pydantic import ValidationError -from nemo_fabric import RecordConfig +from nemo_fabric import Fabric, RecordConfig from nemo_fabric.errors import FabricRuntimeError from nemo_fabric.runtime import _run_native_lifecycle @@ -91,6 +91,20 @@ async def test_recorded_failure_preserves_native_lifecycle_stage(recorded_plan): assert caught.value.__cause__ is original +async def test_recorded_process_launch_failure_retains_start_stage( + blocking_adapter_config, tmp_path +): + descriptor_path = tmp_path / "blocking.fabric-adapter.json" + descriptor = json.loads(descriptor_path.read_text()) + descriptor["runner"]["command"] = str(tmp_path / "missing-runner") + descriptor_path.write_text(json.dumps(descriptor)) + blocking_adapter_config.runtime.record = RecordConfig(enabled=True) + + with pytest.raises(FabricRuntimeError) as caught: + await Fabric().run(blocking_adapter_config, base_dir=tmp_path, input="hello") + assert caught.value.stage == "start" + + async def test_recorded_cancellation_keeps_cancellation_and_evidence_location( recorded_plan, tmp_path ): From e3a2355b585e1eb231a7395cb1f4e7945c76d12b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 01:41:33 -0700 Subject: [PATCH 092/102] fix: classify empty usage coverage as unverified Signed-off-by: mnajafian-nv --- .../python/src/nemo_fabric_adapter_contract/usage.py | 2 +- adapter-contract/typescript/src/usage.ts | 2 +- adapter-contract/typescript/test/usage-reducer.test.js | 4 ++++ tests/fixtures/usage_accounting/reducer-cases.json | 10 ++++++++++ tests/python/test_usage_reducer_parity.py | 3 +++ 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py index d28998d36..f296d5d21 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py @@ -79,7 +79,7 @@ def _reduce( if total is not None and not causes: complete[key] = total elif not causes: - causes.add(GapReason.MISSING_USAGE) + causes.add(GapReason.SCOPE_UNVERIFIED if not records else GapReason.MISSING_USAGE) for cause in sorted(causes): gaps[cause].append(dimension) for flag, counter in ( diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index 16c8cc5ae..bf75221c4 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -111,7 +111,7 @@ function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage if (values.length !== records.length) causes.add("missing_usage"); if (values.length && value === undefined) causes.add("scope_unverified"); if (value !== undefined && !causes.size) reported[key] = value; - else if (!causes.size) causes.add("missing_usage"); + else if (!causes.size) causes.add(records.length ? "missing_usage" : "scope_unverified"); for (const reason of causes) gap(reason, key); } for (const [flag, counter] of [["input_tokens_include_cache", "input_tokens"], ["output_tokens_include_reasoning", "output_tokens"]] as const) { diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js index de7fbdfb2..4c6e7040c 100644 --- a/adapter-contract/typescript/test/usage-reducer.test.js +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -12,6 +12,10 @@ for (const vector of cases) { const usage = usageFromObservations(vector.records); const accounting = usage.extensions[USAGE_ACCOUNTING_KEY]; for (const [field, value] of Object.entries(vector.expected)) { + if (field === "gap_reason") { + assert.deepEqual(accounting.gaps.map(gap => gap.reason), [value]); + continue; + } const actual = field === "exhaustive" ? accounting.models_exhaustive : field.startsWith("observed_") ? accounting.observed[field.slice(9) + "_tokens"] : field === "peak_request_input_tokens" ? usage.extensions["nemo_fabric.detailed_token_usage.v1"][field] diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json index cba63db2e..39b4c4959 100644 --- a/tests/fixtures/usage_accounting/reducer-cases.json +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -1,4 +1,14 @@ [ + { + "name": "empty_observations", + "records": [], + "expected": { + "input_tokens": null, + "output_tokens": null, + "exhaustive": false, + "gap_reason": "scope_unverified" + } + }, { "name": "complete_calls", "records": [ diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py index c305b4b33..ccdf1628c 100644 --- a/tests/python/test_usage_reducer_parity.py +++ b/tests/python/test_usage_reducer_parity.py @@ -51,6 +51,9 @@ def test_python_reducer_matches_shared_independent_expectations(reduced): for vector, usage in zip(VECTORS, reduced, strict=True): accounting = usage["extensions"][KEY] for field, value in vector["expected"].items(): + if field == "gap_reason": + assert [gap["reason"] for gap in accounting["gaps"]] == [value] + continue if field == "exhaustive": actual = accounting["models_exhaustive"] elif field.startswith("observed_"): From a8f6fd807b9a2dc5cd3ae3f1b01538aec4820a7b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 02:05:52 -0700 Subject: [PATCH 093/102] test: account for unverified empty usage coverage Signed-off-by: mnajafian-nv --- tests/adapters/test_deepagents_accounting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py index dccaf29a3..4b674815a 100644 --- a/tests/adapters/test_deepagents_accounting.py +++ b/tests/adapters/test_deepagents_accounting.py @@ -210,7 +210,8 @@ async def test_remote_tool_callback_dispatch_marks_new_work_only_unit( dispatched = operation in {"start_async_task", "update_async_task"} assert result.input_tokens == (100 if foreground_call and not dispatched else None) reasons = {gap["reason"] for gap in result.extensions[KEY]["gaps"]} - assert ("scope_unverified" in reasons) is dispatched + # With no foreground call, an empty callback list cannot prove zero usage. + assert ("scope_unverified" in reasons) is (dispatched or not foreground_call) if foreground_call: assert result.extensions[KEY]["observed"]["input_tokens"] == 100 elif dispatched: From 46e7ccfa8cdc774e41142b00dd289e53ae6103e3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 02:53:46 -0700 Subject: [PATCH 094/102] docs: clarify providerless pricing lookup order Signed-off-by: mnajafian-nv --- crates/fabric-core/src/price_catalog.rs | 6 ++++-- .../nemo-fabric-core/price-catalog/index.mdx | 2 +- .../price-catalog/struct-pricingcatalog.mdx | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs index 108bb3563..a7473c119 100644 --- a/crates/fabric-core/src/price_catalog.rs +++ b/crates/fabric-core/src/price_catalog.rs @@ -7,7 +7,9 @@ //! one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and //! does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. //! An explicit provider restricts aliases and model-path suffixes to that provider. -//! Without a provider, model-only lookup uses the first matching entry in listed order. +//! Without a provider, lookup tries provider-qualified keys from the model path +//! before model-only suffixes. For each candidate key, the first matching +//! catalog entry wins. //! //! NeMo Fabric ships one catalog, embedded at build time, with the list prices //! of models that its presets, docs, and examples use. Each entry cites the @@ -59,7 +61,7 @@ struct BundledCatalog { pub struct PricingCatalog { /// Catalog format version. Must be `1`. pub version: u32, - /// Price entries. Lookup uses the first entry that matches. + /// Price entries. For each lookup key, the first matching entry wins. pub entries: Vec, } diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx index 07a60a689..44a339036 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx @@ -12,7 +12,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Model price catalogs for estimating token cost. -A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, model-only lookup uses the first matching entry in listed order. +A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, lookup tries provider-qualified keys from the model path before model-only suffixes. For each candidate key, the first matching catalog entry wins. NeMo Fabric ships one catalog, embedded at build time, with the list prices of models that its presets, docs, and examples use. Each entry cites the provider pricing page it was taken from and the date it was checked. A caller can add a catalog of their own, whose entries take precedence. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx index 01d5d4765..027636e72 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx @@ -24,7 +24,7 @@ Catalog format version. Must be `1`. ### `entries: Vec` -Price entries. Lookup uses the first entry that matches. +Price entries. For each lookup key, the first matching entry wins. ## Implementations From 05132d83996be4487d8e1aeabdaa0a36b8611dca Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 03:17:41 -0700 Subject: [PATCH 095/102] docs: regenerate pricing run-plan schema Signed-off-by: mnajafian-nv --- schemas/sdk/run-plan.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index aa7c4c4c7..786559ed2 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -2050,7 +2050,7 @@ "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", "properties": { "entries": { - "description": "Price entries. Lookup uses the first entry that matches.", + "description": "Price entries. For each lookup key, the first matching entry wins.", "items": { "$ref": "#/$defs/ModelPricing" }, From 85100cfec65cf021090ee896a7ce0a54ed13a88c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 07:49:53 -0700 Subject: [PATCH 096/102] docs: clarify pricing references and usage summary guidance Signed-off-by: mnajafian-nv --- crates/fabric-core/src/price_catalog.rs | 2 +- .../nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx | 2 +- schemas/sdk/run-plan.schema.json | 2 +- skills/nemo-fabric-integrate/references/sdk-api-inventory.md | 4 +++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs index a7473c119..97a646139 100644 --- a/crates/fabric-core/src/price_catalog.rs +++ b/crates/fabric-core/src/price_catalog.rs @@ -54,7 +54,7 @@ struct BundledCatalog { /// A model price catalog in the NeMo Relay model pricing catalog format, /// version 1. /// -/// NeMo Fabric reads and writes the format unchanged, with one extra check: +/// NVIDIA NeMo Fabric reads and writes the format unchanged, with one extra check: /// every rate must be below one million US dollars per million tokens. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx index 027636e72..4b180e69b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx @@ -14,7 +14,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. A model price catalog in the NeMo Relay model pricing catalog format, version 1. -NeMo Fabric reads and writes the format unchanged, with one extra check: every rate must be below one million US dollars per million tokens. +NVIDIA NeMo Fabric reads and writes the format unchanged, with one extra check: every rate must be below one million US dollars per million tokens. ## Fields diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index 786559ed2..7016616fd 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -2047,7 +2047,7 @@ }, "PricingCatalog": { "additionalProperties": false, - "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", + "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNVIDIA NeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", "properties": { "entries": { "description": "Price entries. For each lookup key, the first matching entry wins.", diff --git a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md index 6bfe68637..6b83beb23 100644 --- a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md +++ b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md @@ -90,10 +90,12 @@ another turn. For native OpenAI streaming, `result()` also finalizes and discards unread chunks. Relay `InvokeStream.result()` does not consume unread ATOF records. -## Execution Model +## Usage Summaries `summarize_usage(results, *, expected=...)` reduces normalized `RunResult.model_usage` with an explicit inventory of expected `(runtime_id, invocation_id)` pairs. It returns complete totals, observed sums and coverage without certifying legacy counters or silently dropping missing results. See [results and errors](results-and-errors.md) for field semantics and a single-invocation example. +## Execution Model + NVIDIA NeMo Fabric separates configuration, planning, runtime lifecycle, and individual invocations: From 4663e407c49e7507c9e5faba4a6f3e56b9093443 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 08:31:23 -0700 Subject: [PATCH 097/102] fix: preserve one-shot startup error stages Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 46 +++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index db9d90fed..b38e3c30d 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -907,7 +907,7 @@ pub fn run_plan_with_stage( plan: &RunPlan, request: RunRequest, ) -> std::result::Result { - let runtime = start_runtime(plan).map_err(|error| (error, ErrorStage::Start))?; + let runtime = start_runtime_staged(plan)?; let mut result = match invoke_runtime_unpersisted(plan, &runtime, request) { Ok(result) => result, Err(error) => { @@ -1191,15 +1191,24 @@ fn validate_start_plan(plan: &RunPlan) -> Result<()> { /// For a non-local provider, prepare or attach to the environment explicitly /// and call [`start_runtime_in`] with the returned handle. pub fn start_runtime(plan: &RunPlan) -> Result { - validate_start_plan(plan)?; + start_runtime_staged(plan).map_err(|(error, _)| error) +} + +fn start_runtime_staged( + plan: &RunPlan, +) -> std::result::Result { + validate_start_plan(plan).map_err(|error| (error, ErrorStage::Plan))?; let provider = planned_environment_provider(plan); if provider != "local" { - return Err(FabricError::EnvironmentHandleRequired { - provider: provider.to_string(), - }); + return Err(( + FabricError::EnvironmentHandleRequired { + provider: provider.to_string(), + }, + ErrorStage::Prepare, + )); } - let environment = prepare_environment(plan)?; - start_runtime_in_validated(plan, &environment) + let environment = prepare_environment(plan).map_err(|error| (error, ErrorStage::Prepare))?; + start_runtime_in_validated(plan, &environment).map_err(|error| (error, ErrorStage::Start)) } /// Invoke a started harness runtime. @@ -5758,7 +5767,9 @@ for line in sys.stdin: fn local_host_start_failure_preserves_stage_and_diagnostics() { let (root, plan) = local_host_plan("start_failure"); - let error = start_runtime(&plan).expect_err("start must fail"); + let (error, stage) = + run_plan_with_stage(&plan, RunRequest::text("input")).expect_err("start must fail"); + assert_eq!(stage, ErrorStage::Start); let message = error.to_string(); assert!(message.contains("lifecycle start"), "{message}"); assert!(message.contains("fake_start"), "{message}"); @@ -5767,6 +5778,25 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn one_shot_reports_prepare_stage_when_environment_is_required() { + let (root, mut plan) = local_host_plan("success"); + plan.environment_plan + .as_mut() + .expect("resolved environment plan") + .provider = "docker".to_string(); + + let (error, stage) = run_plan_with_stage(&plan, RunRequest::text("input")) + .expect_err("non-local provider needs an environment handle"); + assert!(matches!( + error, + FabricError::EnvironmentHandleRequired { .. } + )); + assert_eq!(stage, ErrorStage::Prepare); + + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_supplies_fresh_adapter_deadlines() { let (root, mut plan) = local_host_plan("success"); From 83d8adace3ca3dd886c8769ba1cdc1a01d3a7c3b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 08:55:34 -0700 Subject: [PATCH 098/102] docs: keep adapter usage guidance within current contract Signed-off-by: mnajafian-nv --- skills/nemo-fabric-build-adapter/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 064429506..7b014b3a7 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -231,7 +231,7 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. -Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts and price them. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. +Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then From ebcfe97c48bcc90e93f2ae74078f1b82e9049132 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 10:26:13 -0700 Subject: [PATCH 099/102] test: run OpenShell launcher checks with system Bash Use the system Bash path so macOS tests exercise Bash 3.2 even when PATH contains a newer Bash. This keeps the empty-array regression meaningful in CI. Signed-off-by: mnajafian-nv --- tests/scripts/test_openshell_launcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/test_openshell_launcher.py b/tests/scripts/test_openshell_launcher.py index 15ab68caa..f54157496 100644 --- a/tests/scripts/test_openshell_launcher.py +++ b/tests/scripts/test_openshell_launcher.py @@ -165,7 +165,7 @@ def _operations(root: Path) -> list[dict]: def _run(launcher, workload, env): script, _, base_env = launcher return subprocess.run( - ["bash", str(script), str(workload), "--input", "words with spaces"], + ["/bin/bash", str(script), str(workload), "--input", "words with spaces"], env={**base_env, **env}, text=True, capture_output=True, @@ -385,7 +385,7 @@ def test_invalid_platform_or_mode_stops_before_building(launcher, environment, m "stage_workload_image() { :; }\nrun_workload_consumer() { :; }\n" ) completed = subprocess.run( - ["bash", str(script), str(workload)], + ["/bin/bash", str(script), str(workload)], env={**base_env, **environment}, text=True, capture_output=True, From 8db3c576f7d56d8a86d69c6289d8c0169d6120c2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 10:31:36 -0700 Subject: [PATCH 100/102] fix: preserve recorded startup stages in Python errors Signed-off-by: mnajafian-nv --- .../nemo-fabric-runtime/src/nemo_fabric/runtime.py | 2 +- tests/python/test_recording.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index fa20cf8a2..b8f15423c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -1018,7 +1018,7 @@ async def _run_native_lifecycle( raise except Exception as error: stage = getattr(error, "fabric_stage", "run") - if stage not in {"start", "invoke", "stop"}: + if stage not in {"plan", "prepare", "start", "invoke", "stop"}: stage = "run" failure = _runtime_error(error, stage=stage) root = _record_root(plan, runtime_config) diff --git a/tests/python/test_recording.py b/tests/python/test_recording.py index 471acb785..92af558b6 100644 --- a/tests/python/test_recording.py +++ b/tests/python/test_recording.py @@ -80,14 +80,15 @@ def test_recording_rejects_unrecognized_settings(): RecordConfig(enabled=True, retries=2) -async def test_recorded_failure_preserves_native_lifecycle_stage(recorded_plan): - original = RuntimeError("native start error") - original.fabric_stage = "start" +@pytest.mark.parametrize("stage", ["plan", "prepare", "start", "invoke", "stop"]) +async def test_recorded_failure_preserves_native_lifecycle_stage(recorded_plan, stage): + original = RuntimeError("native lifecycle error") + original.fabric_stage = stage native = MagicMock(spec=["run_plan"]) native.run_plan.side_effect = original with pytest.raises(FabricRuntimeError) as caught: await _run_native_lifecycle(native, recorded_plan, {"request_id": "req"}) - assert caught.value.stage == "start" + assert caught.value.stage == stage assert caught.value.__cause__ is original From 83b5e76b8b5f3629e7ef9215bd5be505e3a13298 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 15:07:21 -0700 Subject: [PATCH 101/102] fix: reject Relay readiness after the startup deadline Signed-off-by: mnajafian-nv --- .../common/relay_gateway.py | 2 ++ .../test_adapters_common_relay_gateway.py | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 1a57c5fe4..f0e6d5cd3 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -137,6 +137,8 @@ def wait_for_relay_gateway( raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) + if time.monotonic() >= deadline: + break return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index fbd9ca893..86c23ede2 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -252,6 +252,26 @@ def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") +def test_wait_for_relay_gateway_rejects_success_after_deadline( + monkeypatch, gateway_clock +): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += 0.26 + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From 56e59c7c7e75e904eb2f65821e9d86489b6db7b3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 17:00:26 -0700 Subject: [PATCH 102/102] test: allow zero coverage in resource sampling checks Sparse valid readings can produce zero covered time. Accept the full coverage range while preserving spike, artifact and result checks. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 3 ++- tests/python/test_system_stats.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 042777da9..cbd1d35b9 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -5286,7 +5286,8 @@ for line in sys.stdin: "peak was lost after worker exit: {summary:?}" ); assert!(summary.sample_count >= 3); - assert!(gauge.covered_fraction > 0.0); + // A spike sample need not have a neighbor within twice the interval. + assert!((0.0..=1.0).contains(&gauge.covered_fraction)); let path = &summary.samples.as_ref().unwrap().path; let rows: Vec = fs::read_to_string(path) .unwrap() diff --git a/tests/python/test_system_stats.py b/tests/python/test_system_stats.py index 3b89672e5..950df16a3 100644 --- a/tests/python/test_system_stats.py +++ b/tests/python/test_system_stats.py @@ -85,7 +85,8 @@ async def test_sdk_returns_typed_samples_and_artifacts_without_relay_or_recordin sampled.resident_memory_bytes.sampled_peak >= sampled.resident_memory_bytes.final_value ) - assert 0 < sampled.resident_memory_bytes.covered_fraction <= 1 + # Gaps above twice the sampling interval do not contribute coverage. + assert 0 <= sampled.resident_memory_bytes.covered_fraction <= 1 assert ( RunResult.from_mapping(result.to_mapping()).system_stats == result.system_stats )