From a5d4ab9db7c4c0431476ca90842de27432c4765c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 15:52:49 -0700 Subject: [PATCH 001/159] feat: add runtime resource snapshots to the Rust core Applications can read a runtime's identity and status but not how much memory or CPU its adapter host tree uses. Issue #267 asks for a timestamped snapshot on the runtime API and a final snapshot on RunResult that reuses the same type. Add RuntimeResourceSnapshot and runtime_resource_snapshot(plan, runtime) with typed ResourceObservationUnavailable and ResourceObservationUnsupported errors. Linux reads /proc and macOS reads proc_pid_rusage, so no dependency is added; other platforms return Unsupported. The host PID lives in a map beside LOCAL_HOSTS, so a snapshot never waits for the host mutex a running invocation holds. Each local invocation's RunResult carries one final snapshot, and a failed read leaves it unset instead of failing the run. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/error.rs | 18 + crates/fabric-core/src/lib.rs | 6 +- crates/fabric-core/src/resource_snapshot.rs | 575 ++++++++++++++++++ crates/fabric-core/src/runtime.rs | 242 ++++++++ crates/fabric-core/src/schema.rs | 11 +- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 34 +- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/index.mdx | 3 + .../resource-snapshot/index.mdx | 21 + .../struct-runtimeresourcesnapshot.mdx | 117 ++++ .../runtime/fn-runtime-resource-snapshot.mdx | 16 + .../runtime/fn-start-runtime-with-service.mdx | 2 +- .../runtime/fn-start-runtime.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 3 +- .../runtime/struct-runresult.mdx | 6 +- .../schema/enum-schemaname.mdx | 9 +- .../nemo-fabric-core/schema/index.mdx | 2 +- schemas/SCHEMA.md | 2 + schemas/sdk/run-result.schema.json | 57 ++ .../sdk/runtime-resource-snapshot.schema.json | 48 ++ 27 files changed, 1172 insertions(+), 18 deletions(-) create mode 100644 crates/fabric-core/src/resource_snapshot.rs create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx create mode 100644 schemas/sdk/runtime-resource-snapshot.schema.json diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index a21340fb7..ce4682c28 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -277,6 +277,24 @@ pub enum FabricError { /// Adapter kind. adapter_kind: AdapterKind, }, + /// A runtime's resources cannot be read now: the runtime stopped, is + /// stopping, its adapter host process exited, or the process table cannot + /// be read. + #[error("resource observation is unavailable for runtime `{runtime_id}`: {reason}")] + ResourceObservationUnavailable { + /// Runtime handle id. + runtime_id: String, + /// Why the runtime cannot be observed. + reason: String, + }, + /// The runtime's environment provider or operating system has no resource reader. + #[error("resource observation is unsupported for runtime `{runtime_id}`: {reason}")] + ResourceObservationUnsupported { + /// Runtime handle id. + runtime_id: String, + /// What has no resource reader. + reason: String, + }, /// Process adapter settings were invalid. #[error("invalid process adapter settings for {path}: {source}")] InvalidProcessSettings { diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index 2dc251c9f..b20d70a09 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -9,6 +9,7 @@ pub mod agent_execution; pub mod config; pub mod doctor; pub mod error; +pub mod resource_snapshot; pub mod runtime; pub mod schema; @@ -38,6 +39,7 @@ pub use config::{ }; pub use doctor::{DoctorCheck, DoctorReport, DoctorStatus, doctor_plan}; pub use error::{FabricError, Result}; +pub use resource_snapshot::RuntimeResourceSnapshot; pub use runtime::{ AdapterInvocation, ArtifactManifest, ArtifactRef, EnvironmentHandle, ErrorInfo, ErrorStage, FabricEvent, InvocationHandle, OpenAiChatCompletionChunk, OpenAiChatCompletionChunkChoice, @@ -46,8 +48,8 @@ pub use runtime::{ OpenAiStreamSink, OpenAiStreamTransport, RunRequest, RunResult, RunStatus, RunUsage, RuntimeContext, RuntimeHandle, RuntimeTelemetryContext, ServiceHandle, ServiceOwnership, ServiceReference, TelemetryRef, attach_service, invoke_openai_stream, invoke_runtime, - prepare_environment, prepare_service, release_service, run_plan, start_runtime, - start_runtime_with_service, stop_runtime, + prepare_environment, prepare_service, release_service, run_plan, runtime_resource_snapshot, + start_runtime, start_runtime_with_service, stop_runtime, }; pub use schema::{ SchemaName, generate_all_schemas, generate_schema, generate_schema_json, write_schema_snapshots, diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs new file mode 100644 index 000000000..8401f096e --- /dev/null +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -0,0 +1,575 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Resource snapshots of a runtime's local adapter host and its descendants. +//! +//! A snapshot reads the process tree rooted at the adapter host process that +//! NeMo Fabric started for a runtime. Each read visits a parent before its +//! children. CPU time adds every process's own CPU time and the CPU time of the +//! children it has already waited for, so work done by finished descendants +//! still counts. A child reaped between the read of its parent and its own +//! read is missed by that one snapshot and is never counted twice. CPU time +//! can drop between snapshots when a descendant is reparented out of the tree, +//! and on macOS while an exited descendant is not yet waited for, because +//! macOS cannot list an unreaped process. +//! +//! Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. Neither +//! reads command lines, environment variables, or open files. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::error::{FabricError, Result}; + +const MILLIS_PER_DAY: i64 = 86_400_000; +const NANOS_PER_SECOND: f64 = 1_000_000_000.0; + +/// Timestamped resource usage of one runtime's local adapter host process tree. +/// +/// Memory, process count, and CPU time cover the adapter host process and its +/// live descendants. For a runtime started on a service, the shared service +/// process is not included. Values Fabric cannot read are returned as an error, +/// never as zero. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct RuntimeResourceSnapshot { + /// Runtime this snapshot describes. + pub runtime_id: String, + /// When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. + #[schemars(extend("format" = "date-time"))] + pub started_at: String, + /// When Fabric read these values, as an RFC 3339 UTC timestamp. + #[schemars(extend("format" = "date-time"))] + pub observed_at: String, + /// Sum of the resident set sizes of the live processes, in bytes. Pages + /// shared between processes count once per process. + pub resident_memory_bytes: u64, + /// Live processes in the tree, including the adapter host. Exited processes + /// that are not yet reaped are not counted. + pub process_count: u64, + /// CPU time consumed by the tree since the adapter host started, in seconds, + /// including descendants that have exited and been waited for. It can drop + /// when a descendant leaves the tree, and on macOS while an exited + /// descendant is not yet waited for. + #[schemars(range(min = 0.0))] + pub cpu_time_seconds: f64, +} + +/// The adapter host process of one runtime, as recorded when the runtime started. +#[derive(Debug, Clone)] +pub(crate) struct LocalHostProcess { + pid: u32, + /// Start time of the process at `pid`, which tells the host apart from a + /// later process that reuses its PID. + identity: Option, + started_at: String, +} + +impl LocalHostProcess { + /// Record the host `pid`, started by Fabric at `started_at`. + pub(crate) fn new(pid: u32, started_at: SystemTime) -> Self { + Self { + pid, + identity: platform::start_identity(pid), + started_at: rfc3339_millis(unix_millis(started_at)), + } + } +} + +/// Read a snapshot of `host`, or explain why none can be read. +/// +/// `host` is `None` when the runtime is not active in this process. +pub(crate) fn snapshot_local_host( + runtime_id: &str, + host: Option<&LocalHostProcess>, +) -> Result { + if !platform::SUPPORTED { + return Err(FabricError::ResourceObservationUnsupported { + runtime_id: runtime_id.to_string(), + reason: format!( + "no process reader for operating system `{}`", + std::env::consts::OS + ), + }); + } + let host = + host.ok_or_else(|| unavailable(runtime_id, "runtime is not active in this process"))?; + let read = host + .identity + .and_then(|identity| platform::read_tree(host.pid, identity)) + .ok_or_else(|| { + unavailable( + runtime_id, + "adapter host process has exited or the process table cannot be read", + ) + })?; + Ok(RuntimeResourceSnapshot { + runtime_id: runtime_id.to_string(), + started_at: host.started_at.clone(), + observed_at: rfc3339_millis(unix_millis(SystemTime::now())), + resident_memory_bytes: read.resident_bytes, + process_count: read.process_count, + cpu_time_seconds: read.cpu_nanos as f64 / NANOS_PER_SECOND, + }) +} + +fn unavailable(runtime_id: &str, reason: &str) -> FabricError { + FabricError::ResourceObservationUnavailable { + runtime_id: runtime_id.to_string(), + reason: reason.to_string(), + } +} + +/// One read of a live process tree. +#[cfg_attr( + not(any(target_os = "linux", target_os = "macos")), + allow( + dead_code, + reason = "only the Linux and macOS readers build a tree read" + ) +)] +#[derive(Debug, Default)] +struct TreeRead { + cpu_nanos: u64, + resident_bytes: u64, + process_count: u64, +} + +fn unix_millis(time: SystemTime) -> i64 { + match time.duration_since(UNIX_EPOCH) { + Ok(after) => i64::try_from(after.as_millis()).unwrap_or(i64::MAX), + Err(before) => -i64::try_from(before.duration().as_millis()).unwrap_or(i64::MAX), + } +} + +/// Format Unix milliseconds as RFC 3339 UTC with three fraction digits. +/// +/// Core has no date crate, so this converts days to a civil date with Howard +/// Hinnant's `civil_from_days` algorithm. +fn rfc3339_millis(unix_millis: i64) -> String { + let days = unix_millis.div_euclid(MILLIS_PER_DAY); + let millis_of_day = unix_millis.rem_euclid(MILLIS_PER_DAY); + let (year, month, day) = civil_from_days(days); + let seconds_of_day = millis_of_day / 1000; + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{:03}Z", + seconds_of_day / 3600, + seconds_of_day / 60 % 60, + seconds_of_day % 60, + millis_of_day % 1000, + ) +} + +fn civil_from_days(days: i64) -> (i64, i64, i64) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + (year, month, day) +} + +#[cfg(target_os = "linux")] +mod platform { + //! `/proc` reads: one pass lists every process's parent, then the tree is + //! read from the root down, each parent before its children. + + use std::collections::{BTreeMap, VecDeque}; + use std::sync::OnceLock; + + use super::TreeRead; + + pub(super) const SUPPORTED: bool = true; + + /// The fields of `/proc//stat` a snapshot needs. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) struct Stat { + pub zombie: bool, + pub ppid: u32, + /// `utime + stime + cutime + cstime`, in clock ticks. + pub cpu_ticks: u64, + /// `starttime`, in clock ticks since boot. + pub start: u64, + pub rss_pages: u64, + } + + /// Parse a `/proc//stat` line. Fields are counted after the last `)`, + /// because the command name can itself hold spaces and parentheses. + pub(super) fn parse_stat(text: &str) -> Option { + let (_, rest) = text.rsplit_once(')')?; + let fields: Vec<&str> = rest.split_whitespace().collect(); + let number = |index: usize| fields.get(index)?.parse::().ok(); + let ticks = [11, 12, 13, 14] + .into_iter() + .map(&number) + .sum::>()?; + Some(Stat { + zombie: fields.first() == Some(&"Z"), + ppid: u32::try_from(number(1)?).ok()?, + cpu_ticks: ticks, + start: number(19)?, + rss_pages: number(21)?, + }) + } + + fn read_stat(pid: u32) -> Option { + parse_stat(&std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?) + } + + pub(super) fn start_identity(pid: u32) -> Option { + read_stat(pid).map(|stat| stat.start) + } + + fn sysconf(name: libc::c_int) -> u64 { + // SAFETY: sysconf has no memory-safety preconditions. + u64::try_from(unsafe { libc::sysconf(name) }).unwrap_or(0) + } + + fn clock_ticks_per_second() -> u64 { + static TICKS: OnceLock = OnceLock::new(); + *TICKS.get_or_init(|| sysconf(libc::_SC_CLK_TCK).max(1)) + } + + fn page_size() -> u64 { + static PAGE: OnceLock = OnceLock::new(); + *PAGE.get_or_init(|| sysconf(libc::_SC_PAGESIZE)) + } + + /// Every process's children, each with its start time, or `None` when + /// `/proc` cannot be listed. + fn children_by_parent() -> Option>> { + let mut children: BTreeMap> = BTreeMap::new(); + for entry in std::fs::read_dir("/proc").ok()?.flatten() { + let Some(pid) = entry + .file_name() + .to_str() + .and_then(|name| name.parse().ok()) + else { + continue; + }; + if let Some(stat) = read_stat(pid) { + children + .entry(stat.ppid) + .or_default() + .push((pid, stat.start)); + } + } + Some(children) + } + + /// Read the tree under `root`, or `None` when the root has exited or its + /// PID now names a process with another start time than `identity`. + pub(super) fn read_tree(root: u32, identity: u64) -> Option { + let children = children_by_parent()?; + let mut read = TreeRead::default(); + let mut cpu_ticks: u64 = 0; + let mut rss_pages: u64 = 0; + let mut pending = VecDeque::from([(root, identity)]); + while let Some((pid, expected_start)) = pending.pop_front() { + let stat = read_stat(pid).filter(|stat| stat.start == expected_start); + if pid == root && stat.is_none_or(|stat| stat.zombie) { + return None; + } + let Some(stat) = stat else { + continue; + }; + cpu_ticks = cpu_ticks.saturating_add(stat.cpu_ticks); + if !stat.zombie { + rss_pages = rss_pages.saturating_add(stat.rss_pages); + read.process_count += 1; + } + // A child never starts before its parent; one that seems to reuses a PID. + let own = children.get(&pid).into_iter().flatten(); + pending.extend(own.filter(|(_, start)| *start >= stat.start).copied()); + } + read.cpu_nanos = ticks_to_nanos(cpu_ticks, clock_ticks_per_second()); + read.resident_bytes = rss_pages.saturating_mul(page_size()); + Some(read) + } + + pub(super) fn ticks_to_nanos(ticks: u64, ticks_per_second: u64) -> u64 { + let nanos = u128::from(ticks) * 1_000_000_000 / u128::from(ticks_per_second); + u64::try_from(nanos).unwrap_or(u64::MAX) + } +} + +#[cfg(target_os = "macos")] +mod platform { + //! `proc_pid_rusage` reads, from the root down, each parent before its children. + + use std::collections::{BTreeMap, VecDeque}; + use std::sync::OnceLock; + + use super::TreeRead; + + pub(super) const SUPPORTED: bool = true; + + fn rusage(pid: u32) -> Option { + let pid = libc::c_int::try_from(pid).ok()?; + // SAFETY: rusage_info_v2 is plain data; zero is a valid initial value. + let mut info: libc::rusage_info_v2 = unsafe { std::mem::zeroed() }; + // SAFETY: `info` is a writable rusage_info_v2, the layout RUSAGE_INFO_V2 names. + let status = unsafe { + libc::proc_pid_rusage( + pid, + libc::RUSAGE_INFO_V2, + (&raw mut info).cast::(), + ) + }; + (status == 0).then_some(info) + } + + /// `mach_timebase_info_data_t`. + #[repr(C)] + struct MachTimebaseInfo { + numer: u32, + denom: u32, + } + + // libc marks its binding deprecated in favour of the `mach2` crate; one + // system call does not justify a new dependency. + unsafe extern "C" { + fn mach_timebase_info(info: *mut MachTimebaseInfo) -> libc::c_int; + } + + /// Nanoseconds per Mach absolute time unit, as `(numer, denom)`, or `None` + /// when the kernel does not report it. + fn timebase() -> Option<(u64, u64)> { + static TIMEBASE: OnceLock> = OnceLock::new(); + *TIMEBASE.get_or_init(|| { + let mut info = MachTimebaseInfo { numer: 0, denom: 0 }; + // SAFETY: `info` is a writable mach_timebase_info_data_t. + let status = unsafe { mach_timebase_info(&mut info) }; + (status == 0 && info.denom != 0).then(|| (u64::from(info.numer), u64::from(info.denom))) + }) + } + + /// Convert Mach absolute time units, which `rusage_info` CPU times use on + /// Apple silicon, to nanoseconds. + pub(super) fn mach_to_nanos(units: u64, (numer, denom): (u64, u64)) -> u64 { + let nanos = u128::from(units) * u128::from(numer) / u128::from(denom); + u64::try_from(nanos).unwrap_or(u64::MAX) + } + + pub(super) fn start_identity(pid: u32) -> Option { + rusage(pid).map(|info| info.ri_proc_start_abstime) + } + + /// Every live process's children. One `proc_listallpids` call and one + /// `proc_pidinfo` per process: `proc_listchildpids` scans the whole + /// process table on each call, which costs too much for a wide tree. + /// `PROC_PIDTBSDINFO` fails for a zombie, so a zombie is never listed. + /// Returns `None` when the process table cannot be listed. + fn children_by_parent() -> Option>> { + let mut pids = vec![0 as libc::pid_t; 1024]; + let count = loop { + let bytes = libc::c_int::try_from(std::mem::size_of_val(pids.as_slice())).ok()?; + // SAFETY: the buffer is writable and sized in bytes as libproc requires. + let count = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) }; + let count = usize::try_from(count).ok().filter(|count| *count > 0)?; + if count < pids.len() { + break count; + } + pids.resize(pids.len() * 2, 0); + }; + let mut children: BTreeMap> = BTreeMap::new(); + for &pid in &pids[..count] { + if let (Ok(pid), Some(ppid)) = (u32::try_from(pid), parent_pid(pid)) { + children.entry(ppid).or_default().push(pid); + } + } + Some(children) + } + + /// The parent of a live process, or `None` for a zombie or a missing process. + fn parent_pid(pid: libc::pid_t) -> Option { + // SAFETY: proc_bsdinfo is plain data; zero is a valid initial value. + let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; + let size = libc::c_int::try_from(std::mem::size_of::()).ok()?; + // SAFETY: `info` is writable and `size` bytes long, as PROC_PIDTBSDINFO needs. + let written = unsafe { + libc::proc_pidinfo(pid, libc::PROC_PIDTBSDINFO, 0, (&raw mut info).cast(), size) + }; + (written == size).then_some(info.pbi_ppid) + } + + /// Read the tree under `root`, or `None` when the root has exited or its + /// PID now names a process with another start time than `identity`. A + /// child that seems to start before its parent reuses a PID and is left out. + pub(super) fn read_tree(root: u32, identity: u64) -> Option { + parent_pid(libc::pid_t::try_from(root).ok()?)?; + let root_info = rusage(root).filter(|info| info.ri_proc_start_abstime == identity)?; + let timebase = timebase()?; + let children = children_by_parent()?; + let mut read = TreeRead::default(); + let mut cpu_units: u64 = 0; + let mut resident: u64 = 0; + let mut pending = VecDeque::from([(root, Some(root_info))]); + while let Some((pid, known)) = pending.pop_front() { + let Some(info) = known.or_else(|| rusage(pid)) else { + continue; + }; + cpu_units = cpu_units + .saturating_add(info.ri_user_time) + .saturating_add(info.ri_system_time) + .saturating_add(info.ri_child_user_time) + .saturating_add(info.ri_child_system_time); + resident = resident.saturating_add(info.ri_resident_size); + read.process_count += 1; + let start = info.ri_proc_start_abstime; + let own = children.get(&pid).into_iter().flatten(); + pending.extend( + own.filter_map(|child| rusage(*child).map(|info| (*child, info))) + // Skip a child that exited after it was listed, and one + // that seems to start before its parent (a reused PID). + .filter(|(_, info)| { + info.ri_proc_exit_abstime == 0 && info.ri_proc_start_abstime >= start + }) + .map(|(child, info)| (child, Some(info))), + ); + } + read.cpu_nanos = mach_to_nanos(cpu_units, timebase); + read.resident_bytes = resident; + Some(read) + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +mod platform { + use super::TreeRead; + + pub(super) const SUPPORTED: bool = false; + + pub(super) fn start_identity(_pid: u32) -> Option { + None + } + + pub(super) fn read_tree(_root: u32, _identity: u64) -> Option { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rfc3339_millis_formats_fixed_vectors() { + assert_eq!(rfc3339_millis(0), "1970-01-01T00:00:00.000Z"); + assert_eq!( + rfc3339_millis(1_792_000_870_205), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!(rfc3339_millis(951_782_400_000), "2000-02-29T00:00:00.000Z"); + assert_eq!( + rfc3339_millis(4_107_542_399_999), + "2100-02-28T23:59:59.999Z" + ); + assert_eq!(rfc3339_millis(-1), "1969-12-31T23:59:59.999Z"); + } + + #[test] + fn snapshot_local_host_without_active_host_is_unavailable_or_unsupported() { + let error = snapshot_local_host("runtime-1", None).expect_err("no host"); + + if platform::SUPPORTED { + assert!(matches!( + error, + FabricError::ResourceObservationUnavailable { ref runtime_id, .. } + if runtime_id == "runtime-1" + )); + } else { + assert!(matches!( + error, + FabricError::ResourceObservationUnsupported { .. } + )); + } + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn snapshot_local_host_reads_this_process_tree() { + let host = LocalHostProcess::new(std::process::id(), SystemTime::now()); + + let snapshot = snapshot_local_host("runtime-1", Some(&host)).expect("snapshot"); + + assert_eq!(snapshot.runtime_id, "runtime-1"); + assert!(snapshot.process_count >= 1); + assert!(snapshot.resident_memory_bytes > 0); + // Linux counts CPU in 10 ms ticks, so a young test process can read zero. + assert!(snapshot.cpu_time_seconds.is_finite() && snapshot.cpu_time_seconds >= 0.0); + assert!(snapshot.observed_at >= snapshot.started_at); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn snapshot_local_host_with_reused_pid_identity_is_unavailable() { + let mut host = LocalHostProcess::new(std::process::id(), SystemTime::now()); + host.identity = host.identity.map(|identity| identity.wrapping_add(1)); + + let error = snapshot_local_host("runtime-1", Some(&host)).expect_err("other process"); + + assert!(matches!( + error, + FabricError::ResourceObservationUnavailable { .. } + )); + } + + /// A host that exited but is not yet reaped is not a live process. + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn snapshot_local_host_with_unreaped_exited_host_is_unavailable() { + let mut child = std::process::Command::new("sleep") + .arg("30") + .spawn() + .expect("spawn child"); + let host = LocalHostProcess::new(child.id(), SystemTime::now()); + child.kill().expect("kill child"); + std::thread::sleep(std::time::Duration::from_millis(200)); + + let error = snapshot_local_host("runtime-1", Some(&host)).expect_err("zombie host"); + child.wait().expect("reap child"); + + assert!(matches!( + error, + FabricError::ResourceObservationUnavailable { .. } + )); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_stat_parse_reads_cpu_with_waited_children_start_time_and_ticks() { + let stat = "42 (a (weird) name) S 7 42 42 0 -1 4194560 100 0 0 0 30 20 5 4 20 0 1 0 \ + 9001 1000000 250 18446744073709551615"; + + let parsed = platform::parse_stat(stat).expect("stat parses"); + + assert_eq!(parsed.ppid, 7); + assert_eq!(parsed.cpu_ticks, 30 + 20 + 5 + 4); + assert_eq!(parsed.start, 9001); + assert_eq!(parsed.rss_pages, 250); + assert!(!parsed.zombie); + assert_eq!(platform::ticks_to_nanos(150, 100), 1_500_000_000); + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_mach_units_convert_with_the_timebase() { + // Apple silicon's timebase: 125 ns per 3 units. + assert_eq!(platform::mach_to_nanos(24_000_000, (125, 3)), 1_000_000_000); + assert_eq!( + platform::mach_to_nanos(1_000_000_000, (1, 1)), + 1_000_000_000 + ); + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index ab16add07..9630f9f6e 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -31,6 +31,7 @@ use crate::config::{ validate_tool_definitions, validate_workflow, }; use crate::error::{FabricError, Result}; +use crate::resource_snapshot::{LocalHostProcess, RuntimeResourceSnapshot, snapshot_local_host}; static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; @@ -72,6 +73,10 @@ const DEFAULT_PYTHON: &str = "python.exe"; static TEST_STOPPED_AGENTS: Mutex> = Mutex::new(Vec::new()); static LOCAL_HOSTS: LazyLock>>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); +// Adapter host processes by runtime id. A host's own mutex stays locked for a +// whole invocation, so a resource snapshot reads the PID from here instead. +static LOCAL_HOST_PROCESSES: LazyLock>> = + LazyLock::new(|| Mutex::new(BTreeMap::new())); static LOCAL_SERVICES: LazyLock>>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); @@ -136,6 +141,10 @@ pub struct RunResult { /// Normalized invocation usage when reported by the adapter target. #[serde(default, skip_serializing_if = "Option::is_none")] pub usage: Option, + /// Resource snapshot of the runtime, read once after the adapter returned this + /// result. Absent when the runtime's resources could not be observed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resource_snapshot: Option, /// Artifacts produced or collected by this run. #[serde(default)] pub artifacts: ArtifactManifest, @@ -1163,6 +1172,46 @@ pub fn invoke_runtime( }) } +/// Read a timestamped resource snapshot of a started runtime. +/// +/// The snapshot covers the runtime's local adapter host process and its live +/// descendants. It does not wait for a running invocation. It returns +/// [`FabricError::ResourceObservationUnavailable`] when the runtime stopped, is +/// stopping, its host exited, or the process table cannot be read, and +/// [`FabricError::ResourceObservationUnsupported`] when the environment provider +/// or operating system has no resource reader. +pub fn runtime_resource_snapshot( + plan: &RunPlan, + runtime: &RuntimeHandle, +) -> Result { + validate_runtime_handle(plan, runtime)?; + if !uses_local_host(plan) { + return Err(FabricError::ResourceObservationUnsupported { + runtime_id: runtime.runtime_id.clone(), + reason: format!( + "adapter kind `{}` has no resource reader", + adapter_kind_name(runtime.adapter_kind) + ), + }); + } + local_runtime_resource_snapshot(runtime) +} + +fn local_runtime_resource_snapshot(runtime: &RuntimeHandle) -> Result { + if runtime.environment.provider != "local" { + return Err(FabricError::ResourceObservationUnsupported { + runtime_id: runtime.runtime_id.clone(), + reason: format!( + "environment provider `{}` has no resource reader", + runtime.environment.provider + ), + }); + } + // Clone the entry so the map lock is released before the process tree is read. + let host = local_host_processes().get(&runtime.runtime_id).cloned(); + snapshot_local_host(&runtime.runtime_id, host.as_ref()) +} + /// Invoke a started harness runtime and pass through native OpenAI chat-completion chunks. pub fn invoke_openai_stream( plan: &RunPlan, @@ -1523,6 +1572,7 @@ impl RuntimeAdapter for LocalHostAdapter { } fn stop(&self, runtime: &RuntimeHandle) -> Result> { + local_host_processes().remove(&runtime.runtime_id); let Some(host) = local_hosts().remove(&runtime.runtime_id) else { return Ok(vec![local_host_stop_event(runtime, true, false)]); }; @@ -1620,6 +1670,7 @@ fn start_local_runtime( )?; let environment_env = start.runtime_context.environment.env.clone(); let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Start(Box::new(start))); + let started_at = SystemTime::now(); let mut host = spawn_local_host(plan, &runtime, artifacts, relay_config, environment_env)?; if let Err(error) = exchange_lifecycle_message( &mut host, @@ -1631,6 +1682,10 @@ fn start_local_runtime( let _ = remove_local_host_files(&host); return Err(error); } + local_host_processes().insert( + runtime.runtime_id.clone(), + LocalHostProcess::new(host.child.id(), started_at), + ); local_hosts().insert(runtime.runtime_id.clone(), Arc::new(Mutex::new(host))); Ok(runtime) } @@ -1814,8 +1869,11 @@ fn run_local_host_invocation_with_timeout( ) { Ok(output) => { let stderr = take_local_host_stderr(&mut host_guard); + // Read while the host lock is held, so no later invocation is counted. + let resource_snapshot = local_runtime_resource_snapshot(runtime).ok(); Ok(( output, + resource_snapshot, stderr, host_guard.command.clone(), host_guard.child.id(), @@ -1838,6 +1896,7 @@ fn run_local_host_invocation_with_timeout( }; let ( adapter_output, + resource_snapshot, stderr, host_command, host_pid, @@ -1989,6 +2048,7 @@ fn run_local_host_invocation_with_timeout( output, error, usage: agent_result.usage.as_ref().map(run_usage), + resource_snapshot, artifacts, telemetry: telemetry_ref(plan, relay_config.as_ref()), events, @@ -2008,6 +2068,7 @@ fn invalidate_timed_out_local_host( .is_some_and(|host| Arc::ptr_eq(host, expected_host)) { hosts.remove(runtime_id); + local_host_processes().remove(runtime_id); } } @@ -2163,6 +2224,12 @@ fn local_hosts() -> std::sync::MutexGuard<'static, BTreeMap std::sync::MutexGuard<'static, BTreeMap> { + LOCAL_HOST_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) +} + fn local_services() -> std::sync::MutexGuard<'static, BTreeMap>>> { LOCAL_SERVICES @@ -3936,6 +4003,10 @@ for line in sys.stdin: "choices": [{"index": 0, "delta": {"content": "lo"}}], }, ]) + if MODE == "busy": + # A descendant that burns 1.2 s of CPU, then exits and is waited for. + subprocess.run([sys.executable, "-c", + "import time\nwhile time.process_time() < 1.2:\n pass"], check=True) output = { "host_pid": os.getpid(), "invocation_count": invocations, @@ -4670,6 +4741,177 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + fn assert_unavailable(result: &Result, runtime_id: &str) { + match result { + Err(FabricError::ResourceObservationUnavailable { + runtime_id: actual, .. + }) => assert_eq!(actual, runtime_id), + other => panic!("expected ResourceObservationUnavailable, got {other:?}"), + } + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn runtime_resource_snapshot_of_active_runtime_reads_host_tree() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + + let snapshot = runtime_resource_snapshot(&plan, &runtime).expect("snapshot"); + stop_runtime(&plan, &runtime).expect("stop local host"); + + assert_eq!(snapshot.runtime_id, runtime.runtime_id); + assert!(snapshot.process_count >= 1); + assert!(snapshot.resident_memory_bytes > 0); + assert!(snapshot.cpu_time_seconds >= 0.0); + assert!(snapshot.started_at.ends_with('Z')); + assert!(snapshot.started_at <= snapshot.observed_at); + let _ = fs::remove_dir_all(root); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn runtime_resource_snapshot_during_invocation_returns_before_invocation_ends() { + let (root, plan) = local_host_plan("busy"); + let runtime = start_runtime(&plan).expect("start local host"); + + let saw_descendant = thread::scope(|scope| { + let invocation = + scope.spawn(|| invoke_runtime(&plan, &runtime, RunRequest::text("work"))); + let mut saw_descendant = false; + while !saw_descendant && !invocation.is_finished() { + let snapshot = runtime_resource_snapshot(&plan, &runtime).expect("live snapshot"); + saw_descendant = snapshot.process_count >= 2 && !invocation.is_finished(); + thread::sleep(Duration::from_millis(20)); + } + invocation + .join() + .expect("invocation thread") + .expect("invoke local host"); + saw_descendant + }); + stop_runtime(&plan, &runtime).expect("stop local host"); + + assert!(saw_descendant, "no snapshot saw the running descendant"); + let _ = fs::remove_dir_all(root); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn invocation_result_carries_final_resource_snapshot_with_waited_descendant_cpu() { + let (root, plan) = local_host_plan("busy"); + let runtime = start_runtime(&plan).expect("start local host"); + + let result = + invoke_runtime(&plan, &runtime, RunRequest::text("work")).expect("invoke local host"); + stop_runtime(&plan, &runtime).expect("stop local host"); + + let snapshot = result.resource_snapshot.expect("final resource snapshot"); + assert_eq!(snapshot.runtime_id, result.runtime_id); + assert!(snapshot.process_count >= 1); + assert!( + snapshot.cpu_time_seconds >= 1.0, + "descendant CPU missing: {snapshot:?}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn runtime_resource_snapshot_after_stop_is_unavailable() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + stop_runtime(&plan, &runtime).expect("stop local host"); + + let result = runtime_resource_snapshot(&plan, &runtime); + + if cfg!(any(target_os = "linux", target_os = "macos")) { + assert_unavailable(&result, &runtime.runtime_id); + } else { + assert!(matches!( + result, + Err(FabricError::ResourceObservationUnsupported { .. }) + )); + } + let _ = fs::remove_dir_all(root); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn runtime_resource_snapshot_after_host_exit_is_unavailable() { + let (root, plan) = local_host_plan("crash_after_start"); + let runtime = start_runtime(&plan).expect("start local host"); + + let deadline = Instant::now() + Duration::from_secs(10); + let mut result = runtime_resource_snapshot(&plan, &runtime); + while result.is_ok() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(50)); + result = runtime_resource_snapshot(&plan, &runtime); + } + let _ = stop_runtime(&plan, &runtime); + + assert_unavailable(&result, &runtime.runtime_id); + let _ = fs::remove_dir_all(root); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn runtime_resource_snapshot_during_concurrent_stop_is_snapshot_or_unavailable() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + + let results = thread::scope(|scope| { + let snapshots = scope.spawn(|| { + let mut results = Vec::new(); + loop { + let result = runtime_resource_snapshot(&plan, &runtime); + let stopped = result.is_err(); + results.push(result); + if stopped { + return results; + } + } + }); + thread::sleep(Duration::from_millis(50)); + stop_runtime(&plan, &runtime).expect("stop local host"); + snapshots.join().expect("snapshot thread") + }); + + let (last, earlier) = results.split_last().expect("at least one snapshot"); + assert!(earlier.iter().all(Result::is_ok)); + assert_unavailable(last, &runtime.runtime_id); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn runtime_resource_snapshot_of_non_local_environment_is_unsupported() { + let (root, plan) = local_host_plan("success"); + let mut environment = prepare_environment(&plan).expect("prepare environment"); + environment.provider = "remote".to_string(); + let runtime_id = new_id("runtime"); + let runtime = RuntimeHandle { + runtime_binding: runtime_binding(&runtime_id, None, &plan, &environment) + .expect("runtime binding"), + runtime_id, + agent_name: plan.agent_name.clone(), + harness: harness(&plan), + adapter_kind: adapter_kind(&plan), + adapter_id: adapter_id(&plan), + service_id: None, + environment, + }; + + let result = runtime_resource_snapshot(&plan, &runtime); + + assert!( + matches!( + &result, + Err(FabricError::ResourceObservationUnsupported { runtime_id, reason }) + if *runtime_id == runtime.runtime_id && reason.contains("remote") + ), + "{result:?}" + ); + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_relay_config_is_runtime_scoped() { let (root, plan) = local_host_plan_with_relay("success", true); diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index ee15fda12..69589cfa9 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -13,6 +13,7 @@ use crate::config::{ AdapterDescriptor, AdapterTargetDescriptor, AgentConfig, FabricConfig, RunPlan, }; use crate::error::{FabricError, Result}; +use crate::resource_snapshot::RuntimeResourceSnapshot; use crate::runtime::{ AdapterInvocation, ArtifactManifest, EnvironmentHandle, ErrorInfo, FabricEvent, InvocationHandle, OpenAiStreamInvocation, OpenAiStreamRecord, RunRequest, RunResult, @@ -65,11 +66,13 @@ pub enum SchemaName { ErrorInfo, /// NeMo Fabric lifecycle event schema. FabricEvent, + /// Runtime resource snapshot schema. + RuntimeResourceSnapshot, } impl SchemaName { /// All public schemas in stable output order. - pub const ALL: [Self; 21] = [ + pub const ALL: [Self; 22] = [ Self::Agent, Self::AgentConfig, Self::AgentRunRequest, @@ -91,6 +94,7 @@ impl SchemaName { Self::ArtifactManifest, Self::ErrorInfo, Self::FabricEvent, + Self::RuntimeResourceSnapshot, ]; /// Stable file stem for this schema. @@ -117,6 +121,7 @@ impl SchemaName { Self::ArtifactManifest => "artifact-manifest", Self::ErrorInfo => "error-info", Self::FabricEvent => "fabric-event", + Self::RuntimeResourceSnapshot => "runtime-resource-snapshot", } } @@ -170,6 +175,9 @@ impl SchemaName { "artifact-manifest" | "artifact_manifest" => Ok(Self::ArtifactManifest), "error-info" | "error_info" => Ok(Self::ErrorInfo), "fabric-event" | "fabric_event" => Ok(Self::FabricEvent), + "runtime-resource-snapshot" | "runtime_resource_snapshot" => { + Ok(Self::RuntimeResourceSnapshot) + } _ => Err(FabricError::UnknownSchema { schema: value.to_string(), available: Self::ALL @@ -205,6 +213,7 @@ pub fn generate_schema(schema: SchemaName) -> Result { SchemaName::ArtifactManifest => to_value(schema_for!(ArtifactManifest)), SchemaName::ErrorInfo => to_value(schema_for!(ErrorInfo)), SchemaName::FabricEvent => to_value(schema_for!(FabricEvent)), + SchemaName::RuntimeResourceSnapshot => to_value(schema_for!(RuntimeResourceSnapshot)), } } diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 52a3b6f57..6c276ae6b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 125 +position: 127 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 26a286a0c..fa45b1f21 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 126 +position: 128 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 2c89a0e39..218c1dde6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 127 +position: 129 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index fa7255361..7a918b620 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 128 +position: 130 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index ea299ed94..f0f77cf94 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 129 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 95cf2f59a..4c5c5da33 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -497,6 +497,38 @@ Environment provider. Adapter kind. +### `ResourceObservationUnavailable` + +
+ +A runtime's resources cannot be read now: the runtime stopped, is stopping, its adapter host process exited, or the process table cannot be read. + +#### Fields + +##### `runtime_id: String` + +Runtime handle id. + +##### `reason: String` + +Why the runtime cannot be observed. + +### `ResourceObservationUnsupported` + +
+ +The runtime's environment provider or operating system has no resource reader. + +#### Fields + +##### `runtime_id: String` + +Runtime handle id. + +##### `reason: String` + +What has no resource reader. + ### `InvalidProcessSettings`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index cf7208b32..ca57e0275 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 130 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 8af97f312..6b00f4d18 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 133 +position: 136 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 6670ab4d8..045826ad4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -92,6 +92,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use doctor::doctor_plan;` - `pub use error::FabricError;` - `pub use error::Result;` +- `pub use resource_snapshot::RuntimeResourceSnapshot;` - `pub use runtime::AdapterInvocation;` - `pub use runtime::ArtifactManifest;` - `pub use runtime::ArtifactRef;` @@ -129,6 +130,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use runtime::prepare_service;` - `pub use runtime::release_service;` - `pub use runtime::run_plan;` +- `pub use runtime::runtime_resource_snapshot;` - `pub use runtime::start_runtime;` - `pub use runtime::start_runtime_with_service;` - `pub use runtime::stop_runtime;` @@ -146,6 +148,7 @@ Core config and runtime contract for NeMo Fabric. - [config](config/index.mdx): NeMo Fabric config models and loading helpers. - [doctor](doctor/index.mdx): Plan diagnostics for NeMo Fabric. - [error](error/index.mdx): Error types for NeMo Fabric core. +- [resource_snapshot](resource-snapshot/index.mdx): Resource snapshots of a runtime's local adapter host and its descendants. - [runtime](runtime/index.mdx): Runtime invocation helpers. - [schema](schema/index.mdx): JSON Schema generation for the public NeMo Fabric contract. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx new file mode 100644 index 000000000..7ff9eab7c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -0,0 +1,21 @@ +--- +title: "Module resource_snapshot" +sidebar-title: "resource_snapshot" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" +description: "Resource snapshots of a runtime's local adapter host and its descendants." +position: 133 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Resource snapshots of a runtime's local adapter host and its descendants. + +A snapshot reads the process tree rooted at the adapter host process that NeMo Fabric started for a runtime. Each read visits a parent before its children. CPU time adds every process's own CPU time and the CPU time of the children it has already waited for, so work done by finished descendants still counts. A child reaped between the read of its parent and its own read is missed by that one snapshot and is never counted twice. CPU time can drop between snapshots when a descendant is reparented out of the tree, and on macOS while an exited descendant is not yet waited for, because macOS cannot list an unreaped process. + +Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. Neither reads command lines, environment variables, or open files. + +## Structs + +- [RuntimeResourceSnapshot](struct-runtimeresourcesnapshot.mdx): Timestamped resource usage of one runtime's local adapter host process tree. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx new file mode 100644 index 000000000..82f8493e1 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx @@ -0,0 +1,117 @@ +--- +title: "Struct Runtime Resource Snapshot" +sidebar-title: "RuntimeResourceSnapshot" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot/struct-runtimeresourcesnapshot" +description: "Timestamped resource usage of one runtime's local adapter host process tree." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub started_at: String,\n    pub observed_at: String,\n    pub resident_memory_bytes: u64,\n    pub process_count: u64,\n    pub cpu_time_seconds: f64,\n}"}} />
+ +Timestamped resource usage of one runtime's local adapter host process tree. + +Memory, process count, and CPU time cover the adapter host process and its live descendants. For a runtime started on a service, the shared service process is not included. Values Fabric cannot read are returned as an error, never as zero. + +## Fields + +### `runtime_id: String` + +Runtime this snapshot describes. + +### `started_at: String` + +When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. + +### `observed_at: String` + +When Fabric read these values, as an RFC 3339 UTC timestamp. + +### `resident_memory_bytes: u64` + +Sum of the resident set sizes of the live processes, in bytes. Pages shared between processes count once per process. + +### `process_count: u64` + +Live processes in the tree, including the adapter host. Exited processes that are not yet reaped are not counted. + +### `cpu_time_seconds: f64` + +CPU time consumed by the tree since the adapter host started, in seconds, including descendants that have exited and been waited for. It can drop when a descendant leaves the tree, and on macOS while an exited descendant is not yet waited for. + +## Trait Implementations + +### `impl Clone for RuntimeResourceSnapshot` + +
Clone for RuntimeResourceSnapshot"}} />
+ +#### `clone` + +
clone(&self) -> RuntimeResourceSnapshot"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RuntimeResourceSnapshot` + +
Debug for RuntimeResourceSnapshot"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for RuntimeResourceSnapshot` + +
Deserialize<'de> for RuntimeResourceSnapshot"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RuntimeResourceSnapshot` + +
RuntimeResourceSnapshot"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RuntimeResourceSnapshot` + +
PartialEq for RuntimeResourceSnapshot"}} />
+ +#### `eq` + +
eq(&self, other: &RuntimeResourceSnapshot) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RuntimeResourceSnapshot` + +
Serialize for RuntimeResourceSnapshot"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for RuntimeResourceSnapshot` + +
StructuralPartialEq for RuntimeResourceSnapshot"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx new file mode 100644 index 000000000..cbe246229 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx @@ -0,0 +1,16 @@ +--- +title: "Function runtime_resource_snapshot" +sidebar-title: "runtime_resource_snapshot" +description: "Read a timestamped resource snapshot of a started runtime." +position: 38 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
RunPlan,\n    runtime: &RuntimeHandle,\n) -> Result<RuntimeResourceSnapshot>"}} />
+ +Read a timestamped resource snapshot of a started runtime. + +The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, or the process table cannot be read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the environment provider or operating system has no resource reader. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx index 547b47760..c61b166c3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_with_service" sidebar-title: "start_runtime_with_service" description: "Start a runtime connected to an active service." -position: 39 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx index 0f135dffc..04a3c9019 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime" sidebar-title: "start_runtime" description: "Start or connect to a harness runtime." -position: 38 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index 71febb037..a55e5bd91 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 40 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index e4ce802a1..309f520c4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 131 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -56,6 +56,7 @@ Runtime invocation helpers. - [prepare_service](fn-prepare-service.mdx): Prepare a Fabric-owned long-lived service for a resolved run plan. - [release_service](fn-release-service.mdx): Release a Fabric-owned service or detach from a caller-owned service. - [run_plan](fn-run-plan.mdx): Invoke a NeMo Fabric run plan. +- [runtime_resource_snapshot](fn-runtime-resource-snapshot.mdx): Read a timestamped resource snapshot of a started runtime. - [start_runtime](fn-start-runtime.mdx): Start or connect to a harness runtime. - [start_runtime_with_service](fn-start-runtime-with-service.mdx): Start a runtime connected to an active service. - [stop_runtime](fn-stop-runtime.mdx): Stop or detach from a harness runtime. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 1cb5bd052..3e05964a5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub resource_snapshot: Option<RuntimeResourceSnapshot>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
Result from a NeMo Fabric-managed harness invocation. @@ -59,6 +59,10 @@ Error metadata when applicable. Normalized invocation usage when reported by the adapter target. +### `resource_snapshot: Option` + +Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed. + ### `artifacts: ArtifactManifest` Artifacts produced or collected by this run. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index 9ae563e23..9b8e9dd87 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -32,6 +32,7 @@ pub enum SchemaName { ArtifactManifest, ErrorInfo, FabricEvent, + RuntimeResourceSnapshot, } ``` @@ -165,6 +166,12 @@ Structured error metadata schema. NeMo Fabric lifecycle event schema. +### `RuntimeResourceSnapshot` + +
+ +Runtime resource snapshot schema. + ## Implementations ### `impl SchemaName` @@ -173,7 +180,7 @@ NeMo Fabric lifecycle event schema. #### `ALL` -
21]"}} />
+
22]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 0569b34b2..a60dea2c9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 132 +position: 135 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 43867290c..37c9904c3 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -127,6 +127,8 @@ descriptor schema. - `sdk/artifact-manifest`: normalized artifact references. - `sdk/error-info`: structured runtime or adapter error metadata. - `sdk/fabric-event`: NeMo Fabric lifecycle or progress event. +- `sdk/runtime-resource-snapshot`: timestamped memory, process count, and CPU + time of a runtime's local adapter host and its descendants. ### Deferred Core Objects diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 034c17e82..e27c1d1cc 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -279,6 +279,52 @@ }, "type": "object" }, + "RuntimeResourceSnapshot": { + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", + "properties": { + "cpu_time_seconds": { + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding descendants that have exited and been waited for. It can drop\nwhen a descendant leaves the tree, and on macOS while an exited\ndescendant is not yet waited for.", + "format": "double", + "minimum": 0.0, + "type": "number" + }, + "observed_at": { + "description": "When Fabric read these values, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + }, + "process_count": { + "description": "Live processes in the tree, including the adapter host. Exited processes\nthat are not yet reaped are not counted.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "resident_memory_bytes": { + "description": "Sum of the resident set sizes of the live processes, in bytes. Pages\nshared between processes count once per process.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "runtime_id": { + "description": "Runtime this snapshot describes.", + "type": "string" + }, + "started_at": { + "description": "When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + } + }, + "required": [ + "runtime_id", + "started_at", + "observed_at", + "resident_memory_bytes", + "process_count", + "cpu_time_seconds" + ], + "type": "object" + }, "TelemetryRef": { "description": "Reference to telemetry emitted by Relay or another configured telemetry path.", "properties": { @@ -360,6 +406,17 @@ "description": "Request id.", "type": "string" }, + "resource_snapshot": { + "anyOf": [ + { + "$ref": "#/$defs/RuntimeResourceSnapshot" + }, + { + "type": "null" + } + ], + "description": "Resource snapshot of the runtime, read once after the adapter returned this\nresult. Absent when the runtime's resources could not be observed." + }, "runtime_id": { "description": "Runtime handle id.", "type": "string" diff --git a/schemas/sdk/runtime-resource-snapshot.schema.json b/schemas/sdk/runtime-resource-snapshot.schema.json new file mode 100644 index 000000000..86a36a5af --- /dev/null +++ b/schemas/sdk/runtime-resource-snapshot.schema.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", + "properties": { + "cpu_time_seconds": { + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding descendants that have exited and been waited for. It can drop\nwhen a descendant leaves the tree, and on macOS while an exited\ndescendant is not yet waited for.", + "format": "double", + "minimum": 0.0, + "type": "number" + }, + "observed_at": { + "description": "When Fabric read these values, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + }, + "process_count": { + "description": "Live processes in the tree, including the adapter host. Exited processes\nthat are not yet reaped are not counted.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "resident_memory_bytes": { + "description": "Sum of the resident set sizes of the live processes, in bytes. Pages\nshared between processes count once per process.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "runtime_id": { + "description": "Runtime this snapshot describes.", + "type": "string" + }, + "started_at": { + "description": "When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + } + }, + "required": [ + "runtime_id", + "started_at", + "observed_at", + "resident_memory_bytes", + "process_count", + "cpu_time_seconds" + ], + "title": "RuntimeResourceSnapshot", + "type": "object" +} \ No newline at end of file From 0e016fda139504419fe8ffa536ef312bb2d6eb3c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 15:52:50 -0700 Subject: [PATCH 002/159] feat: expose runtime resource snapshots in the Python SDK Python applications need the same snapshot as Rust callers, with the same typed failures, to monitor a runtime while an invocation runs. Add Runtime.resource_snapshot(), the RuntimeResourceSnapshot type, RunResult.resource_snapshot, and ResourceObservationUnavailableError and ResourceObservationUnsupportedError (FabricRuntimeError subclasses). The PyO3 binding raises dedicated native exceptions so the SDK maps them by type rather than by message text. Signed-off-by: mnajafian-nv --- crates/fabric-python/src/lib.rs | 53 ++++- .../api/python-library-reference/index.md | 3 + .../nemo_fabric.errors.md | 82 +++++++ .../nemo_fabric.runtime.md | 21 ++ .../nemo_fabric.types.md | 85 +++++++ .../src/nemo_fabric/__init__.py | 6 + .../src/nemo_fabric/_native.pyi | 4 + .../src/nemo_fabric/errors.py | 12 + .../src/nemo_fabric/runtime.py | 65 +++++- .../src/nemo_fabric/types.py | 75 +++++++ .../python/test_runtime_resource_snapshot.py | 211 ++++++++++++++++++ 11 files changed, 610 insertions(+), 7 deletions(-) create mode 100644 tests/python/test_runtime_resource_snapshot.py diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 2f9cb1f37..44587f33b 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -24,6 +24,18 @@ pyo3::create_exception!( PyRuntimeError, "Raised when a service release is rejected because runtimes are still active." ); +pyo3::create_exception!( + _native, + ResourceObservationUnavailableError, + PyRuntimeError, + "Raised when a runtime stopped, is stopping, its adapter host exited, or the process table cannot be read." +); +pyo3::create_exception!( + _native, + ResourceObservationUnsupportedError, + PyRuntimeError, + "Raised when a runtime's environment provider or operating system has no resource reader." +); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; const PYTHON_DATA_PATH_QUERY_TIMEOUT: Duration = Duration::from_secs(5); @@ -246,9 +258,32 @@ fn stop_runtime(py: Python<'_>, plan_json: String, runtime_json: String) -> PyRe to_json(&events) } +/// Read a resource snapshot of a started runtime and return RuntimeResourceSnapshot JSON. +#[pyfunction] +fn runtime_resource_snapshot( + py: Python<'_>, + plan_json: String, + runtime_json: String, +) -> PyResult { + let plan = parse_run_plan(plan_json)?; + let runtime = parse_runtime_handle(runtime_json)?; + let snapshot = py + .detach(|| nemo_fabric_core::runtime_resource_snapshot(&plan, &runtime)) + .map_err(to_py_error)?; + to_json(&snapshot) +} + #[pymodule] fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("ServiceInUseError", m.py().get_type::())?; + m.add( + "ResourceObservationUnavailableError", + m.py().get_type::(), + )?; + m.add( + "ResourceObservationUnsupportedError", + m.py().get_type::(), + )?; m.add_function(wrap_pyfunction!(version, m)?)?; m.add_function(wrap_pyfunction!(plan_config, m)?)?; m.add_function(wrap_pyfunction!(inspect_adapter_metadata, m)?)?; @@ -262,6 +297,7 @@ fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(invoke_runtime, m)?)?; m.add_function(wrap_pyfunction!(invoke_openai_stream, m)?)?; m.add_function(wrap_pyfunction!(stop_runtime, m)?)?; + m.add_function(wrap_pyfunction!(runtime_resource_snapshot, m)?)?; Ok(()) } @@ -273,13 +309,18 @@ where } fn to_py_error(error: nemo_fabric_core::FabricError) -> PyErr { - if matches!( - &error, - FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" - ) { - return PyTimeoutError::new_err(error.to_string()); + match &error { + FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" => { + PyTimeoutError::new_err(error.to_string()) + } + FabricError::ResourceObservationUnavailable { .. } => { + ResourceObservationUnavailableError::new_err(error.to_string()) + } + FabricError::ResourceObservationUnsupported { .. } => { + ResourceObservationUnsupportedError::new_err(error.to_string()) + } + _ => PyRuntimeError::new_err(error.to_string()), } - PyRuntimeError::new_err(error.to_string()) } fn resolve_context( diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index c1bdd6e0c..24fb1d403 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -75,6 +75,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`types.RunUsage`](./nemo_fabric.types.md#class-runusage): Normalized invocation-local usage reported by an adapter target. - [`types.RuntimeCapabilities`](./nemo_fabric.types.md#class-runtimecapabilities): Operations declared by the resolved runtime and adapter. - [`types.RuntimeHandle`](./nemo_fabric.types.md#class-runtimehandle): Opaque identity and binding for one started runtime. +- [`types.RuntimeResourceSnapshot`](./nemo_fabric.types.md#class-runtimeresourcesnapshot): Timestamped resource usage of one runtime's local adapter host process tree. - [`types.ServiceHandle`](./nemo_fabric.types.md#class-servicehandle): Opaque identity and binding for one prepared or attached service. - [`types.ServiceReference`](./nemo_fabric.types.md#class-servicereference): Reference to an already-running caller-owned service. - [`types.TelemetryRef`](./nemo_fabric.types.md#class-telemetryref): Reference to external or persisted telemetry for a run. @@ -84,6 +85,8 @@ SPDX-License-Identifier: Apache-2.0 --> - [`errors.FabricNativeUnavailableError`](./nemo_fabric.errors.md#class-fabricnativeunavailableerror): SDK call requires the PyO3 extension, but it is not installed or importable. - [`errors.FabricRuntimeError`](./nemo_fabric.errors.md#class-fabricruntimeerror): Failure while starting, invoking, stopping, or otherwise driving a runtime. - [`errors.FabricStateError`](./nemo_fabric.errors.md#class-fabricstateerror): Operation rejected because a local runtime is in the wrong state. +- [`errors.ResourceObservationUnavailableError`](./nemo_fabric.errors.md#class-resourceobservationunavailableerror): Runtime resources cannot be read now. +- [`errors.ResourceObservationUnsupportedError`](./nemo_fabric.errors.md#class-resourceobservationunsupportederror): The runtime's environment provider or operating system has no resource reader. ## Functions diff --git a/docs/reference/api/python-library-reference/nemo_fabric.errors.md b/docs/reference/api/python-library-reference/nemo_fabric.errors.md index 4b5559100..73b6ba7a8 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.errors.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.errors.md @@ -275,6 +275,88 @@ Initialize a structured NeMo Fabric exception. +--- + +## class `ResourceObservationUnavailableError` + +Runtime resources cannot be read now. + +The runtime stopped, is stopping, its adapter host process exited, or the process table cannot be read. + + +### Inheritance + +Direct base: `FabricRuntimeError`. + +### method `__init__` + +```python +def __init__( + message: str, + *, + stage: str | None = None, + code: str | None = None, + retryable: bool = False, + details: Mapping[str, Any] | None = None, +) -> None +``` + +Initialize a structured NeMo Fabric exception. + + + +**Args:** + + - `message`: Human-readable failure description. + - `stage`: Optional lifecycle stage that failed. + - `code`: Optional stable machine-readable error code. + - `retryable`: Whether callers may safely retry unchanged input. + - `details`: Optional structured diagnostic context. The exception stores a deep copy. + + + + + +--- + +## class `ResourceObservationUnsupportedError` + +The runtime's environment provider or operating system has no resource reader. + + +### Inheritance + +Direct base: `FabricRuntimeError`. + +### method `__init__` + +```python +def __init__( + message: str, + *, + stage: str | None = None, + code: str | None = None, + retryable: bool = False, + details: Mapping[str, Any] | None = None, +) -> None +``` + +Initialize a structured NeMo Fabric exception. + + + +**Args:** + + - `message`: Human-readable failure description. + - `stage`: Optional lifecycle stage that failed. + - `code`: Optional stable machine-readable error code. + - `retryable`: Whether callers may safely retry unchanged input. + - `details`: Optional structured diagnostic context. The exception stores a deep copy. + + + + + --- diff --git a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md index a9bde67df..b77596c57 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md @@ -183,6 +183,27 @@ Start one turn and stream raw NeMo Relay ATOF records as they arrive. --- +### method `resource_snapshot` + +```python +async def resource_snapshot() -> RuntimeResourceSnapshot +``` + +Return a timestamped resource snapshot of this runtime. + +The snapshot covers the local adapter host process and its live descendants. It does not wait for a running invocation, so it can be called while ``invoke()`` is in flight. It observes resources only and enforces no limits. + + + +**Raises:** + + - `ResourceObservationUnavailableError`: If the runtime stopped, is stopping, its adapter host process exited, or the process table cannot be read. + - `ResourceObservationUnsupportedError`: If the runtime's environment provider or operating system has no resource reader. + - `FabricNativeUnavailableError`: If the native extension is missing. + - `FabricRuntimeError`: If the native snapshot call fails otherwise. + +--- + ### method `stop` ```python diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 21a7ff445..528624f3e 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1244,6 +1244,89 @@ Return a detached, JSON-compatible mapping for serialization. --- +## class `RuntimeResourceSnapshot` + +Timestamped resource usage of one runtime's local adapter host process tree. + +Memory, process count, and CPU time cover the adapter host process and its live descendants, not a shared service process. Values that cannot be read raise an error instead of reading as zero. + + + +**Attributes:** + + - `runtime_id`: Runtime this snapshot describes. + - `started_at`: When Fabric started the adapter host, as RFC 3339 UTC. + - `observed_at`: When Fabric read these values, as RFC 3339 UTC. + - `resident_memory_bytes`: Sum of the live processes' resident set sizes. Pages shared between processes count once per process. + - `process_count`: Live processes in the tree, including the adapter host. + - `cpu_time_seconds`: CPU time used since the adapter host started, including descendants that have exited and been waited for. It can drop when a descendant leaves the tree. + + +### Fields + +The mapping exposes the following typed fields: + +| Field | Type | +| --- | --- | +| `runtime_id` | `str` | +| `started_at` | `str` | +| `observed_at` | `str` | +| `resident_memory_bytes` | `int` | +| `process_count` | `int` | +| `cpu_time_seconds` | `float` | + +### method `__init__` + +```python +def __init__(mapping: Mapping[str, Any]) -> None +``` + + + + + + +--- + +### property extra_fields + +Return an immutable view of preserved extension fields. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(mapping: Mapping[str, Any]) -> Self +``` + +Validate and copy a mapping into the requested typed model. + +--- + +### method `to_dict` + +```python +def to_dict() -> dict[str, Any] +``` + +Return the same detached representation as ``to_mapping()``. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached, JSON-compatible mapping for serialization. + + +--- + ## class `RunResult` Normalized terminal result from one NeMo Fabric invocation. @@ -1265,6 +1348,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - `error`: Structured failure, or ``None`` on success. - `usage`: Normalized invocation usage, or ``None`` when unavailable. + - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. - `artifacts`: Normalized artifact manifest. - `telemetry`: Ordered telemetry references. - `events`: Ordered lifecycle and invocation events. @@ -1289,6 +1373,7 @@ The mapping exposes the following typed fields: | `output` | `RunOutput \| JSONValue` | | `error` | `ErrorInfo \| None` | | `usage` | `RunUsage \| None` | +| `resource_snapshot` | `RuntimeResourceSnapshot \| None` | | `artifacts` | `ArtifactManifest` | | `telemetry` | `Sequence[TelemetryRef]` | | `events` | `Sequence[FabricEvent]` | diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index aaa15bfe0..bfc60118f 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -12,6 +12,8 @@ from nemo_fabric.errors import FabricNativeUnavailableError from nemo_fabric.errors import FabricRuntimeError from nemo_fabric.errors import FabricStateError +from nemo_fabric.errors import ResourceObservationUnavailableError +from nemo_fabric.errors import ResourceObservationUnsupportedError from nemo_fabric.models import EnvironmentConfig from nemo_fabric.models import DiscoveryConfig from nemo_fabric.models import FabricBaseModel @@ -63,6 +65,7 @@ from nemo_fabric.types import RunUsage from nemo_fabric.types import RuntimeCapabilities from nemo_fabric.types import RuntimeHandle +from nemo_fabric.types import RuntimeResourceSnapshot from nemo_fabric.types import ServiceHandle from nemo_fabric.types import ServiceReference from nemo_fabric.types import TelemetryRef @@ -109,6 +112,8 @@ "FabricNativeUnavailableError", "FabricRuntimeError", "FabricStateError", + "ResourceObservationUnavailableError", + "ResourceObservationUnsupportedError", "RunOutput", "RunPlan", "RunRequest", @@ -116,6 +121,7 @@ "RunUsage", "RuntimeCapabilities", "RuntimeHandle", + "RuntimeResourceSnapshot", "RuntimeConfig", "Runtime", "RuntimeStatus", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi index 30322d181..9b969fc02 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_native.pyi @@ -36,3 +36,7 @@ def invoke_openai_stream( transport_json: str, ) -> str: ... def stop_runtime(plan_json: str, runtime_json: str) -> str: ... +def runtime_resource_snapshot(plan_json: str, runtime_json: str) -> str: ... + +class ResourceObservationUnavailableError(RuntimeError): ... +class ResourceObservationUnsupportedError(RuntimeError): ... diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index 1ac906fcf..5edb2bbf4 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -79,3 +79,15 @@ class FabricCapabilityError(FabricRuntimeError): class FabricNativeUnavailableError(FabricRuntimeError): """SDK call requires the PyO3 extension, but it is not installed or importable.""" + + +class ResourceObservationUnavailableError(FabricRuntimeError): + """Runtime resources cannot be read now. + + The runtime stopped, is stopping, its adapter host process exited, or the + process table cannot be read. + """ + + +class ResourceObservationUnsupportedError(FabricRuntimeError): + """The runtime's environment provider or operating system has no resource reader.""" diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index b5a6e11eb..28be1b623 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -22,13 +22,21 @@ FabricCapabilityError, FabricConfigError, FabricError, + FabricRuntimeError, FabricStateError, + ResourceObservationUnavailableError, + ResourceObservationUnsupportedError, _runtime_error, ) from nemo_fabric.models import RunRequest from nemo_fabric.openai_streaming import OpenAIInvokeStream from nemo_fabric.streaming import InvokeStream -from nemo_fabric.types import RunPlan, RunResult, RuntimeHandle +from nemo_fabric.types import ( + RunPlan, + RunResult, + RuntimeHandle, + RuntimeResourceSnapshot, +) logger = logging.getLogger(__name__) @@ -731,6 +739,41 @@ def _ensure_invocable(self) -> None: if self._current_task is not None: raise FabricStateError("runtime is already running an invocation") + async def resource_snapshot(self) -> RuntimeResourceSnapshot: + """Return a timestamped resource snapshot of this runtime. + + The snapshot covers the local adapter host process and its live + descendants. It does not wait for a running invocation, so it can be + called while ``invoke()`` is in flight. It observes resources only and + enforces no limits. + + Raises: + ResourceObservationUnavailableError: If the runtime stopped, is + stopping, its adapter host process exited, or the process + table cannot be read. + ResourceObservationUnsupportedError: If the runtime's environment + provider or operating system has no resource reader. + FabricNativeUnavailableError: If the native extension is missing. + FabricRuntimeError: If the native snapshot call fails otherwise. + """ + + native = self._client._require_native_module("resource_snapshot") + + def snapshot() -> Any: + return json.loads( + native.runtime_resource_snapshot( + json.dumps(self._plan.to_mapping()), + json.dumps(self._runtime.to_mapping()), + ) + ) + + try: + return RuntimeResourceSnapshot.from_mapping(await _call_blocking(snapshot)) + except FabricError: + raise + except Exception as error: + raise _resource_snapshot_error(native, error) from error + async def stop(self) -> None: """Destroy an idle runtime exactly once. @@ -985,6 +1028,26 @@ def run() -> dict[str, Any]: raise _runtime_error(error, stage="run") from error +def _resource_snapshot_error(native: Any, error: Exception) -> FabricRuntimeError: + """Map the native resource-observation errors to their SDK types.""" + typed_errors: tuple[tuple[str, type[FabricRuntimeError], str], ...] = ( + ( + "ResourceObservationUnavailableError", + ResourceObservationUnavailableError, + "resource_observation_unavailable", + ), + ( + "ResourceObservationUnsupportedError", + ResourceObservationUnsupportedError, + "resource_observation_unsupported", + ), + ) + for native_name, sdk_type, code in typed_errors: + if isinstance(error, getattr(native, native_name, ())): + return sdk_type(str(error), stage="resource_snapshot", code=code) + return _runtime_error(error, stage="resource_snapshot") + + async def _call_blocking(func: Any) -> Any: worker = asyncio.create_task(asyncio.to_thread(func)) try: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 04504e4dc..07e22c7ac 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1834,6 +1834,71 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: return data +class RuntimeResourceSnapshot(FabricMapping): + """Timestamped resource usage of one runtime's local adapter host process tree. + + Memory, process count, and CPU time cover the adapter host process and its + live descendants, not a shared service process. Values that cannot be read + raise an error instead of reading as zero. + + Attributes: + runtime_id: Runtime this snapshot describes. + started_at: When Fabric started the adapter host, as RFC 3339 UTC. + observed_at: When Fabric read these values, as RFC 3339 UTC. + resident_memory_bytes: Sum of the live processes' resident set sizes. + Pages shared between processes count once per process. + process_count: Live processes in the tree, including the adapter host. + cpu_time_seconds: CPU time used since the adapter host started, + including descendants that have exited and been waited for. It + can drop when a descendant leaves the tree. + """ + + runtime_id: str + started_at: str + observed_at: str + resident_memory_bytes: int + process_count: int + cpu_time_seconds: float + _fields = frozenset( + { + "runtime_id", + "started_at", + "observed_at", + "resident_memory_bytes", + "process_count", + "cpu_time_seconds", + } + ) + + @classmethod + def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: + for field in ("runtime_id", "started_at", "observed_at"): + data[field] = _required_text(data.get(field), field.replace("_", " ")) + for field in ("resident_memory_bytes", "process_count"): + value = data.get(field) + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 0 <= value <= _UINT64_MAX + ): + raise FabricConfigError( + f"{field.replace('_', ' ')} must be a nonnegative integer " + f"no greater than {_UINT64_MAX}" + ) + cpu = data.get("cpu_time_seconds") + if ( + isinstance(cpu, bool) + or not isinstance(cpu, (int, float)) + or not math.isfinite(cpu) + or cpu < 0 + ): + raise FabricConfigError( + "cpu time seconds must be a finite nonnegative number" + ) + data["cpu_time_seconds"] = float(cpu) + return data + + class RunResult(FabricMapping): """Normalized terminal result from one NeMo Fabric invocation. @@ -1854,6 +1919,9 @@ class RunResult(FabricMapping): are preserved as-is. error: Structured failure, or ``None`` on success. usage: Normalized invocation usage, or ``None`` when unavailable. + resource_snapshot: Runtime resource snapshot read once after the + adapter returned this result, or ``None`` when the runtime's + resources could not be observed. artifacts: Normalized artifact manifest. telemetry: Ordered telemetry references. events: Ordered lifecycle and invocation events. @@ -1871,6 +1939,7 @@ class RunResult(FabricMapping): output: RunOutput | JSONValue error: ErrorInfo | None usage: RunUsage | None + resource_snapshot: RuntimeResourceSnapshot | None artifacts: ArtifactManifest telemetry: Sequence[TelemetryRef] events: Sequence[FabricEvent] @@ -1888,6 +1957,7 @@ class RunResult(FabricMapping): "output", "error", "usage", + "resource_snapshot", "artifacts", "telemetry", "events", @@ -1914,6 +1984,11 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: data["usage"] = ( None if data.get("usage") is None else RunUsage.from_mapping(data["usage"]) ) + data["resource_snapshot"] = ( + None + if data.get("resource_snapshot") is None + else RuntimeResourceSnapshot.from_mapping(data["resource_snapshot"]) + ) data["artifacts"] = ArtifactManifest.from_mapping( data.get("artifacts", {"artifacts": []}) ) diff --git a/tests/python/test_runtime_resource_snapshot.py b/tests/python/test_runtime_resource_snapshot.py new file mode 100644 index 000000000..db63e546a --- /dev/null +++ b/tests/python/test_runtime_resource_snapshot.py @@ -0,0 +1,211 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Contract tests for runtime resource snapshots.""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import nemo_fabric._native as native +import pytest +from nemo_fabric import Fabric +from nemo_fabric import FabricConfig +from nemo_fabric import FabricConfigError +from nemo_fabric import FabricRuntimeError +from nemo_fabric import ResourceObservationUnavailableError +from nemo_fabric import ResourceObservationUnsupportedError +from nemo_fabric import RunRequest +from nemo_fabric import RunResult +from nemo_fabric import Runtime +from nemo_fabric import RuntimeResourceSnapshot + +_SNAPSHOT = { + "runtime_id": "runtime-1", + "started_at": "2026-10-15T09:00:00.000Z", + "observed_at": "2026-10-15T09:00:05.250Z", + "resident_memory_bytes": 52_428_800, + "process_count": 3, + "cpu_time_seconds": 1.25, +} +_SUPPORTED_PLATFORM = sys.platform in {"linux", "darwin"} + + +def _shim_config() -> FabricConfig: + return FabricConfig.from_mapping( + { + "metadata": {"name": "resource-snapshot-probe"}, + "harness": { + "adapter_id": "test.fabric.hermes_shim", + "resolution": "preinstalled", + }, + "discovery": {"local_paths": ["adapters"]}, + } + ) + + +def _runtime_with_native(native_module: Any) -> Runtime: + client = Fabric() + client._native_module = lambda: native_module # type: ignore[method-assign] + return Runtime( + client=client, + plan={ + "agent_name": "demo", + "base_dir": ".", + "config": { + "metadata": {"name": "demo"}, + "harness": {"adapter_id": "test.fabric.shim"}, + }, + "adapter_descriptor": { + "descriptor": { + "adapter_id": "test.fabric.shim", + "harness": "hermes", + "adapter_kind": "python", + } + }, + "capabilities": { + "service": False, + "streaming": False, + "updates": False, + "cancellation": False, + }, + }, + runtime={ + "runtime_id": "runtime-1", + "runtime_binding": "fabric-runtime-binding-test", + "agent_name": "demo", + "harness": "hermes", + "adapter_kind": "python", + "adapter_id": "test.fabric.shim", + "environment": { + "environment_id": "environment-1", + "provider": "local", + "control_location": "external_control", + "ownership": "caller_owned", + }, + }, + ) + + +def test_runtime_resource_snapshot_with_invalid_field_raises_config_error(): + snapshot = RuntimeResourceSnapshot.from_mapping(_SNAPSHOT) + assert snapshot.cpu_time_seconds == 1.25 + assert snapshot.to_mapping() == _SNAPSHOT + + for field, value in ( + ("resident_memory_bytes", -1), + ("process_count", True), + ("cpu_time_seconds", float("nan")), + ("observed_at", ""), + ): + with pytest.raises(FabricConfigError): + RuntimeResourceSnapshot.from_mapping({**_SNAPSHOT, field: value}) + + +def test_run_result_resource_snapshot_is_typed_or_none(): + base = { + "agent_name": "demo", + "harness": "hermes", + "adapter_kind": "python", + "runtime_id": "runtime-1", + "invocation_id": "invocation-1", + "request_id": "request-1", + "status": "succeeded", + } + + with_snapshot = RunResult.from_mapping({**base, "resource_snapshot": _SNAPSHOT}) + without_snapshot = RunResult.from_mapping(base) + + assert isinstance(with_snapshot.resource_snapshot, RuntimeResourceSnapshot) + assert with_snapshot.resource_snapshot.process_count == 3 + assert without_snapshot.resource_snapshot is None + + +@pytest.mark.parametrize( + ("native_name", "sdk_type", "code"), + [ + ( + "ResourceObservationUnavailableError", + ResourceObservationUnavailableError, + "resource_observation_unavailable", + ), + ( + "ResourceObservationUnsupportedError", + ResourceObservationUnsupportedError, + "resource_observation_unsupported", + ), + ], +) +async def test_resource_snapshot_native_error_maps_to_typed_sdk_error( + native_name, sdk_type, code +): + native_error = getattr(native, native_name) + fake_native = MagicMock() + fake_native.ResourceObservationUnavailableError = ( + native.ResourceObservationUnavailableError + ) + fake_native.ResourceObservationUnsupportedError = ( + native.ResourceObservationUnsupportedError + ) + fake_native.runtime_resource_snapshot.side_effect = native_error("no reader") + runtime = _runtime_with_native(fake_native) + + with pytest.raises(sdk_type) as caught: + await runtime.resource_snapshot() + + assert isinstance(caught.value, FabricRuntimeError) + assert caught.value.code == code + assert caught.value.stage == "resource_snapshot" + + +@pytest.mark.skipif(not _SUPPORTED_PLATFORM, reason="needs a process reader") +async def test_resource_snapshot_reads_active_runtime_and_final_result( + hermes_shim_agent_dir: Path, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setenv("ADAPTER_PYTHON", sys.executable) + runtime = await Fabric().start_runtime( + _shim_config(), base_dir=hermes_shim_agent_dir + ) + try: + invocation = asyncio.create_task( + runtime.invoke( + request=RunRequest(input="slow", context={"delay_seconds": 2}) + ) + ) + await asyncio.sleep(0.3) + live = await runtime.resource_snapshot() + assert not invocation.done(), "snapshot waited for the invocation" + result = await invocation + finally: + await runtime.stop() + + assert live.runtime_id == runtime.runtime_id + assert live.process_count >= 1 + assert live.resident_memory_bytes > 0 + assert isinstance(result.resource_snapshot, RuntimeResourceSnapshot) + assert result.resource_snapshot.runtime_id == runtime.runtime_id + assert result.resource_snapshot.started_at == live.started_at + with pytest.raises(ResourceObservationUnavailableError): + await runtime.resource_snapshot() + + +@pytest.mark.skipif(_SUPPORTED_PLATFORM, reason="platform has a process reader") +async def test_resource_snapshot_on_unsupported_platform_raises_unsupported( + hermes_shim_agent_dir: Path, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setenv("ADAPTER_PYTHON", sys.executable) + runtime = await Fabric().start_runtime( + _shim_config(), base_dir=hermes_shim_agent_dir + ) + try: + with pytest.raises(ResourceObservationUnsupportedError): + await runtime.resource_snapshot() + result = await runtime.invoke(input="hello") + finally: + await runtime.stop() + + assert result.resource_snapshot is None From ce4f5f4d7272404aca6aa44f4c6233fd5b834495 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 15:52:50 -0700 Subject: [PATCH 003/159] docs: document runtime resource snapshots Issue #267 requires the fixed measurement scope to be documented and observation to be kept distinct from resource enforcement. Describe the fields, the adapter host plus descendants scope, the typed errors, the CPU caveats for per-invocation deltas, and that limits belong to the operating system or execution environment, in the SDK guide and the integration skill reference. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 56 +++++++++++++++++++ .../references/results-and-errors.md | 15 +++++ 2 files changed, 71 insertions(+) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 227e0bd44..0a383bcad 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -556,6 +556,7 @@ The following table summarizes the SDK entry points and runtime methods: | `Runtime.invoke(...)` | Yes | You need one turn on an existing runtime. | A runtime permits one active invocation at a time. | | `Runtime.invoke_openai_stream(...)` | No | You need adapter-native OpenAI Chat Completions chunks for one turn. | Requires descriptor `capabilities.streaming`. Returns an async `OpenAIInvokeStream`; await `stream.result()` for the separate terminal `RunResult`. | | `Runtime.invoke_stream(...)` | No | You need live ATOF records generated by NeMo Relay for one turn. | Returns an async `InvokeStream`; await `stream.result()` for the terminal `RunResult`. | +| `Runtime.resource_snapshot()` | Yes | You need the memory, process count, or CPU time of a started runtime. | Does not wait for an active invocation. Refer to [Runtime Resource Snapshots](#runtime-resource-snapshots). | | `Runtime.stop()` | Yes | You need to stop or detach from the runtime. | Called automatically when using `async with`. | | `Service.release()` | Yes | You need to stop a prepared service or detach from an attached service. | Fails while any connected runtime remains active. Called automatically when using `async with`. | @@ -678,6 +679,57 @@ on one live thread, Deep Agents invokes one compiled graph and checkpointer, Hermes Agent reuses one agent and session database, and Claude keeps one connected SDK client. Harness-native identifiers remain adapter-internal. +## Runtime Resource Snapshots + +`Runtime.resource_snapshot()` returns a timestamped `RuntimeResourceSnapshot` +for a started local runtime. It does not wait for an active invocation, so a +monitor can call it while `invoke()` runs: + +```python +snapshot = await runtime.resource_snapshot() +print(snapshot.observed_at, snapshot.resident_memory_bytes) +print(snapshot.process_count, snapshot.cpu_time_seconds) +``` + +| Field | Meaning | +| --- | --- | +| `runtime_id` | Runtime the snapshot describes. | +| `started_at` | When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. | +| `observed_at` | When NeMo Fabric read the values, as an RFC 3339 UTC timestamp. | +| `resident_memory_bytes` | Sum of the resident set sizes of the live processes. Pages shared between processes count once per process. | +| `process_count` | Live processes, including the adapter host. Exited processes that are not yet reaped are not counted. | +| `cpu_time_seconds` | CPU time used since the adapter host started, including descendants that have exited and been waited for. It can drop when a descendant leaves the tree. | + +The scope is fixed: the local adapter host process that NeMo Fabric started for +the runtime, plus its descendants. For a runtime started on a shared service, +the service process is not included. A descendant that starts a new session or +process group still counts while its parent chain leads to the host. A process +whose parent exited is reparented outside the tree and no longer counts. The +snapshot reads `/proc` on Linux and `proc_pid_rusage` on macOS. It uses only +process IDs, parent IDs, start times, CPU times, and resident sizes, never reads +command lines or environment variables, and reports only the runtime's tree. +Descendants that NeMo Fabric is not permitted to inspect are left out. + +When NeMo Fabric cannot read the values, it raises an error instead of +returning zeros: + +- `ResourceObservationUnavailableError` when the runtime stopped, is stopping, + its adapter host process exited, or the process table cannot be read. +- `ResourceObservationUnsupportedError` when the runtime's environment provider + or operating system has no resource reader. Windows raises this error. + +Each `RunResult` from a local runtime also carries `resource_snapshot`, read +once after the adapter returned the result. It is `None` when the read failed; +a failed read never fails the invocation. To estimate one invocation's CPU time, +take a snapshot before it and subtract its `cpu_time_seconds` from the +result's snapshot. CPU time can drop when a descendant is reparented out of the +tree, and on macOS while an exited descendant is not yet waited for, so treat a +difference as an estimate and a negative difference as unknown. + +Snapshots observe resources only. They do not enforce limits, stop runtimes, or +queue requests. Use the operating system or execution environment, such as +cgroups or container limits, to enforce resource limits. + ## Native OpenAI Streaming Use `Runtime.invoke_openai_stream()` when the selected adapter provides native @@ -908,6 +960,7 @@ Important fields: | `output` | Harness output normalized to the configured output schema. | | `error` | Structured failure metadata when available. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | +| `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. | | `artifacts` | Output files, logs, patches, native artifacts, and other materialized references. | | `telemetry` | References to NeMo Relay or other telemetry streams produced by the run. | | `events` | Ordered normalized lifecycle and invocation events. | @@ -1062,6 +1115,8 @@ All public SDK errors inherit from `FabricError`. | `FabricRuntimeError` | Runtime startup, invocation, or shutdown failed before a normalized result could be returned. | | `FabricStateError` | Invalid runtime state transition, such as invoking after stop or starting overlapping invocations. | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | +| `ResourceObservationUnavailableError` | `Runtime.resource_snapshot()` cannot read the runtime because it stopped, is stopping, its adapter host exited, or the process table cannot be read. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnsupportedError` | The runtime's environment provider or operating system has no resource reader. Subclass of `FabricRuntimeError`. | Consumers own job-level retries and rollout-level failure policy. Core host-operation deadlines (`host_timeout`) become Python `TimeoutError`, then @@ -1098,6 +1153,7 @@ generated API reference for exact Python signatures. | Request, result, and events | `schemas/sdk/run-request.schema.json`, `schemas/sdk/run-result.schema.json`, `schemas/sdk/fabric-event.schema.json` | | Runtime and invocation handles | `schemas/sdk/runtime-handle.schema.json`, `schemas/sdk/invocation-handle.schema.json` | | Artifacts and errors | `schemas/sdk/artifact-manifest.schema.json`, `schemas/sdk/error-info.schema.json` | +| Runtime resource snapshot | `schemas/sdk/runtime-resource-snapshot.schema.json` | ### Versioning diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index f2660d5e3..6ef4570d8 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -17,6 +17,7 @@ Every invocation that reaches the adapter boundary returns a normalized | `error` | Structured failure metadata when available. Branch on `status`, not on this field. | | `output` | Harness output normalized to the configured output schema. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | +| `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed. | | `artifacts` | Output files, logs, patches, and other materialized references. | | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | @@ -46,6 +47,18 @@ if result.usage is not None: Treat missing counters and cost as unknown, not zero. `cached_input_tokens` reports cached prompt tokens; `input_tokens_include_cache` declares whether `input_tokens` includes them (`True`), excludes them (`False`), or has unknown semantics (absent). Do not add cache to an inclusive input count. Adapters can put cache-write, reasoning, peak request, and per-model detail in the descriptor-declared `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` object. NeMo Fabric preserves that object at `result.usage["metadata"]["nemo_fabric.detailed_token_usage.v1"]`. Treat detailed input and output independently: any detailed accounting field for a dimension makes that dimension authoritative, with no fallback to its top-level counters. The views can diverge. Calculate an inclusive total only when the selected detailed counters provide enough information; otherwise leave the total unknown. Keep estimated costs separate in provider metadata; `cost_usd` is reported cost, not an estimate. +## Runtime Resource Snapshots + +`await runtime.resource_snapshot()` returns a `RuntimeResourceSnapshot` with +`runtime_id`, `started_at` and `observed_at` (RFC 3339 UTC), `resident_memory_bytes`, +`process_count`, and `cpu_time_seconds`. It covers the local adapter host process +and its descendants (not a shared service process), and it does not wait for +an active invocation. CPU time is cumulative since the host started, so +subtract an earlier snapshot to estimate one invocation; it can drop when a +descendant leaves the tree, so treat a negative difference as unknown. Values that cannot be read raise an error, never read as zero. +Snapshots observe only; enforce limits in the operating system or execution +environment. + ## Correlation IDs `runtime_id` identifies the runtime lifecycle, `invocation_id` identifies one @@ -69,6 +82,8 @@ Refer to the [errors reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/ | `FabricRuntimeError` | Startup, invocation, or shutdown failed before a normalized result. | | `FabricStateError` | Invalid runtime state transition (invoking after stop, overlapping invocations). | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | +| `ResourceObservationUnavailableError` | A resource snapshot cannot be read because the runtime stopped, is stopping, its adapter host exited, or the process table cannot be read. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnsupportedError` | The runtime's environment provider or operating system has no resource reader. Subclass of `FabricRuntimeError`. | ## Cleanup And Resilience From d5a09f9beb737b83ee12818a3a52e1456c90f2bd Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 16:02:56 -0700 Subject: [PATCH 004/159] docs: state which exited descendants count toward snapshot CPU time Linux counts an exited descendant's CPU before its parent waits for it, while macOS counts it only after the wait. The field docs said only waited-for descendants count, which understated Linux. Say both, and re-wrap the integration skill paragraph. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 6 +++--- .../python-library-reference/nemo_fabric.types.md | 2 +- .../struct-runtimeresourcesnapshot.mdx | 2 +- docs/sdk/python.mdx | 2 +- schemas/sdk/run-result.schema.json | 2 +- schemas/sdk/runtime-resource-snapshot.schema.json | 2 +- .../nemo-fabric-runtime/src/nemo_fabric/types.py | 4 ++-- .../references/results-and-errors.md | 13 +++++++------ 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 8401f096e..b76aa535d 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -49,9 +49,9 @@ pub struct RuntimeResourceSnapshot { /// that are not yet reaped are not counted. pub process_count: u64, /// CPU time consumed by the tree since the adapter host started, in seconds, - /// including descendants that have exited and been waited for. It can drop - /// when a descendant leaves the tree, and on macOS while an exited - /// descendant is not yet waited for. + /// including exited descendants (on macOS, only once they are waited for). + /// It can drop when a descendant leaves the tree, and on macOS while an + /// exited descendant is not yet waited for. #[schemars(range(min = 0.0))] pub cpu_time_seconds: f64, } diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 528624f3e..623c34593 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1259,7 +1259,7 @@ Memory, process count, and CPU time cover the adapter host process and its live - `observed_at`: When Fabric read these values, as RFC 3339 UTC. - `resident_memory_bytes`: Sum of the live processes' resident set sizes. Pages shared between processes count once per process. - `process_count`: Live processes in the tree, including the adapter host. - - `cpu_time_seconds`: CPU time used since the adapter host started, including descendants that have exited and been waited for. It can drop when a descendant leaves the tree. + - `cpu_time_seconds`: CPU time used since the adapter host started, including exited descendants (on macOS, only once they are waited for). It can drop when a descendant leaves the tree. ### Fields diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx index 82f8493e1..ba0517771 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx @@ -40,7 +40,7 @@ Live processes in the tree, including the adapter host. Exited processes that ar ### `cpu_time_seconds: f64` -CPU time consumed by the tree since the adapter host started, in seconds, including descendants that have exited and been waited for. It can drop when a descendant leaves the tree, and on macOS while an exited descendant is not yet waited for. +CPU time consumed by the tree since the adapter host started, in seconds, including exited descendants (on macOS, only once they are waited for). It can drop when a descendant leaves the tree, and on macOS while an exited descendant is not yet waited for. ## Trait Implementations diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 0a383bcad..6fbc919b8 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -698,7 +698,7 @@ print(snapshot.process_count, snapshot.cpu_time_seconds) | `observed_at` | When NeMo Fabric read the values, as an RFC 3339 UTC timestamp. | | `resident_memory_bytes` | Sum of the resident set sizes of the live processes. Pages shared between processes count once per process. | | `process_count` | Live processes, including the adapter host. Exited processes that are not yet reaped are not counted. | -| `cpu_time_seconds` | CPU time used since the adapter host started, including descendants that have exited and been waited for. It can drop when a descendant leaves the tree. | +| `cpu_time_seconds` | CPU time used since the adapter host started, including exited descendants (on macOS, only once they are waited for). It can drop when a descendant leaves the tree. | The scope is fixed: the local adapter host process that NeMo Fabric started for the runtime, plus its descendants. For a runtime started on a shared service, diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index e27c1d1cc..072be7097 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -283,7 +283,7 @@ "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", "properties": { "cpu_time_seconds": { - "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding descendants that have exited and been waited for. It can drop\nwhen a descendant leaves the tree, and on macOS while an exited\ndescendant is not yet waited for.", + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants (on macOS, only once they are waited for).\nIt can drop when a descendant leaves the tree, and on macOS while an\nexited descendant is not yet waited for.", "format": "double", "minimum": 0.0, "type": "number" diff --git a/schemas/sdk/runtime-resource-snapshot.schema.json b/schemas/sdk/runtime-resource-snapshot.schema.json index 86a36a5af..8a5bf94ad 100644 --- a/schemas/sdk/runtime-resource-snapshot.schema.json +++ b/schemas/sdk/runtime-resource-snapshot.schema.json @@ -3,7 +3,7 @@ "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", "properties": { "cpu_time_seconds": { - "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding descendants that have exited and been waited for. It can drop\nwhen a descendant leaves the tree, and on macOS while an exited\ndescendant is not yet waited for.", + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants (on macOS, only once they are waited for).\nIt can drop when a descendant leaves the tree, and on macOS while an\nexited descendant is not yet waited for.", "format": "double", "minimum": 0.0, "type": "number" diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 07e22c7ac..c1ddf0323 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1849,8 +1849,8 @@ class RuntimeResourceSnapshot(FabricMapping): Pages shared between processes count once per process. process_count: Live processes in the tree, including the adapter host. cpu_time_seconds: CPU time used since the adapter host started, - including descendants that have exited and been waited for. It - can drop when a descendant leaves the tree. + including exited descendants (on macOS, only once they are + waited for). It can drop when a descendant leaves the tree. """ runtime_id: str diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 6ef4570d8..094b75614 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -50,12 +50,13 @@ Treat missing counters and cost as unknown, not zero. `cached_input_tokens` repo ## Runtime Resource Snapshots `await runtime.resource_snapshot()` returns a `RuntimeResourceSnapshot` with -`runtime_id`, `started_at` and `observed_at` (RFC 3339 UTC), `resident_memory_bytes`, -`process_count`, and `cpu_time_seconds`. It covers the local adapter host process -and its descendants (not a shared service process), and it does not wait for -an active invocation. CPU time is cumulative since the host started, so -subtract an earlier snapshot to estimate one invocation; it can drop when a -descendant leaves the tree, so treat a negative difference as unknown. Values that cannot be read raise an error, never read as zero. +`runtime_id`, `started_at` and `observed_at` (RFC 3339 UTC), +`resident_memory_bytes`, `process_count`, and `cpu_time_seconds`. It covers the +local adapter host process and its descendants (not a shared service process), +and it does not wait for an active invocation. CPU time is cumulative since the +host started, so subtract an earlier snapshot to estimate one invocation; it +can drop when a descendant leaves the tree, so treat a negative difference as +unknown. Values that cannot be read raise an error, never read as zero. Snapshots observe only; enforce limits in the operating system or execution environment. From 85f1e5a851e2d94439faa71f7032456ae9a2dfc5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 22:47:18 -0700 Subject: [PATCH 005/159] refactor: queue macOS snapshot children with their usage Every entry the macOS reader queues already carries the rusage it was filtered on, so the optional value and the fallback re-read were dead paths. Queue the usage directly. Behavior is unchanged. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index b76aa535d..a1049aab1 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -414,11 +414,8 @@ mod platform { let mut read = TreeRead::default(); let mut cpu_units: u64 = 0; let mut resident: u64 = 0; - let mut pending = VecDeque::from([(root, Some(root_info))]); - while let Some((pid, known)) = pending.pop_front() { - let Some(info) = known.or_else(|| rusage(pid)) else { - continue; - }; + let mut pending = VecDeque::from([(root, root_info)]); + while let Some((pid, info)) = pending.pop_front() { cpu_units = cpu_units .saturating_add(info.ri_user_time) .saturating_add(info.ri_system_time) @@ -434,8 +431,7 @@ mod platform { // that seems to start before its parent (a reused PID). .filter(|(_, info)| { info.ri_proc_exit_abstime == 0 && info.ri_proc_start_abstime >= start - }) - .map(|(child, info)| (child, Some(info))), + }), ); } read.cpu_nanos = mach_to_nanos(cpu_units, timebase); From c846c7841c9150848356e6845badbedd027d8fae Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 00:36:54 -0700 Subject: [PATCH 006/159] fix: check each macOS snapshot child is still the listed process The macOS reader listed each child's PID and parent, then read the child's usage later by PID alone. A child that exited after the listing could have its PID reused by an unrelated process, whose usage was then counted in the runtime's snapshot. The listing now keeps each child's parent and start time, and the child's usage is counted only when the same parent and start time read back after the usage read. The host is read after the listing, so its listed children are known to be its own. The libproc calls sit behind a small trait so tests can script the reuse. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 340 +++++++++++++++----- 1 file changed, 264 insertions(+), 76 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index a1049aab1..043aa894c 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -305,7 +305,9 @@ mod platform { #[cfg(target_os = "macos")] mod platform { - //! `proc_pid_rusage` reads, from the root down, each parent before its children. + //! `libproc` reads. One pass lists every process's parent and start time, + //! then the tree is read from the root down, each parent before its + //! children. use std::collections::{BTreeMap, VecDeque}; use std::sync::OnceLock; @@ -314,19 +316,118 @@ mod platform { pub(super) const SUPPORTED: bool = true; - fn rusage(pid: u32) -> Option { - let pid = libc::c_int::try_from(pid).ok()?; - // SAFETY: rusage_info_v2 is plain data; zero is a valid initial value. - let mut info: libc::rusage_info_v2 = unsafe { std::mem::zeroed() }; - // SAFETY: `info` is a writable rusage_info_v2, the layout RUSAGE_INFO_V2 names. - let status = unsafe { - libc::proc_pid_rusage( - pid, - libc::RUSAGE_INFO_V2, - (&raw mut info).cast::(), + /// Why one process could not be read. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) enum ReadError { + /// The process exited, or is a zombie the call cannot describe. + Gone, + /// The process exists but this process may not read it. + Denied, + } + + /// A process's parent and start time, which together identify it. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) struct Parent { + pub ppid: u32, + /// `pbi_start_tvsec` and `pbi_start_tvusec`. + pub started: (u64, u64), + } + + /// The `proc_pid_rusage` fields a snapshot needs, CPU times in Mach units. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) struct Usage { + pub cpu_units: u64, + pub resident_bytes: u64, + /// `ri_proc_start_abstime`. + pub start: u64, + pub exited: bool, + } + + /// The process reads a tree read makes. Tests script them. + pub(super) trait Libproc { + /// Every PID, or `None` when the process table cannot be listed. + fn pids(&self) -> Option>; + fn parent(&self, pid: u32) -> Result; + fn usage(&self, pid: u32) -> Result; + } + + struct Live; + + impl Libproc for Live { + fn pids(&self) -> Option> { + let mut pids = vec![0 as libc::pid_t; 1024]; + let count = loop { + let bytes = libc::c_int::try_from(std::mem::size_of_val(pids.as_slice())).ok()?; + // SAFETY: the buffer is writable and sized in bytes as libproc requires. + let count = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) }; + let count = usize::try_from(count).ok().filter(|count| *count > 0)?; + if count < pids.len() { + break count; + } + pids.resize(pids.len() * 2, 0); + }; + Some( + pids[..count] + .iter() + .filter_map(|pid| u32::try_from(*pid).ok()) + .collect(), ) - }; - (status == 0).then_some(info) + } + + /// `PROC_PIDTBSDINFO` fails for a zombie, so a zombie reads as gone. + fn parent(&self, pid: u32) -> Result { + let pid = libc::pid_t::try_from(pid).map_err(|_| ReadError::Gone)?; + // SAFETY: proc_bsdinfo is plain data; zero is a valid initial value. + let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; + let size = libc::c_int::try_from(std::mem::size_of::()) + .map_err(|_| ReadError::Denied)?; + // SAFETY: `info` is writable and `size` bytes long, as PROC_PIDTBSDINFO needs. + let written = unsafe { + libc::proc_pidinfo(pid, libc::PROC_PIDTBSDINFO, 0, (&raw mut info).cast(), size) + }; + if written == size { + return Ok(Parent { + ppid: info.pbi_ppid, + started: (info.pbi_start_tvsec, info.pbi_start_tvusec), + }); + } + Err(last_read_error()) + } + + fn usage(&self, pid: u32) -> Result { + let pid = libc::c_int::try_from(pid).map_err(|_| ReadError::Gone)?; + // SAFETY: rusage_info_v2 is plain data; zero is a valid initial value. + let mut info: libc::rusage_info_v2 = unsafe { std::mem::zeroed() }; + // SAFETY: `info` is a writable rusage_info_v2, the layout RUSAGE_INFO_V2 names. + let status = unsafe { + libc::proc_pid_rusage( + pid, + libc::RUSAGE_INFO_V2, + (&raw mut info).cast::(), + ) + }; + if status != 0 { + return Err(last_read_error()); + } + Ok(Usage { + cpu_units: info + .ri_user_time + .saturating_add(info.ri_system_time) + .saturating_add(info.ri_child_user_time) + .saturating_add(info.ri_child_system_time), + resident_bytes: info.ri_resident_size, + start: info.ri_proc_start_abstime, + exited: info.ri_proc_exit_abstime != 0, + }) + } + } + + /// Classify the error of the libproc call that just failed. + fn last_read_error() -> ReadError { + match std::io::Error::last_os_error().raw_os_error() { + Some(libc::ESRCH) => ReadError::Gone, + _ => ReadError::Denied, + } } /// `mach_timebase_info_data_t`. @@ -362,82 +463,67 @@ mod platform { } pub(super) fn start_identity(pid: u32) -> Option { - rusage(pid).map(|info| info.ri_proc_start_abstime) - } - - /// Every live process's children. One `proc_listallpids` call and one - /// `proc_pidinfo` per process: `proc_listchildpids` scans the whole - /// process table on each call, which costs too much for a wide tree. - /// `PROC_PIDTBSDINFO` fails for a zombie, so a zombie is never listed. - /// Returns `None` when the process table cannot be listed. - fn children_by_parent() -> Option>> { - let mut pids = vec![0 as libc::pid_t; 1024]; - let count = loop { - let bytes = libc::c_int::try_from(std::mem::size_of_val(pids.as_slice())).ok()?; - // SAFETY: the buffer is writable and sized in bytes as libproc requires. - let count = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) }; - let count = usize::try_from(count).ok().filter(|count| *count > 0)?; - if count < pids.len() { - break count; - } - pids.resize(pids.len() * 2, 0); - }; - let mut children: BTreeMap> = BTreeMap::new(); - for &pid in &pids[..count] { - if let (Ok(pid), Some(ppid)) = (u32::try_from(pid), parent_pid(pid)) { - children.entry(ppid).or_default().push(pid); + Live.usage(pid).ok().map(|usage| usage.start) + } + + pub(super) fn read_tree(root: u32, identity: u64) -> Option { + read_tree_with(&Live, root, identity, timebase()?) + } + + /// Every live process's children, each with its parent record. One + /// `proc_listallpids` call and one `proc_pidinfo` per process: + /// `proc_listchildpids` scans the whole process table on each call, which + /// costs too much for a wide tree. + fn children_by_parent(procs: &impl Libproc) -> Option>> { + let mut children: BTreeMap> = BTreeMap::new(); + for pid in procs.pids()? { + if let Ok(parent) = procs.parent(pid) { + children.entry(parent.ppid).or_default().push((pid, parent)); } } Some(children) } - /// The parent of a live process, or `None` for a zombie or a missing process. - fn parent_pid(pid: libc::pid_t) -> Option { - // SAFETY: proc_bsdinfo is plain data; zero is a valid initial value. - let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; - let size = libc::c_int::try_from(std::mem::size_of::()).ok()?; - // SAFETY: `info` is writable and `size` bytes long, as PROC_PIDTBSDINFO needs. - let written = unsafe { - libc::proc_pidinfo(pid, libc::PROC_PIDTBSDINFO, 0, (&raw mut info).cast(), size) - }; - (written == size).then_some(info.pbi_ppid) - } - /// Read the tree under `root`, or `None` when the root has exited or its - /// PID now names a process with another start time than `identity`. A - /// child that seems to start before its parent reuses a PID and is left out. - pub(super) fn read_tree(root: u32, identity: u64) -> Option { - parent_pid(libc::pid_t::try_from(root).ok()?)?; - let root_info = rusage(root).filter(|info| info.ri_proc_start_abstime == identity)?; - let timebase = timebase()?; - let children = children_by_parent()?; + /// PID now names a process with another start time than `identity`. + pub(super) fn read_tree_with( + procs: &impl Libproc, + root: u32, + identity: u64, + timebase: (u64, u64), + ) -> Option { + let children = children_by_parent(procs)?; + // Read after the listing: a root that is still the same live process + // was alive during the whole listing, so its listed children are its own. + let root_usage = procs + .usage(root) + .ok() + .filter(|usage| usage.start == identity && !usage.exited)?; let mut read = TreeRead::default(); let mut cpu_units: u64 = 0; - let mut resident: u64 = 0; - let mut pending = VecDeque::from([(root, root_info)]); - while let Some((pid, info)) = pending.pop_front() { - cpu_units = cpu_units - .saturating_add(info.ri_user_time) - .saturating_add(info.ri_system_time) - .saturating_add(info.ri_child_user_time) - .saturating_add(info.ri_child_system_time); - resident = resident.saturating_add(info.ri_resident_size); + let mut pending = VecDeque::from([(root, root_usage)]); + while let Some((pid, usage)) = pending.pop_front() { + cpu_units = cpu_units.saturating_add(usage.cpu_units); + read.resident_bytes = read.resident_bytes.saturating_add(usage.resident_bytes); read.process_count += 1; - let start = info.ri_proc_start_abstime; - let own = children.get(&pid).into_iter().flatten(); - pending.extend( - own.filter_map(|child| rusage(*child).map(|info| (*child, info))) - // Skip a child that exited after it was listed, and one - // that seems to start before its parent (a reused PID). - .filter(|(_, info)| { - info.ri_proc_exit_abstime == 0 && info.ri_proc_start_abstime >= start - }), - ); + for &(child, listed) in children.get(&pid).into_iter().flatten() { + if let Some(usage) = listed_child_usage(procs, child, listed) { + pending.push_back((child, usage)); + } + } } read.cpu_nanos = mach_to_nanos(cpu_units, timebase); - read.resident_bytes = resident; Some(read) } + + /// The usage of `child`, or `None` when it exited or its PID no longer + /// names the listed process. A process that still has the listed parent + /// and start time after the usage read was the listed one throughout, so + /// the usage is its own. + fn listed_child_usage(procs: &impl Libproc, child: u32, listed: Parent) -> Option { + let usage = procs.usage(child).ok().filter(|usage| !usage.exited)?; + (procs.parent(child).ok()? == listed).then_some(usage) + } } #[cfg(not(any(target_os = "linux", target_os = "macos")))] @@ -558,6 +644,108 @@ mod tests { assert_eq!(platform::ticks_to_nanos(150, 100), 1_500_000_000); } + /// Scripted `libproc` answers for the macOS tree walk. + #[cfg(target_os = "macos")] + #[derive(Default)] + struct ScriptedLibproc { + pids: Option>, + /// Answers to `parent`, first the listing's, then the later checks'. + parents: std::collections::BTreeMap< + u32, + std::cell::RefCell>>, + >, + usages: std::collections::BTreeMap< + u32, + std::result::Result, + >, + } + + #[cfg(target_os = "macos")] + impl ScriptedLibproc { + fn process( + mut self, + pid: u32, + parents: Vec>, + usage: std::result::Result, + ) -> Self { + self.pids.get_or_insert_with(Vec::new).push(pid); + self.parents.insert(pid, std::cell::RefCell::new(parents)); + self.usages.insert(pid, usage); + self + } + } + + #[cfg(target_os = "macos")] + impl platform::Libproc for ScriptedLibproc { + fn pids(&self) -> Option> { + self.pids.clone() + } + + fn parent(&self, pid: u32) -> std::result::Result { + let mut answers = self.parents.get(&pid).expect("scripted pid").borrow_mut(); + if answers.len() > 1 { + answers.remove(0) + } else { + answers[0] + } + } + + fn usage(&self, pid: u32) -> std::result::Result { + self.usages[&pid] + } + } + + #[cfg(target_os = "macos")] + fn mac_parent(ppid: u32, started: u64) -> platform::Parent { + platform::Parent { + ppid, + started: (started, 0), + } + } + + #[cfg(target_os = "macos")] + fn mac_usage(start: u64, cpu_units: u64) -> platform::Usage { + platform::Usage { + cpu_units, + resident_bytes: 1000, + start, + exited: false, + } + } + + /// The host (PID 10) has one listed child (PID 11). After the listing the + /// child exits and an unrelated, newer process takes PID 11. + #[cfg(target_os = "macos")] + #[test] + fn macos_tree_read_skips_child_pid_reused_after_listing() { + let procs = ScriptedLibproc::default() + .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) + .process( + 11, + vec![Ok(mac_parent(10, 200)), Ok(mac_parent(1, 300))], + Ok(mac_usage(9, 1_000_000)), + ); + + let read = platform::read_tree_with(&procs, 10, 5, (1, 1)).expect("tree read"); + + assert_eq!(read.process_count, 1); + assert_eq!(read.cpu_nanos, 7); + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_tree_read_counts_child_that_is_still_the_listed_process() { + let procs = ScriptedLibproc::default() + .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) + .process(11, vec![Ok(mac_parent(10, 200))], Ok(mac_usage(9, 3))); + + let read = platform::read_tree_with(&procs, 10, 5, (1, 1)).expect("tree read"); + + assert_eq!(read.process_count, 2); + assert_eq!(read.cpu_nanos, 10); + assert_eq!(read.resident_bytes, 2000); + } + #[cfg(target_os = "macos")] #[test] fn macos_mach_units_convert_with_the_timebase() { From ba3404ae69804ee5c89b3891d8dbf42d4309bd80 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 00:41:35 -0700 Subject: [PATCH 007/159] fix: report a snapshot as unavailable when a runtime process cannot be read Both readers turned every failed read into "skip this process", so a child that had exited and a live child that could not be inspected gave the same successful, smaller snapshot. #267 asks for an inaccessible runtime to return ResourceObservationUnavailable and for unavailable measurements never to read as zero. Each read now reports whether the process is gone or unreadable. A gone or replaced child is still skipped. A live process in the tree that cannot be read, or a process table that cannot be listed in full, now returns ResourceObservationUnavailable with the reason. On macOS the listing falls back to the short BSD record for another user's process, so its parent is still known, and such a process inside the tree makes the snapshot unavailable. The Linux reads move behind the same kind of trait as macOS so tests can script denied, reaped and replaced processes. A new test covers a snapshot after an invocation timeout. The SDK guide, error docstrings and skill reference now name the new condition, and say that discovery reads other processes' parent IDs and start times while the snapshot exposes only its own fields. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/error.rs | 4 +- crates/fabric-core/src/resource_snapshot.rs | 536 +++++++++++++++--- crates/fabric-core/src/runtime.rs | 26 +- crates/fabric-python/src/lib.rs | 2 +- docs/sdk/python.mdx | 16 +- .../src/nemo_fabric/errors.py | 4 +- .../src/nemo_fabric/runtime.py | 4 +- .../references/results-and-errors.md | 2 +- 8 files changed, 505 insertions(+), 89 deletions(-) diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index ce4682c28..fb4075af0 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -278,8 +278,8 @@ pub enum FabricError { adapter_kind: AdapterKind, }, /// A runtime's resources cannot be read now: the runtime stopped, is - /// stopping, its adapter host process exited, or the process table cannot - /// be read. + /// stopping, its adapter host process exited, or a live process in its + /// tree or the process table cannot be read. #[error("resource observation is unavailable for runtime `{runtime_id}`: {reason}")] ResourceObservationUnavailable { /// Runtime handle id. diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 043aa894c..579381e15 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -13,8 +13,14 @@ //! and on macOS while an exited descendant is not yet waited for, because //! macOS cannot list an unreaped process. //! -//! Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. Neither -//! reads command lines, environment variables, or open files. +//! A live process in the tree that cannot be read, or a process table that +//! cannot be listed in full, makes the snapshot unavailable instead of +//! leaving the process out. +//! +//! Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. To find +//! the tree, the readers also read other processes' parent IDs and start +//! times. A snapshot exposes only its own fields: never command lines, +//! environment variables, open files, or processes outside the tree. use std::time::{SystemTime, UNIX_EPOCH}; @@ -26,6 +32,18 @@ use crate::error::{FabricError, Result}; const MILLIS_PER_DAY: i64 = 86_400_000; const NANOS_PER_SECOND: f64 = 1_000_000_000.0; +/// Reasons a tree read returns when the snapshot is unavailable. +#[cfg_attr( + not(any(target_os = "linux", target_os = "macos")), + allow(dead_code, reason = "only the Linux and macOS readers read a tree") +)] +mod reason { + pub(super) const HOST_EXITED: &str = "adapter host process has exited"; + pub(super) const PROCESS_UNREADABLE: &str = + "a live process in the runtime's process tree cannot be read"; + pub(super) const TABLE_UNREADABLE: &str = "the process table cannot be read in full"; +} + /// Timestamped resource usage of one runtime's local adapter host process tree. /// /// Memory, process count, and CPU time cover the adapter host process and its @@ -95,15 +113,14 @@ pub(crate) fn snapshot_local_host( } let host = host.ok_or_else(|| unavailable(runtime_id, "runtime is not active in this process"))?; - let read = host - .identity - .and_then(|identity| platform::read_tree(host.pid, identity)) - .ok_or_else(|| { - unavailable( - runtime_id, - "adapter host process has exited or the process table cannot be read", - ) - })?; + let identity = host.identity.ok_or_else(|| { + unavailable( + runtime_id, + "adapter host process could not be read when the runtime started", + ) + })?; + let read = platform::read_tree(host.pid, identity) + .map_err(|reason| unavailable(runtime_id, reason))?; Ok(RuntimeResourceSnapshot { runtime_id: runtime_id.to_string(), started_at: host.started_at.clone(), @@ -187,10 +204,19 @@ mod platform { use std::collections::{BTreeMap, VecDeque}; use std::sync::OnceLock; - use super::TreeRead; + use super::{TreeRead, reason}; pub(super) const SUPPORTED: bool = true; + /// Why one process could not be read. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) enum ReadError { + /// The process exited. + Gone, + /// The process exists but cannot be read, or its record cannot be parsed. + Unreadable, + } + /// The fields of `/proc//stat` a snapshot needs. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct Stat { @@ -222,12 +248,46 @@ mod platform { }) } - fn read_stat(pid: u32) -> Option { - parse_stat(&std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?) + /// The `/proc` reads a tree read makes. Tests script them. + pub(super) trait Procfs { + /// Every PID in `/proc`. + fn pids(&self) -> Result, ReadError>; + fn stat(&self, pid: u32) -> Result; + } + + struct Live; + + impl Procfs for Live { + fn pids(&self) -> Result, ReadError> { + let mut pids = Vec::new(); + for entry in std::fs::read_dir("/proc").map_err(read_error)? { + let name = entry.map_err(read_error)?.file_name(); + if let Some(pid) = name.to_str().and_then(|name| name.parse().ok()) { + pids.push(pid); + } + } + Ok(pids) + } + + fn stat(&self, pid: u32) -> Result { + let text = std::fs::read_to_string(format!("/proc/{pid}/stat")).map_err(read_error)?; + parse_stat(&text).ok_or(ReadError::Unreadable) + } + } + + /// `ENOENT` and `ESRCH` mean the process exited; anything else means it + /// cannot be read. + fn read_error(error: std::io::Error) -> ReadError { + if error.kind() == std::io::ErrorKind::NotFound || error.raw_os_error() == Some(libc::ESRCH) + { + ReadError::Gone + } else { + ReadError::Unreadable + } } pub(super) fn start_identity(pid: u32) -> Option { - read_stat(pid).map(|stat| stat.start) + Live.stat(pid).ok().map(|stat| stat.start) } fn sysconf(name: libc::c_int) -> u64 { @@ -245,43 +305,54 @@ mod platform { *PAGE.get_or_init(|| sysconf(libc::_SC_PAGESIZE)) } - /// Every process's children, each with its start time, or `None` when - /// `/proc` cannot be listed. - fn children_by_parent() -> Option>> { - let mut children: BTreeMap> = BTreeMap::new(); - for entry in std::fs::read_dir("/proc").ok()?.flatten() { - let Some(pid) = entry - .file_name() - .to_str() - .and_then(|name| name.parse().ok()) - else { - continue; - }; - if let Some(stat) = read_stat(pid) { - children + pub(super) fn read_tree(root: u32, identity: u64) -> Result { + read_tree_with(&Live, root, identity, clock_ticks_per_second(), page_size()) + } + + /// Child PIDs and start times by parent PID. + type ChildrenByParent = BTreeMap>; + + /// Every process's children, each with its start time. A process that + /// exited during the pass is left out; one that cannot be read leaves the + /// listing incomplete. + fn children_by_parent(procfs: &impl Procfs) -> Result { + let pids = procfs.pids().map_err(|_| reason::TABLE_UNREADABLE)?; + let mut children = ChildrenByParent::new(); + for pid in pids { + match procfs.stat(pid) { + Ok(stat) => children .entry(stat.ppid) .or_default() - .push((pid, stat.start)); + .push((pid, stat.start)), + Err(ReadError::Gone) => {} + Err(ReadError::Unreadable) => return Err(reason::TABLE_UNREADABLE), } } - Some(children) + Ok(children) } - /// Read the tree under `root`, or `None` when the root has exited or its - /// PID now names a process with another start time than `identity`. - pub(super) fn read_tree(root: u32, identity: u64) -> Option { - let children = children_by_parent()?; + /// Read the tree under `root`. Fails when the root exited or its PID now + /// names a process with another start time than `identity`, or when a + /// live process in the tree cannot be read. + pub(super) fn read_tree_with( + procfs: &impl Procfs, + root: u32, + identity: u64, + ticks_per_second: u64, + page_size: u64, + ) -> Result { + let children = children_by_parent(procfs)?; let mut read = TreeRead::default(); let mut cpu_ticks: u64 = 0; let mut rss_pages: u64 = 0; let mut pending = VecDeque::from([(root, identity)]); while let Some((pid, expected_start)) = pending.pop_front() { - let stat = read_stat(pid).filter(|stat| stat.start == expected_start); - if pid == root && stat.is_none_or(|stat| stat.zombie) { - return None; - } - let Some(stat) = stat else { - continue; + let stat = match procfs.stat(pid) { + Ok(stat) if stat.start == expected_start && !(pid == root && stat.zombie) => stat, + // An exited child, or one whose PID now names another process. + Ok(_) | Err(ReadError::Gone) if pid != root => continue, + Ok(_) | Err(ReadError::Gone) => return Err(reason::HOST_EXITED), + Err(ReadError::Unreadable) => return Err(reason::PROCESS_UNREADABLE), }; cpu_ticks = cpu_ticks.saturating_add(stat.cpu_ticks); if !stat.zombie { @@ -292,9 +363,9 @@ mod platform { let own = children.get(&pid).into_iter().flatten(); pending.extend(own.filter(|(_, start)| *start >= stat.start).copied()); } - read.cpu_nanos = ticks_to_nanos(cpu_ticks, clock_ticks_per_second()); - read.resident_bytes = rss_pages.saturating_mul(page_size()); - Some(read) + read.cpu_nanos = ticks_to_nanos(cpu_ticks, ticks_per_second); + read.resident_bytes = rss_pages.saturating_mul(page_size); + Ok(read) } pub(super) fn ticks_to_nanos(ticks: u64, ticks_per_second: u64) -> u64 { @@ -312,7 +383,7 @@ mod platform { use std::collections::{BTreeMap, VecDeque}; use std::sync::OnceLock; - use super::TreeRead; + use super::{TreeRead, reason}; pub(super) const SUPPORTED: bool = true; @@ -329,10 +400,23 @@ mod platform { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct Parent { pub ppid: u32, - /// `pbi_start_tvsec` and `pbi_start_tvusec`. - pub started: (u64, u64), + /// `pbi_start_tvsec` and `pbi_start_tvusec`, or `None` for another + /// user's process, whose parent only the short record shows. + pub started: Option<(u64, u64)>, + } + + /// `struct proc_bsdshortinfo` from ``, which the `libc` + /// crate version in use does not bind: PID, parent PID, then 56 bytes of + /// group, status, name, flags, and user and group IDs that the reader ignores. + #[repr(C)] + struct ProcBsdShortInfo { + pid: u32, + ppid: u32, + rest: [u32; 14], } + const PROC_PIDT_SHORTBSDINFO: libc::c_int = 13; + /// The `proc_pid_rusage` fields a snapshot needs, CPU times in Mach units. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct Usage { @@ -375,6 +459,8 @@ mod platform { } /// `PROC_PIDTBSDINFO` fails for a zombie, so a zombie reads as gone. + /// It is denied for another user's process, whose parent the short + /// record still shows. fn parent(&self, pid: u32) -> Result { let pid = libc::pid_t::try_from(pid).map_err(|_| ReadError::Gone)?; // SAFETY: proc_bsdinfo is plain data; zero is a valid initial value. @@ -388,10 +474,13 @@ mod platform { if written == size { return Ok(Parent { ppid: info.pbi_ppid, - started: (info.pbi_start_tvsec, info.pbi_start_tvusec), + started: Some((info.pbi_start_tvsec, info.pbi_start_tvusec)), }); } - Err(last_read_error()) + match last_read_error() { + ReadError::Denied => short_parent(pid), + gone => Err(gone), + } } fn usage(&self, pid: u32) -> Result { @@ -422,6 +511,27 @@ mod platform { } } + fn short_parent(pid: libc::pid_t) -> Result { + let mut info = ProcBsdShortInfo { + pid: 0, + ppid: 0, + rest: [0; 14], + }; + let size = libc::c_int::try_from(std::mem::size_of::()) + .map_err(|_| ReadError::Denied)?; + // SAFETY: `info` is writable and `size` bytes long, as PROC_PIDT_SHORTBSDINFO needs. + let written = unsafe { + libc::proc_pidinfo(pid, PROC_PIDT_SHORTBSDINFO, 0, (&raw mut info).cast(), size) + }; + if written == size { + return Ok(Parent { + ppid: info.ppid, + started: None, + }); + } + Err(last_read_error()) + } + /// Classify the error of the libproc call that just failed. fn last_read_error() -> ReadError { match std::io::Error::last_os_error().raw_os_error() { @@ -466,39 +576,48 @@ mod platform { Live.usage(pid).ok().map(|usage| usage.start) } - pub(super) fn read_tree(root: u32, identity: u64) -> Option { - read_tree_with(&Live, root, identity, timebase()?) + pub(super) fn read_tree(root: u32, identity: u64) -> Result { + let timebase = timebase().ok_or(reason::PROCESS_UNREADABLE)?; + read_tree_with(&Live, root, identity, timebase) } /// Every live process's children, each with its parent record. One /// `proc_listallpids` call and one `proc_pidinfo` per process: /// `proc_listchildpids` scans the whole process table on each call, which - /// costs too much for a wide tree. - fn children_by_parent(procs: &impl Libproc) -> Option>> { + /// costs too much for a wide tree. A process that exited during the pass + /// is left out; one that cannot be read leaves the listing incomplete. + fn children_by_parent( + procs: &impl Libproc, + ) -> Result>, &'static str> { + let pids = procs.pids().ok_or(reason::TABLE_UNREADABLE)?; let mut children: BTreeMap> = BTreeMap::new(); - for pid in procs.pids()? { - if let Ok(parent) = procs.parent(pid) { - children.entry(parent.ppid).or_default().push((pid, parent)); + for pid in pids { + match procs.parent(pid) { + Ok(parent) => children.entry(parent.ppid).or_default().push((pid, parent)), + Err(ReadError::Gone) => {} + Err(ReadError::Denied) => return Err(reason::TABLE_UNREADABLE), } } - Some(children) + Ok(children) } - /// Read the tree under `root`, or `None` when the root has exited or its - /// PID now names a process with another start time than `identity`. + /// Read the tree under `root`. Fails when the root exited or its PID now + /// names a process with another start time than `identity`, or when a + /// live process in the tree cannot be read. pub(super) fn read_tree_with( procs: &impl Libproc, root: u32, identity: u64, timebase: (u64, u64), - ) -> Option { + ) -> Result { let children = children_by_parent(procs)?; // Read after the listing: a root that is still the same live process // was alive during the whole listing, so its listed children are its own. - let root_usage = procs - .usage(root) - .ok() - .filter(|usage| usage.start == identity && !usage.exited)?; + let root_usage = match procs.usage(root) { + Ok(usage) if usage.start == identity && !usage.exited => usage, + Ok(_) | Err(ReadError::Gone) => return Err(reason::HOST_EXITED), + Err(ReadError::Denied) => return Err(reason::PROCESS_UNREADABLE), + }; let mut read = TreeRead::default(); let mut cpu_units: u64 = 0; let mut pending = VecDeque::from([(root, root_usage)]); @@ -507,22 +626,40 @@ mod platform { read.resident_bytes = read.resident_bytes.saturating_add(usage.resident_bytes); read.process_count += 1; for &(child, listed) in children.get(&pid).into_iter().flatten() { - if let Some(usage) = listed_child_usage(procs, child, listed) { + if let Some(usage) = listed_child_usage(procs, child, listed)? { pending.push_back((child, usage)); } } } read.cpu_nanos = mach_to_nanos(cpu_units, timebase); - Some(read) + Ok(read) } /// The usage of `child`, or `None` when it exited or its PID no longer /// names the listed process. A process that still has the listed parent /// and start time after the usage read was the listed one throughout, so - /// the usage is its own. - fn listed_child_usage(procs: &impl Libproc, child: u32, listed: Parent) -> Option { - let usage = procs.usage(child).ok().filter(|usage| !usage.exited)?; - (procs.parent(child).ok()? == listed).then_some(usage) + /// the usage is its own. Fails when the child is live but cannot be read. + fn listed_child_usage( + procs: &impl Libproc, + child: u32, + listed: Parent, + ) -> Result, &'static str> { + if listed.started.is_none() { + return Err(reason::PROCESS_UNREADABLE); + } + let usage = match procs.usage(child) { + Ok(usage) if !usage.exited => usage, + Ok(_) | Err(ReadError::Gone) => return Ok(None), + Err(ReadError::Denied) => return Err(reason::PROCESS_UNREADABLE), + }; + match procs.parent(child) { + Ok(now) if now == listed => Ok(Some(usage)), + Ok(now) if now.ppid == listed.ppid && now.started.is_none() => { + Err(reason::PROCESS_UNREADABLE) + } + Ok(_) | Err(ReadError::Gone) => Ok(None), + Err(ReadError::Denied) => Err(reason::PROCESS_UNREADABLE), + } } } @@ -536,8 +673,8 @@ mod platform { None } - pub(super) fn read_tree(_root: u32, _identity: u64) -> Option { - None + pub(super) fn read_tree(_root: u32, _identity: u64) -> Result { + Err("no process reader for this operating system") } } @@ -699,7 +836,7 @@ mod tests { fn mac_parent(ppid: u32, started: u64) -> platform::Parent { platform::Parent { ppid, - started: (started, 0), + started: Some((started, 0)), } } @@ -746,6 +883,259 @@ mod tests { assert_eq!(read.resident_bytes, 2000); } + #[cfg(target_os = "macos")] + fn mac_tree_with_child( + child_parents: Vec>, + child_usage: std::result::Result, + ) -> std::result::Result { + let procs = ScriptedLibproc::default() + .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) + .process(11, child_parents, child_usage); + platform::read_tree_with(&procs, 10, 5, (1, 1)) + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_tree_read_with_unreadable_live_child_is_unavailable() { + let denied_usage = mac_tree_with_child( + vec![Ok(mac_parent(10, 200))], + Err(platform::ReadError::Denied), + ); + let other_user = mac_tree_with_child( + vec![Ok(platform::Parent { + ppid: 10, + started: None, + })], + Ok(mac_usage(9, 3)), + ); + let denied_recheck = mac_tree_with_child( + vec![Ok(mac_parent(10, 200)), Err(platform::ReadError::Denied)], + Ok(mac_usage(9, 3)), + ); + + assert_eq!(denied_usage.unwrap_err(), reason::PROCESS_UNREADABLE); + assert_eq!(other_user.unwrap_err(), reason::PROCESS_UNREADABLE); + assert_eq!(denied_recheck.unwrap_err(), reason::PROCESS_UNREADABLE); + } + + /// The child is listed, then exits before its usage read or before the + /// identity check that follows it. + #[cfg(target_os = "macos")] + #[test] + fn macos_tree_read_skips_child_reaped_between_reads() { + let before_usage = mac_tree_with_child( + vec![Ok(mac_parent(10, 200))], + Err(platform::ReadError::Gone), + ) + .expect("tree read"); + let before_check = mac_tree_with_child( + vec![Ok(mac_parent(10, 200)), Err(platform::ReadError::Gone)], + Ok(mac_usage(9, 3)), + ) + .expect("tree read"); + let exited = mac_tree_with_child( + vec![Ok(mac_parent(10, 200))], + Ok(platform::Usage { + exited: true, + ..mac_usage(9, 3) + }), + ) + .expect("tree read"); + + for read in [before_usage, before_check, exited] { + assert_eq!(read.process_count, 1); + assert_eq!(read.cpu_nanos, 7); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_tree_read_with_incomplete_process_table_is_unavailable() { + let unlisted = ScriptedLibproc::default(); + let unreadable_other = ScriptedLibproc::default() + .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) + .process( + 50, + vec![Err(platform::ReadError::Denied)], + Err(platform::ReadError::Denied), + ); + + assert_eq!( + platform::read_tree_with(&unlisted, 10, 5, (1, 1)).unwrap_err(), + reason::TABLE_UNREADABLE + ); + assert_eq!( + platform::read_tree_with(&unreadable_other, 10, 5, (1, 1)).unwrap_err(), + reason::TABLE_UNREADABLE + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_tree_read_of_exited_or_unreadable_host_is_unavailable() { + let host = |usage| { + let procs = ScriptedLibproc::default().process(10, vec![Ok(mac_parent(1, 100))], usage); + platform::read_tree_with(&procs, 10, 5, (1, 1)).unwrap_err() + }; + + assert_eq!(host(Err(platform::ReadError::Gone)), reason::HOST_EXITED); + assert_eq!(host(Ok(mac_usage(6, 7))), reason::HOST_EXITED); + assert_eq!( + host(Ok(platform::Usage { + exited: true, + ..mac_usage(5, 7) + })), + reason::HOST_EXITED + ); + assert_eq!( + host(Err(platform::ReadError::Denied)), + reason::PROCESS_UNREADABLE + ); + } + + /// Scripted `/proc` answers for the Linux tree walk. + #[cfg(target_os = "linux")] + #[derive(Default)] + struct ScriptedProcfs { + pids: Vec, + unlisted: bool, + /// Answers to `stat`, first the listing's, then the walk's. + stats: std::collections::BTreeMap< + u32, + std::cell::RefCell>>, + >, + } + + #[cfg(target_os = "linux")] + impl ScriptedProcfs { + fn process( + mut self, + pid: u32, + stats: Vec>, + ) -> Self { + self.pids.push(pid); + self.stats.insert(pid, std::cell::RefCell::new(stats)); + self + } + } + + #[cfg(target_os = "linux")] + impl platform::Procfs for ScriptedProcfs { + fn pids(&self) -> std::result::Result, platform::ReadError> { + if self.unlisted { + return Err(platform::ReadError::Unreadable); + } + Ok(self.pids.clone()) + } + + fn stat(&self, pid: u32) -> std::result::Result { + let mut answers = self.stats.get(&pid).expect("scripted pid").borrow_mut(); + if answers.len() > 1 { + answers.remove(0) + } else { + answers[0] + } + } + } + + #[cfg(target_os = "linux")] + fn linux_stat(ppid: u32, start: u64, cpu_ticks: u64) -> platform::Stat { + platform::Stat { + zombie: false, + ppid, + cpu_ticks, + start, + rss_pages: 1, + } + } + + /// The host is PID 10 started at tick 5; its child is PID 11. + #[cfg(target_os = "linux")] + fn linux_tree_with_child( + child: Vec>, + ) -> std::result::Result { + let procfs = ScriptedProcfs::default() + .process(10, vec![Ok(linux_stat(1, 5, 7))]) + .process(11, child); + platform::read_tree_with(&procfs, 10, 5, 1, 1) + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_with_unreadable_live_child_is_unavailable() { + let read = linux_tree_with_child(vec![ + Ok(linux_stat(10, 9, 3)), + Err(platform::ReadError::Unreadable), + ]); + + assert_eq!(read.unwrap_err(), reason::PROCESS_UNREADABLE); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_skips_child_reaped_or_replaced_between_reads() { + let reaped = linux_tree_with_child(vec![ + Ok(linux_stat(10, 9, 3)), + Err(platform::ReadError::Gone), + ]) + .expect("tree read"); + let replaced = + linux_tree_with_child(vec![Ok(linux_stat(10, 9, 3)), Ok(linux_stat(1, 12, 900))]) + .expect("tree read"); + let counted = linux_tree_with_child(vec![Ok(linux_stat(10, 9, 3))]).expect("tree read"); + + for read in [reaped, replaced] { + assert_eq!(read.process_count, 1); + assert_eq!(read.cpu_nanos, 7_000_000_000); + } + assert_eq!(counted.process_count, 2); + assert_eq!(counted.cpu_nanos, 10_000_000_000); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_with_incomplete_process_table_is_unavailable() { + let unlisted = ScriptedProcfs { + unlisted: true, + ..ScriptedProcfs::default() + }; + let unreadable_other = ScriptedProcfs::default() + .process(10, vec![Ok(linux_stat(1, 5, 7))]) + .process(50, vec![Err(platform::ReadError::Unreadable)]); + + assert_eq!( + platform::read_tree_with(&unlisted, 10, 5, 1, 1).unwrap_err(), + reason::TABLE_UNREADABLE + ); + assert_eq!( + platform::read_tree_with(&unreadable_other, 10, 5, 1, 1).unwrap_err(), + reason::TABLE_UNREADABLE + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_of_exited_or_unreadable_host_is_unavailable() { + let host = |walk| { + let procfs = ScriptedProcfs::default().process(10, vec![Ok(linux_stat(1, 5, 7)), walk]); + platform::read_tree_with(&procfs, 10, 5, 1, 1).unwrap_err() + }; + + assert_eq!(host(Err(platform::ReadError::Gone)), reason::HOST_EXITED); + assert_eq!(host(Ok(linux_stat(1, 6, 7))), reason::HOST_EXITED); + assert_eq!( + host(Ok(platform::Stat { + zombie: true, + ..linux_stat(1, 5, 7) + })), + reason::HOST_EXITED + ); + assert_eq!( + host(Err(platform::ReadError::Unreadable)), + reason::PROCESS_UNREADABLE + ); + } + #[cfg(target_os = "macos")] #[test] fn macos_mach_units_convert_with_the_timebase() { diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 9630f9f6e..589cdbd78 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1177,7 +1177,8 @@ pub fn invoke_runtime( /// The snapshot covers the runtime's local adapter host process and its live /// descendants. It does not wait for a running invocation. It returns /// [`FabricError::ResourceObservationUnavailable`] when the runtime stopped, is -/// stopping, its host exited, or the process table cannot be read, and +/// stopping, its host exited, or a live process in its tree or the process +/// table cannot be read, and /// [`FabricError::ResourceObservationUnsupported`] when the environment provider /// or operating system has no resource reader. pub fn runtime_resource_snapshot( @@ -4881,6 +4882,29 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn runtime_resource_snapshot_after_invocation_timeout_is_unavailable() { + let (root, plan) = local_host_plan("invoke_timeout"); + let runtime = start_runtime(&plan).expect("start local host"); + runtime_resource_snapshot(&plan, &runtime).expect("snapshot before the timeout"); + + run_local_host_adapter_with_timeout( + &plan, + &runtime, + RunRequest::text("hang"), + Duration::from_millis(100), + ) + .expect_err("unresponsive host must time out"); + + assert_unavailable( + &runtime_resource_snapshot(&plan, &runtime), + &runtime.runtime_id, + ); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } + #[test] fn runtime_resource_snapshot_of_non_local_environment_is_unsupported() { let (root, plan) = local_host_plan("success"); diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 44587f33b..c068a184a 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -28,7 +28,7 @@ pyo3::create_exception!( _native, ResourceObservationUnavailableError, PyRuntimeError, - "Raised when a runtime stopped, is stopping, its adapter host exited, or the process table cannot be read." + "Raised when a runtime stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read." ); pyo3::create_exception!( _native, diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 6fbc919b8..999a90775 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -705,16 +705,18 @@ the runtime, plus its descendants. For a runtime started on a shared service, the service process is not included. A descendant that starts a new session or process group still counts while its parent chain leads to the host. A process whose parent exited is reparented outside the tree and no longer counts. The -snapshot reads `/proc` on Linux and `proc_pid_rusage` on macOS. It uses only -process IDs, parent IDs, start times, CPU times, and resident sizes, never reads -command lines or environment variables, and reports only the runtime's tree. -Descendants that NeMo Fabric is not permitted to inspect are left out. +snapshot reads `/proc` on Linux and `proc_pid_rusage` on macOS. To find the +tree, it reads the parent ID and start time of every process on the machine. +The snapshot exposes only the fields above, never command lines, environment +variables, credentials, or processes outside the runtime's tree. When NeMo Fabric cannot read the values, it raises an error instead of -returning zeros: +returning zeros or leaving a process out: - `ResourceObservationUnavailableError` when the runtime stopped, is stopping, - its adapter host process exited, or the process table cannot be read. + its adapter host process exited, or a live process in its tree or the + process table cannot be read, for example a macOS descendant that runs as + another user. - `ResourceObservationUnsupportedError` when the runtime's environment provider or operating system has no resource reader. Windows raises this error. @@ -1115,7 +1117,7 @@ All public SDK errors inherit from `FabricError`. | `FabricRuntimeError` | Runtime startup, invocation, or shutdown failed before a normalized result could be returned. | | `FabricStateError` | Invalid runtime state transition, such as invoking after stop or starting overlapping invocations. | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | -| `ResourceObservationUnavailableError` | `Runtime.resource_snapshot()` cannot read the runtime because it stopped, is stopping, its adapter host exited, or the process table cannot be read. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnavailableError` | `Runtime.resource_snapshot()` cannot read the runtime because it stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read. Subclass of `FabricRuntimeError`. | | `ResourceObservationUnsupportedError` | The runtime's environment provider or operating system has no resource reader. Subclass of `FabricRuntimeError`. | Consumers own job-level retries and rollout-level failure policy. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index 5edb2bbf4..6f6b8e0c9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -84,8 +84,8 @@ class FabricNativeUnavailableError(FabricRuntimeError): class ResourceObservationUnavailableError(FabricRuntimeError): """Runtime resources cannot be read now. - The runtime stopped, is stopping, its adapter host process exited, or the - process table cannot be read. + The runtime stopped, is stopping, its adapter host process exited, or a + live process in its tree or the process table cannot be read. """ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 28be1b623..a421eb3ff 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -749,8 +749,8 @@ async def resource_snapshot(self) -> RuntimeResourceSnapshot: Raises: ResourceObservationUnavailableError: If the runtime stopped, is - stopping, its adapter host process exited, or the process - table cannot be read. + stopping, its adapter host process exited, or a live process + in its tree or the process table cannot be read. ResourceObservationUnsupportedError: If the runtime's environment provider or operating system has no resource reader. FabricNativeUnavailableError: If the native extension is missing. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 094b75614..2500fa488 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -83,7 +83,7 @@ Refer to the [errors reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/ | `FabricRuntimeError` | Startup, invocation, or shutdown failed before a normalized result. | | `FabricStateError` | Invalid runtime state transition (invoking after stop, overlapping invocations). | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | -| `ResourceObservationUnavailableError` | A resource snapshot cannot be read because the runtime stopped, is stopping, its adapter host exited, or the process table cannot be read. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnavailableError` | A resource snapshot cannot be read because the runtime stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read. Subclass of `FabricRuntimeError`. | | `ResourceObservationUnsupportedError` | The runtime's environment provider or operating system has no resource reader. Subclass of `FabricRuntimeError`. | ## Cleanup And Resilience From 5cf94dc011d84b61c5cb3792a281d69551b4e9da Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 00:50:42 -0700 Subject: [PATCH 008/159] perf: read only the runtime's own processes on Linux Each local invocation reads a snapshot while it holds its host's lock. The Linux reader parsed /proc//stat for every process on the machine, so the cost grew with the machine's process count. In a release build in Docker (rust:1.94, aarch64), the median over 60 reads was 0.66 ms with 108 processes, 5.8 to 7.1 ms with 1,008, 35 to 38 ms with 5,008 and 72 to 74 ms with 10,008. The reader now lists each process's children from /proc//task//children and reads only the runtime's tree. The same benchmark reads 0.027 to 0.028 ms at every process count. A child counts only when its stat, read after the listing, still names the expected parent. The kernel can skip a live child when a sibling is reaped during the read, so the children list is read until two reads agree, and the snapshot is unavailable after four reads that differ. Kernels without children files keep the full scan. Also name the macOS children map type, which clippy flagged. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 214 +++++++++++++++++--- docs/sdk/python.mdx | 7 +- 2 files changed, 189 insertions(+), 32 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 579381e15..c6d004828 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -18,9 +18,10 @@ //! leaving the process out. //! //! Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. To find -//! the tree, the readers also read other processes' parent IDs and start -//! times. A snapshot exposes only its own fields: never command lines, -//! environment variables, open files, or processes outside the tree. +//! the tree, the macOS reader, and the Linux reader on a kernel without +//! `/proc//task//children`, also read other processes' parent IDs. +//! A snapshot exposes only its own fields: never command lines, environment +//! variables, open files, or processes outside the tree. use std::time::{SystemTime, UNIX_EPOCH}; @@ -198,8 +199,10 @@ fn civil_from_days(days: i64) -> (i64, i64, i64) { #[cfg(target_os = "linux")] mod platform { - //! `/proc` reads: one pass lists every process's parent, then the tree is - //! read from the root down, each parent before its children. + //! `/proc` reads, from the root down, each parent before its children. + //! Each process's children come from `/proc//task//children`, + //! so a read visits only the runtime's own processes. A kernel built + //! without those files falls back to one pass over every process's parent. use std::collections::{BTreeMap, VecDeque}; use std::sync::OnceLock; @@ -248,8 +251,15 @@ mod platform { }) } + /// Times a process's children are read while waiting for two reads that agree. + const MAX_CHILDREN_READS: usize = 4; + /// The `/proc` reads a tree read makes. Tests script them. pub(super) trait Procfs { + /// Whether the kernel provides `/proc//task//children`. + fn has_children_files(&self) -> bool; + /// The children of every thread of `pid`, sorted. + fn children(&self, pid: u32) -> Result, ReadError>; /// Every PID in `/proc`. fn pids(&self) -> Result, ReadError>; fn stat(&self, pid: u32) -> Result; @@ -258,6 +268,18 @@ mod platform { struct Live; impl Procfs for Live { + fn has_children_files(&self) -> bool { + static PRESENT: OnceLock = OnceLock::new(); + *PRESENT.get_or_init(|| { + let pid = std::process::id(); + std::path::Path::new(&format!("/proc/{pid}/task/{pid}/children")).exists() + }) + } + + fn children(&self, pid: u32) -> Result, ReadError> { + read_until_stable(|| children_once(pid)) + } + fn pids(&self) -> Result, ReadError> { let mut pids = Vec::new(); for entry in std::fs::read_dir("/proc").map_err(read_error)? { @@ -275,6 +297,43 @@ mod platform { } } + /// One read of the children files of every thread of `pid`. A thread that + /// exited during the read is skipped; its children move to another thread. + fn children_once(pid: u32) -> Result, ReadError> { + let mut children = Vec::new(); + for entry in std::fs::read_dir(format!("/proc/{pid}/task")).map_err(read_error)? { + let tid = entry.map_err(read_error)?.file_name(); + let path = format!("/proc/{pid}/task/{}/children", tid.to_string_lossy()); + let text = match std::fs::read_to_string(path).map_err(read_error) { + Ok(text) => text, + Err(ReadError::Gone) => continue, + Err(unreadable) => return Err(unreadable), + }; + for child in text.split_whitespace() { + children.push(child.parse().map_err(|_| ReadError::Unreadable)?); + } + } + children.sort_unstable(); + Ok(children) + } + + /// Read a process's children until two reads in a row agree. The kernel + /// can skip a live child when a sibling is reaped during the read, and a + /// thread's children move when it exits; either changes the next read. + pub(super) fn read_until_stable( + mut read: impl FnMut() -> Result, ReadError>, + ) -> Result, ReadError> { + let mut last = read()?; + for _ in 1..MAX_CHILDREN_READS { + let next = read()?; + if next == last { + return Ok(next); + } + last = next; + } + Err(ReadError::Unreadable) + } + /// `ENOENT` and `ESRCH` mean the process exited; anything else means it /// cannot be read. fn read_error(error: std::io::Error) -> ReadError { @@ -309,21 +368,15 @@ mod platform { read_tree_with(&Live, root, identity, clock_ticks_per_second(), page_size()) } - /// Child PIDs and start times by parent PID. - type ChildrenByParent = BTreeMap>; - - /// Every process's children, each with its start time. A process that - /// exited during the pass is left out; one that cannot be read leaves the - /// listing incomplete. - fn children_by_parent(procfs: &impl Procfs) -> Result { + /// Every process's children, for a kernel without children files. A + /// process that exited during the pass is left out; one that cannot be + /// read leaves the listing incomplete. + fn children_by_parent(procfs: &impl Procfs) -> Result>, &'static str> { let pids = procfs.pids().map_err(|_| reason::TABLE_UNREADABLE)?; - let mut children = ChildrenByParent::new(); + let mut children: BTreeMap> = BTreeMap::new(); for pid in pids { match procfs.stat(pid) { - Ok(stat) => children - .entry(stat.ppid) - .or_default() - .push((pid, stat.start)), + Ok(stat) => children.entry(stat.ppid).or_default().push(pid), Err(ReadError::Gone) => {} Err(ReadError::Unreadable) => return Err(reason::TABLE_UNREADABLE), } @@ -341,16 +394,32 @@ mod platform { ticks_per_second: u64, page_size: u64, ) -> Result { - let children = children_by_parent(procfs)?; + let listing = if procfs.has_children_files() { + None + } else { + Some(children_by_parent(procfs)?) + }; let mut read = TreeRead::default(); let mut cpu_ticks: u64 = 0; let mut rss_pages: u64 = 0; - let mut pending = VecDeque::from([(root, identity)]); - while let Some((pid, expected_start)) = pending.pop_front() { - let stat = match procfs.stat(pid) { - Ok(stat) if stat.start == expected_start && !(pid == root && stat.zombie) => stat, + // Each entry is a PID and the parent it was listed under (none for the root). + let mut pending = VecDeque::from([(root, None)]); + while let Some((pid, parent)) = pending.pop_front() { + // List the children before the stat read: a stat that still names + // the expected process shows the listed children were its own. + let children = match &listing { + Some(listing) => Ok(listing.get(&pid).cloned().unwrap_or_default()), + None => procfs.children(pid), + }; + let found = children.and_then(|children| Ok((procfs.stat(pid)?, children))); + let expected = |stat: &Stat| match parent { + None => stat.start == identity && !stat.zombie, + Some(parent) => stat.ppid == parent, + }; + let (stat, children) = match found { + Ok((stat, children)) if expected(&stat) => (stat, children), // An exited child, or one whose PID now names another process. - Ok(_) | Err(ReadError::Gone) if pid != root => continue, + Ok(_) | Err(ReadError::Gone) if parent.is_some() => continue, Ok(_) | Err(ReadError::Gone) => return Err(reason::HOST_EXITED), Err(ReadError::Unreadable) => return Err(reason::PROCESS_UNREADABLE), }; @@ -359,9 +428,7 @@ mod platform { rss_pages = rss_pages.saturating_add(stat.rss_pages); read.process_count += 1; } - // A child never starts before its parent; one that seems to reuses a PID. - let own = children.get(&pid).into_iter().flatten(); - pending.extend(own.filter(|(_, start)| *start >= stat.start).copied()); + pending.extend(children.into_iter().map(|child| (child, Some(pid)))); } read.cpu_nanos = ticks_to_nanos(cpu_ticks, ticks_per_second); read.resident_bytes = rss_pages.saturating_mul(page_size); @@ -581,16 +648,17 @@ mod platform { read_tree_with(&Live, root, identity, timebase) } + /// Child PIDs, each with its parent record, by parent PID. + type ChildrenByParent = BTreeMap>; + /// Every live process's children, each with its parent record. One /// `proc_listallpids` call and one `proc_pidinfo` per process: /// `proc_listchildpids` scans the whole process table on each call, which /// costs too much for a wide tree. A process that exited during the pass /// is left out; one that cannot be read leaves the listing incomplete. - fn children_by_parent( - procs: &impl Libproc, - ) -> Result>, &'static str> { + fn children_by_parent(procs: &impl Libproc) -> Result { let pids = procs.pids().ok_or(reason::TABLE_UNREADABLE)?; - let mut children: BTreeMap> = BTreeMap::new(); + let mut children = ChildrenByParent::new(); for pid in pids { match procs.parent(pid) { Ok(parent) => children.entry(parent.ppid).or_default().push((pid, parent)), @@ -997,6 +1065,11 @@ mod tests { #[cfg(target_os = "linux")] #[derive(Default)] struct ScriptedProcfs { + /// Children by parent from children files, or `None` for a kernel + /// without them. + children: Option< + std::collections::BTreeMap, platform::ReadError>>, + >, pids: Vec, unlisted: bool, /// Answers to `stat`, first the listing's, then the walk's. @@ -1021,6 +1094,15 @@ mod tests { #[cfg(target_os = "linux")] impl platform::Procfs for ScriptedProcfs { + fn has_children_files(&self) -> bool { + self.children.is_some() + } + + fn children(&self, pid: u32) -> std::result::Result, platform::ReadError> { + let children = self.children.as_ref().expect("children files"); + children.get(&pid).cloned().unwrap_or(Ok(Vec::new())) + } + fn pids(&self) -> std::result::Result, platform::ReadError> { if self.unlisted { return Err(platform::ReadError::Unreadable); @@ -1136,6 +1218,78 @@ mod tests { ); } + /// A kernel with children files: the host is PID 10 started at tick 5 and + /// lists child PID 11. The process table is never listed. + #[cfg(target_os = "linux")] + fn linux_children_tree( + host_children: std::result::Result, platform::ReadError>, + child: std::result::Result, + ) -> std::result::Result { + let mut procfs = ScriptedProcfs { + children: Some(std::collections::BTreeMap::from([(10, host_children)])), + unlisted: true, + ..ScriptedProcfs::default() + } + .process(10, vec![Ok(linux_stat(1, 5, 7))]) + .process(11, vec![child]); + procfs.pids.clear(); + platform::read_tree_with(&procfs, 10, 5, 1, 1) + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_with_children_files_reads_only_the_tree() { + let read = linux_children_tree(Ok(vec![11]), Ok(linux_stat(10, 9, 3))).expect("tree read"); + + assert_eq!(read.process_count, 2); + assert_eq!(read.cpu_nanos, 10_000_000_000); + assert_eq!(read.resident_bytes, 2); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_with_children_files_skips_reaped_or_replaced_child() { + let reaped = + linux_children_tree(Ok(vec![11]), Err(platform::ReadError::Gone)).expect("tree read"); + let replaced = + linux_children_tree(Ok(vec![11]), Ok(linux_stat(1, 12, 900))).expect("tree read"); + + for read in [reaped, replaced] { + assert_eq!(read.process_count, 1); + assert_eq!(read.cpu_nanos, 7_000_000_000); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_tree_read_with_children_files_and_unreadable_process_is_unavailable() { + let child = linux_children_tree(Ok(vec![11]), Err(platform::ReadError::Unreadable)); + let host_children = linux_children_tree( + Err(platform::ReadError::Unreadable), + Ok(linux_stat(10, 9, 3)), + ); + let host_gone = + linux_children_tree(Err(platform::ReadError::Gone), Ok(linux_stat(10, 9, 3))); + + assert_eq!(child.unwrap_err(), reason::PROCESS_UNREADABLE); + assert_eq!(host_children.unwrap_err(), reason::PROCESS_UNREADABLE); + assert_eq!(host_gone.unwrap_err(), reason::HOST_EXITED); + } + + /// The second read finds a child the first skipped; the third agrees. + #[cfg(target_os = "linux")] + #[test] + fn linux_children_read_waits_for_two_reads_that_agree() { + let mut reads = vec![vec![11, 13], vec![11, 12, 13], vec![11, 12, 13]].into_iter(); + let mut changing = (0..).map(|n| vec![n]); + + let stable = platform::read_until_stable(|| Ok(reads.next().expect("read"))); + let unstable = platform::read_until_stable(|| Ok(changing.next().expect("read"))); + + assert_eq!(stable, Ok(vec![11, 12, 13])); + assert_eq!(unstable, Err(platform::ReadError::Unreadable)); + } + #[cfg(target_os = "macos")] #[test] fn macos_mach_units_convert_with_the_timebase() { diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 999a90775..ae9926b55 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -705,8 +705,11 @@ the runtime, plus its descendants. For a runtime started on a shared service, the service process is not included. A descendant that starts a new session or process group still counts while its parent chain leads to the host. A process whose parent exited is reparented outside the tree and no longer counts. The -snapshot reads `/proc` on Linux and `proc_pid_rusage` on macOS. To find the -tree, it reads the parent ID and start time of every process on the machine. +snapshot reads `/proc` on Linux and `proc_pid_rusage` on macOS. On Linux it +finds the tree from each process's `/proc//task//children` list, so it +reads only the runtime's own processes. On macOS, and on a Linux kernel without +those lists, it reads the parent ID of every process on the machine to find the +tree, and macOS also reads each process's start time to detect a reused PID. The snapshot exposes only the fields above, never command lines, environment variables, credentials, or processes outside the runtime's tree. From 9c54d0a619e5a3c9b62b5b1d3a098e46831ba23a Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 00:51:42 -0700 Subject: [PATCH 009/159] docs: regenerate API references for unavailable snapshot reads The FabricError, runtime_resource_snapshot, resource_snapshot module and Python error docstrings now name an unreadable live process as a cause of ResourceObservationUnavailable, and the module docs describe how each reader finds the tree. Regenerated with Rust 1.94.0 and the locked docs tools; only the changed sentences are applied. Signed-off-by: mnajafian-nv --- .../api/python-library-reference/nemo_fabric.errors.md | 2 +- .../api/python-library-reference/nemo_fabric.runtime.md | 2 +- .../nemo-fabric-core/error/enum-fabricerror.mdx | 2 +- .../nemo-fabric-core/resource-snapshot/index.mdx | 4 +++- .../nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/reference/api/python-library-reference/nemo_fabric.errors.md b/docs/reference/api/python-library-reference/nemo_fabric.errors.md index 73b6ba7a8..e3acafc12 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.errors.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.errors.md @@ -281,7 +281,7 @@ Initialize a structured NeMo Fabric exception. Runtime resources cannot be read now. -The runtime stopped, is stopping, its adapter host process exited, or the process table cannot be read. +The runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. ### Inheritance diff --git a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md index b77596c57..98ed49750 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md @@ -197,7 +197,7 @@ The snapshot covers the local adapter host process and its live descendants. It **Raises:** - - `ResourceObservationUnavailableError`: If the runtime stopped, is stopping, its adapter host process exited, or the process table cannot be read. + - `ResourceObservationUnavailableError`: If the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. - `ResourceObservationUnsupportedError`: If the runtime's environment provider or operating system has no resource reader. - `FabricNativeUnavailableError`: If the native extension is missing. - `FabricRuntimeError`: If the native snapshot call fails otherwise. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 4c5c5da33..d4082937d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -501,7 +501,7 @@ Adapter kind.
-A runtime's resources cannot be read now: the runtime stopped, is stopping, its adapter host process exited, or the process table cannot be read. +A runtime's resources cannot be read now: the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. #### Fields diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index 7ff9eab7c..eefcd4bb4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -14,7 +14,9 @@ Resource snapshots of a runtime's local adapter host and its descendants. A snapshot reads the process tree rooted at the adapter host process that NeMo Fabric started for a runtime. Each read visits a parent before its children. CPU time adds every process's own CPU time and the CPU time of the children it has already waited for, so work done by finished descendants still counts. A child reaped between the read of its parent and its own read is missed by that one snapshot and is never counted twice. CPU time can drop between snapshots when a descendant is reparented out of the tree, and on macOS while an exited descendant is not yet waited for, because macOS cannot list an unreaped process. -Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. Neither reads command lines, environment variables, or open files. +A live process in the tree that cannot be read, or a process table that cannot be listed in full, makes the snapshot unavailable instead of leaving the process out. + +Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. To find the tree, the macOS reader, and the Linux reader on a kernel without `/proc//task//children`, also read other processes' parent IDs. A snapshot exposes only its own fields: never command lines, environment variables, open files, or processes outside the tree. ## Structs diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx index cbe246229..745457bd3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx @@ -13,4 +13,4 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Read a timestamped resource snapshot of a started runtime. -The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, or the process table cannot be read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the environment provider or operating system has no resource reader. +The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, or a live process in its tree or the process table cannot be read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the environment provider or operating system has no resource reader. From 888230bfc51da32c6c08059de89ffe820d0f8e08 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:08:54 -0700 Subject: [PATCH 010/159] fix: read macOS snapshots from each process's children The macOS reader listed every process on the machine and read each one's parent to find the runtime's tree. That needed a short BSD record fallback for other users' processes, made a snapshot unavailable when any process on the machine refused both records, and dropped an exited child that its parent had not yet waited for out of CPU time, because PROC_PIDTBSDINFO fails on a zombie. The reader now lists each process's children with proc_listchildpids, as the Linux reader does with /proc children files. proc_listchildpids lists zombies and proc_pid_rusage reads them, so an unreaped child's CPU time counts on macOS the same way it does on Linux. A child counts only when it started before its parent's children were listed, which proves its PID was not reused after the listing. The walk now lives outside the platform modules behind one ProcessTable trait, so its scripted tests run on Linux CI for both identity rules. The Linux reader reports a children list that never settles as a changing tree instead of an unreadable process. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 1317 +++++++---------- docs/sdk/python.mdx | 20 +- schemas/sdk/run-result.schema.json | 2 +- .../sdk/runtime-resource-snapshot.schema.json | 2 +- .../src/nemo_fabric/types.py | 4 +- 5 files changed, 568 insertions(+), 777 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index c6d004828..3c4582cb6 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -4,24 +4,23 @@ //! Resource snapshots of a runtime's local adapter host and its descendants. //! //! A snapshot reads the process tree rooted at the adapter host process that -//! NeMo Fabric started for a runtime. Each read visits a parent before its -//! children. CPU time adds every process's own CPU time and the CPU time of the -//! children it has already waited for, so work done by finished descendants -//! still counts. A child reaped between the read of its parent and its own -//! read is missed by that one snapshot and is never counted twice. CPU time -//! can drop between snapshots when a descendant is reparented out of the tree, -//! and on macOS while an exited descendant is not yet waited for, because -//! macOS cannot list an unreaped process. +//! NeMo Fabric started for a runtime, each parent before its children. CPU +//! time adds every process's own CPU time and the CPU time of the children it +//! has already waited for. An exited child that is not yet waited for counts +//! through its own CPU time, so work done by finished descendants still +//! counts. A child reaped between the read of its parent and its own read is +//! missed by that one snapshot and is never counted twice. CPU time can drop +//! between snapshots when a descendant is reparented out of the tree. //! -//! A live process in the tree that cannot be read, or a process table that -//! cannot be listed in full, makes the snapshot unavailable instead of -//! leaving the process out. +//! Linux reads `/proc//task//children` and `/proc//stat`. On a +//! kernel without those children files, it reads every process's parent ID +//! once per snapshot to find the tree. macOS reads `proc_listchildpids` and +//! `proc_pid_rusage`. A snapshot exposes only its own fields: never command +//! lines, environment variables, open files, or processes outside the tree. //! -//! Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. To find -//! the tree, the macOS reader, and the Linux reader on a kernel without -//! `/proc//task//children`, also read other processes' parent IDs. -//! A snapshot exposes only its own fields: never command lines, environment -//! variables, open files, or processes outside the tree. +//! A live process in the tree that cannot be read, or a process table that +//! the Linux fallback cannot read in full, makes the snapshot unavailable +//! instead of leaving the process out. use std::time::{SystemTime, UNIX_EPOCH}; @@ -33,18 +32,6 @@ use crate::error::{FabricError, Result}; const MILLIS_PER_DAY: i64 = 86_400_000; const NANOS_PER_SECOND: f64 = 1_000_000_000.0; -/// Reasons a tree read returns when the snapshot is unavailable. -#[cfg_attr( - not(any(target_os = "linux", target_os = "macos")), - allow(dead_code, reason = "only the Linux and macOS readers read a tree") -)] -mod reason { - pub(super) const HOST_EXITED: &str = "adapter host process has exited"; - pub(super) const PROCESS_UNREADABLE: &str = - "a live process in the runtime's process tree cannot be read"; - pub(super) const TABLE_UNREADABLE: &str = "the process table cannot be read in full"; -} - /// Timestamped resource usage of one runtime's local adapter host process tree. /// /// Memory, process count, and CPU time cover the adapter host process and its @@ -68,9 +55,8 @@ pub struct RuntimeResourceSnapshot { /// that are not yet reaped are not counted. pub process_count: u64, /// CPU time consumed by the tree since the adapter host started, in seconds, - /// including exited descendants (on macOS, only once they are waited for). - /// It can drop when a descendant leaves the tree, and on macOS while an - /// exited descendant is not yet waited for. + /// including exited descendants. It can drop when a descendant leaves the + /// tree. #[schemars(range(min = 0.0))] pub cpu_time_seconds: f64, } @@ -154,6 +140,119 @@ struct TreeRead { process_count: u64, } +/// The tree walk the Linux and macOS readers share. It is platform-neutral so +/// its tests run on every platform. +#[cfg_attr( + not(any(target_os = "linux", target_os = "macos")), + allow(dead_code, reason = "only the Linux and macOS readers walk a tree") +)] +mod tree { + use std::collections::VecDeque; + + use super::TreeRead; + + pub(super) const HOST_EXITED: &str = "adapter host process has exited"; + pub(super) const PROCESS_UNREADABLE: &str = + "a live process in the runtime's process tree cannot be read"; + + /// Why a process, or the list of its children, could not be read. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) enum ReadError { + /// The process exited and was reaped. + Gone, + /// The process exists but cannot be read, for this reason. + Unavailable(&'static str), + } + + /// One process as a tree read sees it. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) struct Process { + /// Parent PID, when the platform reads it in the same record as the + /// rest of these fields. + pub ppid: Option, + /// Start time, in the clock that [`Children::listed_at`] uses. + pub start: u64, + /// Whether the process exited and is not yet reaped. + pub exited: bool, + /// The process's own CPU time plus that of the children it waited for. + pub cpu_nanos: u64, + pub resident_bytes: u64, + } + + /// The children of one process, listed at one instant. + #[derive(Debug, Clone, PartialEq, Eq)] + pub(super) struct Children { + pub pids: Vec, + /// When the list was read, in the clock that [`Process::start`] uses. + /// Only a process read without its parent PID is checked against it. + pub listed_at: u64, + } + + /// The reads a tree read makes. Each platform implements them; tests + /// script them. + pub(super) trait ProcessTable { + fn children(&self, pid: u32) -> Result; + fn read(&self, pid: u32) -> Result; + } + + /// Whether `process`, listed as a child of `parent` at `listed_at` and + /// read afterwards, is still the listed process. A parent PID read with + /// the record shows it directly. Without one, a process that started + /// before the list was read already held its PID then, so it is the + /// listed process. + fn is_listed_child(process: &Process, parent: u32, listed_at: u64) -> bool { + match process.ppid { + Some(ppid) => ppid == parent, + None => process.start < listed_at, + } + } + + /// Read the tree under `root`. Fails when the root exited or its PID now + /// names a process with another start time than `identity`, or when a + /// live process in the tree cannot be read. + pub(super) fn read_tree_with( + table: &impl ProcessTable, + root: u32, + identity: u64, + ) -> Result { + let mut read = TreeRead::default(); + // Each entry is a PID and, for a child, its parent and the time the + // parent's children were listed. + let mut pending = VecDeque::from([(root, None)]); + while let Some((pid, listed)) = pending.pop_front() { + // List the children before reading the process: a read that still + // shows the expected process proves the listed children were its own. + let found = table + .children(pid) + .and_then(|children| Ok((table.read(pid)?, children))); + let expected = |process: &Process| match listed { + None => process.start == identity && !process.exited, + Some((parent, listed_at)) => is_listed_child(process, parent, listed_at), + }; + let (process, children) = match found { + Ok((process, children)) if expected(&process) => (process, children), + // A reaped child, or one whose PID now names another process. + Ok(_) | Err(ReadError::Gone) if listed.is_some() => continue, + Ok(_) | Err(ReadError::Gone) => return Err(HOST_EXITED), + Err(ReadError::Unavailable(reason)) => return Err(reason), + }; + read.cpu_nanos = read.cpu_nanos.saturating_add(process.cpu_nanos); + if !process.exited { + read.resident_bytes = read.resident_bytes.saturating_add(process.resident_bytes); + read.process_count += 1; + } + let listed_at = children.listed_at; + pending.extend( + children + .pids + .into_iter() + .map(|child| (child, Some((pid, listed_at)))), + ); + } + Ok(read) + } +} + fn unix_millis(time: SystemTime) -> i64 { match time.duration_since(UNIX_EPOCH) { Ok(after) => i64::try_from(after.as_millis()).unwrap_or(i64::MAX), @@ -199,26 +298,25 @@ fn civil_from_days(days: i64) -> (i64, i64, i64) { #[cfg(target_os = "linux")] mod platform { - //! `/proc` reads, from the root down, each parent before its children. - //! Each process's children come from `/proc//task//children`, - //! so a read visits only the runtime's own processes. A kernel built - //! without those files falls back to one pass over every process's parent. + //! `/proc` reads. Each process's children come from + //! `/proc//task//children`, so a read visits only the runtime's + //! own processes. A kernel built without those files falls back to one + //! pass over every process's parent per snapshot. - use std::collections::{BTreeMap, VecDeque}; + use std::collections::BTreeMap; use std::sync::OnceLock; - use super::{TreeRead, reason}; + use super::TreeRead; + use super::tree::{ + Children, PROCESS_UNREADABLE, Process, ProcessTable, ReadError, read_tree_with, + }; pub(super) const SUPPORTED: bool = true; - - /// Why one process could not be read. - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) enum ReadError { - /// The process exited. - Gone, - /// The process exists but cannot be read, or its record cannot be parsed. - Unreadable, - } + pub(super) const TABLE_UNREADABLE: &str = "the process table cannot be read in full"; + pub(super) const TREE_CHANGING: &str = + "the runtime's process tree kept changing during the read"; + /// Times a process's children are read while waiting for two reads that agree. + const MAX_CHILDREN_READS: usize = 4; /// The fields of `/proc//stat` a snapshot needs. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -251,50 +349,45 @@ mod platform { }) } - /// Times a process's children are read while waiting for two reads that agree. - const MAX_CHILDREN_READS: usize = 4; - - /// The `/proc` reads a tree read makes. Tests script them. - pub(super) trait Procfs { - /// Whether the kernel provides `/proc//task//children`. - fn has_children_files(&self) -> bool; - /// The children of every thread of `pid`, sorted. - fn children(&self, pid: u32) -> Result, ReadError>; - /// Every PID in `/proc`. - fn pids(&self) -> Result, ReadError>; - fn stat(&self, pid: u32) -> Result; + /// Where a tree read finds each process's children. + pub(super) enum ChildSource { + /// `/proc//task//children`. + ChildrenFiles, + /// Children by parent PID, from one pass over every process. + Scanned(BTreeMap>), } - struct Live; - - impl Procfs for Live { - fn has_children_files(&self) -> bool { - static PRESENT: OnceLock = OnceLock::new(); - *PRESENT.get_or_init(|| { - let pid = std::process::id(); - std::path::Path::new(&format!("/proc/{pid}/task/{pid}/children")).exists() - }) - } + struct Live { + children: ChildSource, + ticks_per_second: u64, + page_size: u64, + } - fn children(&self, pid: u32) -> Result, ReadError> { - read_until_stable(|| children_once(pid)) + impl ProcessTable for Live { + fn children(&self, pid: u32) -> Result { + let pids = match &self.children { + ChildSource::ChildrenFiles => read_until_stable(|| children_once(pid))?, + ChildSource::Scanned(by_parent) => by_parent.get(&pid).cloned().unwrap_or_default(), + }; + // Each read checks the parent PID in `stat`, so the list time is unused. + Ok(Children { pids, listed_at: 0 }) } - fn pids(&self) -> Result, ReadError> { - let mut pids = Vec::new(); - for entry in std::fs::read_dir("/proc").map_err(read_error)? { - let name = entry.map_err(read_error)?.file_name(); - if let Some(pid) = name.to_str().and_then(|name| name.parse().ok()) { - pids.push(pid); - } - } - Ok(pids) + fn read(&self, pid: u32) -> Result { + let stat = stat(pid)?; + Ok(Process { + ppid: Some(stat.ppid), + start: stat.start, + exited: stat.zombie, + cpu_nanos: ticks_to_nanos(stat.cpu_ticks, self.ticks_per_second), + resident_bytes: stat.rss_pages.saturating_mul(self.page_size), + }) } + } - fn stat(&self, pid: u32) -> Result { - let text = std::fs::read_to_string(format!("/proc/{pid}/stat")).map_err(read_error)?; - parse_stat(&text).ok_or(ReadError::Unreadable) - } + fn stat(pid: u32) -> Result { + let text = std::fs::read_to_string(format!("/proc/{pid}/stat")).map_err(read_error)?; + parse_stat(&text).ok_or(ReadError::Unavailable(PROCESS_UNREADABLE)) } /// One read of the children files of every thread of `pid`. A thread that @@ -310,7 +403,11 @@ mod platform { Err(unreadable) => return Err(unreadable), }; for child in text.split_whitespace() { - children.push(child.parse().map_err(|_| ReadError::Unreadable)?); + children.push( + child + .parse() + .map_err(|_| ReadError::Unavailable(PROCESS_UNREADABLE))?, + ); } } children.sort_unstable(); @@ -331,7 +428,37 @@ mod platform { } last = next; } - Err(ReadError::Unreadable) + Err(ReadError::Unavailable(TREE_CHANGING)) + } + + /// Every process's children, for a kernel without children files. A + /// process that exited during the pass is left out; one whose parent + /// cannot be read leaves the listing incomplete. + pub(super) fn children_by_parent( + pids: Vec, + parent_of: impl Fn(u32) -> Result, + ) -> Result>, &'static str> { + let mut children: BTreeMap> = BTreeMap::new(); + for pid in pids { + match parent_of(pid) { + Ok(ppid) => children.entry(ppid).or_default().push(pid), + Err(ReadError::Gone) => {} + Err(ReadError::Unavailable(_)) => return Err(TABLE_UNREADABLE), + } + } + Ok(children) + } + + fn all_pids() -> Result, &'static str> { + let entries = std::fs::read_dir("/proc").map_err(|_| TABLE_UNREADABLE)?; + let mut pids = Vec::new(); + for entry in entries { + let name = entry.map_err(|_| TABLE_UNREADABLE)?.file_name(); + if let Some(pid) = name.to_str().and_then(|name| name.parse().ok()) { + pids.push(pid); + } + } + Ok(pids) } /// `ENOENT` and `ESRCH` mean the process exited; anything else means it @@ -341,12 +468,12 @@ mod platform { { ReadError::Gone } else { - ReadError::Unreadable + ReadError::Unavailable(PROCESS_UNREADABLE) } } pub(super) fn start_identity(pid: u32) -> Option { - Live.stat(pid).ok().map(|stat| stat.start) + stat(pid).ok().map(|stat| stat.start) } fn sysconf(name: libc::c_int) -> u64 { @@ -354,85 +481,42 @@ mod platform { u64::try_from(unsafe { libc::sysconf(name) }).unwrap_or(0) } - fn clock_ticks_per_second() -> u64 { - static TICKS: OnceLock = OnceLock::new(); - *TICKS.get_or_init(|| sysconf(libc::_SC_CLK_TCK).max(1)) + fn has_children_files() -> bool { + static PRESENT: OnceLock = OnceLock::new(); + *PRESENT.get_or_init(|| { + let pid = std::process::id(); + std::path::Path::new(&format!("/proc/{pid}/task/{pid}/children")).exists() + }) } - fn page_size() -> u64 { - static PAGE: OnceLock = OnceLock::new(); - *PAGE.get_or_init(|| sysconf(libc::_SC_PAGESIZE)) + /// Children by parent PID, from one pass over every process. + pub(super) fn scan_children() -> Result { + let by_parent = children_by_parent(all_pids()?, |pid| stat(pid).map(|stat| stat.ppid))?; + Ok(ChildSource::Scanned(by_parent)) } pub(super) fn read_tree(root: u32, identity: u64) -> Result { - read_tree_with(&Live, root, identity, clock_ticks_per_second(), page_size()) - } - - /// Every process's children, for a kernel without children files. A - /// process that exited during the pass is left out; one that cannot be - /// read leaves the listing incomplete. - fn children_by_parent(procfs: &impl Procfs) -> Result>, &'static str> { - let pids = procfs.pids().map_err(|_| reason::TABLE_UNREADABLE)?; - let mut children: BTreeMap> = BTreeMap::new(); - for pid in pids { - match procfs.stat(pid) { - Ok(stat) => children.entry(stat.ppid).or_default().push(pid), - Err(ReadError::Gone) => {} - Err(ReadError::Unreadable) => return Err(reason::TABLE_UNREADABLE), - } - } - Ok(children) + let children = if has_children_files() { + ChildSource::ChildrenFiles + } else { + scan_children()? + }; + read_tree_from(children, root, identity) } - /// Read the tree under `root`. Fails when the root exited or its PID now - /// names a process with another start time than `identity`, or when a - /// live process in the tree cannot be read. - pub(super) fn read_tree_with( - procfs: &impl Procfs, + pub(super) fn read_tree_from( + children: ChildSource, root: u32, identity: u64, - ticks_per_second: u64, - page_size: u64, ) -> Result { - let listing = if procfs.has_children_files() { - None - } else { - Some(children_by_parent(procfs)?) + static TICKS: OnceLock = OnceLock::new(); + static PAGE: OnceLock = OnceLock::new(); + let live = Live { + children, + ticks_per_second: *TICKS.get_or_init(|| sysconf(libc::_SC_CLK_TCK).max(1)), + page_size: *PAGE.get_or_init(|| sysconf(libc::_SC_PAGESIZE)), }; - let mut read = TreeRead::default(); - let mut cpu_ticks: u64 = 0; - let mut rss_pages: u64 = 0; - // Each entry is a PID and the parent it was listed under (none for the root). - let mut pending = VecDeque::from([(root, None)]); - while let Some((pid, parent)) = pending.pop_front() { - // List the children before the stat read: a stat that still names - // the expected process shows the listed children were its own. - let children = match &listing { - Some(listing) => Ok(listing.get(&pid).cloned().unwrap_or_default()), - None => procfs.children(pid), - }; - let found = children.and_then(|children| Ok((procfs.stat(pid)?, children))); - let expected = |stat: &Stat| match parent { - None => stat.start == identity && !stat.zombie, - Some(parent) => stat.ppid == parent, - }; - let (stat, children) = match found { - Ok((stat, children)) if expected(&stat) => (stat, children), - // An exited child, or one whose PID now names another process. - Ok(_) | Err(ReadError::Gone) if parent.is_some() => continue, - Ok(_) | Err(ReadError::Gone) => return Err(reason::HOST_EXITED), - Err(ReadError::Unreadable) => return Err(reason::PROCESS_UNREADABLE), - }; - cpu_ticks = cpu_ticks.saturating_add(stat.cpu_ticks); - if !stat.zombie { - rss_pages = rss_pages.saturating_add(stat.rss_pages); - read.process_count += 1; - } - pending.extend(children.into_iter().map(|child| (child, Some(pid)))); - } - read.cpu_nanos = ticks_to_nanos(cpu_ticks, ticks_per_second); - read.resident_bytes = rss_pages.saturating_mul(page_size); - Ok(read) + read_tree_with(&live, root, identity) } pub(super) fn ticks_to_nanos(ticks: u64, ticks_per_second: u64) -> u64 { @@ -443,167 +527,90 @@ mod platform { #[cfg(target_os = "macos")] mod platform { - //! `libproc` reads. One pass lists every process's parent and start time, - //! then the tree is read from the root down, each parent before its - //! children. + //! `libproc` reads. `proc_listchildpids` lists a process's children, + //! including exited ones that are not yet reaped, and `proc_pid_rusage` + //! reads any of them. - use std::collections::{BTreeMap, VecDeque}; use std::sync::OnceLock; - use super::{TreeRead, reason}; + use super::TreeRead; + use super::tree::{ + Children, PROCESS_UNREADABLE, Process, ProcessTable, ReadError, read_tree_with, + }; pub(super) const SUPPORTED: bool = true; + /// Child PIDs the first list call has room for. A full buffer doubles it. + const INITIAL_LISTED_CHILDREN: usize = 64; - /// Why one process could not be read. - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) enum ReadError { - /// The process exited, or is a zombie the call cannot describe. - Gone, - /// The process exists but this process may not read it. - Denied, - } - - /// A process's parent and start time, which together identify it. - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) struct Parent { - pub ppid: u32, - /// `pbi_start_tvsec` and `pbi_start_tvusec`, or `None` for another - /// user's process, whose parent only the short record shows. - pub started: Option<(u64, u64)>, - } - - /// `struct proc_bsdshortinfo` from ``, which the `libc` - /// crate version in use does not bind: PID, parent PID, then 56 bytes of - /// group, status, name, flags, and user and group IDs that the reader ignores. - #[repr(C)] - struct ProcBsdShortInfo { - pid: u32, - ppid: u32, - rest: [u32; 14], - } - - const PROC_PIDT_SHORTBSDINFO: libc::c_int = 13; - - /// The `proc_pid_rusage` fields a snapshot needs, CPU times in Mach units. - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) struct Usage { - pub cpu_units: u64, - pub resident_bytes: u64, - /// `ri_proc_start_abstime`. - pub start: u64, - pub exited: bool, - } - - /// The process reads a tree read makes. Tests script them. - pub(super) trait Libproc { - /// Every PID, or `None` when the process table cannot be listed. - fn pids(&self) -> Option>; - fn parent(&self, pid: u32) -> Result; - fn usage(&self, pid: u32) -> Result; + struct Live { + timebase: (u64, u64), } - struct Live; - - impl Libproc for Live { - fn pids(&self) -> Option> { - let mut pids = vec![0 as libc::pid_t; 1024]; - let count = loop { - let bytes = libc::c_int::try_from(std::mem::size_of_val(pids.as_slice())).ok()?; + impl ProcessTable for Live { + fn children(&self, pid: u32) -> Result { + let ppid = libc::pid_t::try_from(pid).map_err(|_| ReadError::Gone)?; + let mut pids = vec![0 as libc::pid_t; INITIAL_LISTED_CHILDREN]; + loop { + let bytes = libc::c_int::try_from(std::mem::size_of_val(pids.as_slice())) + .map_err(|_| ReadError::Unavailable(PROCESS_UNREADABLE))?; + // SAFETY: mach_absolute_time has no preconditions. + let listed_at = unsafe { mach_absolute_time() }; // SAFETY: the buffer is writable and sized in bytes as libproc requires. - let count = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) }; - let count = usize::try_from(count).ok().filter(|count| *count > 0)?; + let count = + unsafe { libc::proc_listchildpids(ppid, pids.as_mut_ptr().cast(), bytes) }; + let count = usize::try_from(count) + .map_err(|_| ReadError::Unavailable(PROCESS_UNREADABLE))?; + // A full buffer can mean the list was cut short. if count < pids.len() { - break count; + pids.truncate(count); + let pids = pids + .into_iter() + .filter_map(|pid| u32::try_from(pid).ok()) + .collect(); + return Ok(Children { pids, listed_at }); } pids.resize(pids.len() * 2, 0); - }; - Some( - pids[..count] - .iter() - .filter_map(|pid| u32::try_from(*pid).ok()) - .collect(), - ) - } - - /// `PROC_PIDTBSDINFO` fails for a zombie, so a zombie reads as gone. - /// It is denied for another user's process, whose parent the short - /// record still shows. - fn parent(&self, pid: u32) -> Result { - let pid = libc::pid_t::try_from(pid).map_err(|_| ReadError::Gone)?; - // SAFETY: proc_bsdinfo is plain data; zero is a valid initial value. - let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; - let size = libc::c_int::try_from(std::mem::size_of::()) - .map_err(|_| ReadError::Denied)?; - // SAFETY: `info` is writable and `size` bytes long, as PROC_PIDTBSDINFO needs. - let written = unsafe { - libc::proc_pidinfo(pid, libc::PROC_PIDTBSDINFO, 0, (&raw mut info).cast(), size) - }; - if written == size { - return Ok(Parent { - ppid: info.pbi_ppid, - started: Some((info.pbi_start_tvsec, info.pbi_start_tvusec)), - }); - } - match last_read_error() { - ReadError::Denied => short_parent(pid), - gone => Err(gone), } } - fn usage(&self, pid: u32) -> Result { - let pid = libc::c_int::try_from(pid).map_err(|_| ReadError::Gone)?; - // SAFETY: rusage_info_v2 is plain data; zero is a valid initial value. - let mut info: libc::rusage_info_v2 = unsafe { std::mem::zeroed() }; - // SAFETY: `info` is a writable rusage_info_v2, the layout RUSAGE_INFO_V2 names. - let status = unsafe { - libc::proc_pid_rusage( - pid, - libc::RUSAGE_INFO_V2, - (&raw mut info).cast::(), - ) - }; - if status != 0 { - return Err(last_read_error()); - } - Ok(Usage { - cpu_units: info - .ri_user_time - .saturating_add(info.ri_system_time) - .saturating_add(info.ri_child_user_time) - .saturating_add(info.ri_child_system_time), - resident_bytes: info.ri_resident_size, + fn read(&self, pid: u32) -> Result { + let info = usage(pid)?; + Ok(Process { + ppid: None, start: info.ri_proc_start_abstime, exited: info.ri_proc_exit_abstime != 0, + cpu_nanos: mach_to_nanos( + info.ri_user_time + .saturating_add(info.ri_system_time) + .saturating_add(info.ri_child_user_time) + .saturating_add(info.ri_child_system_time), + self.timebase, + ), + resident_bytes: info.ri_resident_size, }) } } - fn short_parent(pid: libc::pid_t) -> Result { - let mut info = ProcBsdShortInfo { - pid: 0, - ppid: 0, - rest: [0; 14], - }; - let size = libc::c_int::try_from(std::mem::size_of::()) - .map_err(|_| ReadError::Denied)?; - // SAFETY: `info` is writable and `size` bytes long, as PROC_PIDT_SHORTBSDINFO needs. - let written = unsafe { - libc::proc_pidinfo(pid, PROC_PIDT_SHORTBSDINFO, 0, (&raw mut info).cast(), size) + /// `proc_pid_rusage` of `pid`. CPU and start times are in Mach absolute + /// time units. + fn usage(pid: u32) -> Result { + let pid = libc::c_int::try_from(pid).map_err(|_| ReadError::Gone)?; + // SAFETY: rusage_info_v2 is plain data; zero is a valid initial value. + let mut info: libc::rusage_info_v2 = unsafe { std::mem::zeroed() }; + // SAFETY: `info` is a writable rusage_info_v2, the layout RUSAGE_INFO_V2 names. + let status = unsafe { + libc::proc_pid_rusage( + pid, + libc::RUSAGE_INFO_V2, + (&raw mut info).cast::(), + ) }; - if written == size { - return Ok(Parent { - ppid: info.ppid, - started: None, - }); + if status == 0 { + return Ok(info); } - Err(last_read_error()) - } - - /// Classify the error of the libproc call that just failed. - fn last_read_error() -> ReadError { match std::io::Error::last_os_error().raw_os_error() { - Some(libc::ESRCH) => ReadError::Gone, - _ => ReadError::Denied, + Some(libc::ESRCH) => Err(ReadError::Gone), + _ => Err(ReadError::Unavailable(PROCESS_UNREADABLE)), } } @@ -614,10 +621,12 @@ mod platform { denom: u32, } - // libc marks its binding deprecated in favour of the `mach2` crate; one - // system call does not justify a new dependency. + // libc marks these bindings deprecated in favour of the `mach2` crate; two + // system calls do not justify a new dependency. unsafe extern "C" { fn mach_timebase_info(info: *mut MachTimebaseInfo) -> libc::c_int; + /// The clock that `ri_proc_start_abstime` uses. + fn mach_absolute_time() -> u64; } /// Nanoseconds per Mach absolute time unit, as `(numer, denom)`, or `None` @@ -640,94 +649,12 @@ mod platform { } pub(super) fn start_identity(pid: u32) -> Option { - Live.usage(pid).ok().map(|usage| usage.start) + usage(pid).ok().map(|info| info.ri_proc_start_abstime) } pub(super) fn read_tree(root: u32, identity: u64) -> Result { - let timebase = timebase().ok_or(reason::PROCESS_UNREADABLE)?; - read_tree_with(&Live, root, identity, timebase) - } - - /// Child PIDs, each with its parent record, by parent PID. - type ChildrenByParent = BTreeMap>; - - /// Every live process's children, each with its parent record. One - /// `proc_listallpids` call and one `proc_pidinfo` per process: - /// `proc_listchildpids` scans the whole process table on each call, which - /// costs too much for a wide tree. A process that exited during the pass - /// is left out; one that cannot be read leaves the listing incomplete. - fn children_by_parent(procs: &impl Libproc) -> Result { - let pids = procs.pids().ok_or(reason::TABLE_UNREADABLE)?; - let mut children = ChildrenByParent::new(); - for pid in pids { - match procs.parent(pid) { - Ok(parent) => children.entry(parent.ppid).or_default().push((pid, parent)), - Err(ReadError::Gone) => {} - Err(ReadError::Denied) => return Err(reason::TABLE_UNREADABLE), - } - } - Ok(children) - } - - /// Read the tree under `root`. Fails when the root exited or its PID now - /// names a process with another start time than `identity`, or when a - /// live process in the tree cannot be read. - pub(super) fn read_tree_with( - procs: &impl Libproc, - root: u32, - identity: u64, - timebase: (u64, u64), - ) -> Result { - let children = children_by_parent(procs)?; - // Read after the listing: a root that is still the same live process - // was alive during the whole listing, so its listed children are its own. - let root_usage = match procs.usage(root) { - Ok(usage) if usage.start == identity && !usage.exited => usage, - Ok(_) | Err(ReadError::Gone) => return Err(reason::HOST_EXITED), - Err(ReadError::Denied) => return Err(reason::PROCESS_UNREADABLE), - }; - let mut read = TreeRead::default(); - let mut cpu_units: u64 = 0; - let mut pending = VecDeque::from([(root, root_usage)]); - while let Some((pid, usage)) = pending.pop_front() { - cpu_units = cpu_units.saturating_add(usage.cpu_units); - read.resident_bytes = read.resident_bytes.saturating_add(usage.resident_bytes); - read.process_count += 1; - for &(child, listed) in children.get(&pid).into_iter().flatten() { - if let Some(usage) = listed_child_usage(procs, child, listed)? { - pending.push_back((child, usage)); - } - } - } - read.cpu_nanos = mach_to_nanos(cpu_units, timebase); - Ok(read) - } - - /// The usage of `child`, or `None` when it exited or its PID no longer - /// names the listed process. A process that still has the listed parent - /// and start time after the usage read was the listed one throughout, so - /// the usage is its own. Fails when the child is live but cannot be read. - fn listed_child_usage( - procs: &impl Libproc, - child: u32, - listed: Parent, - ) -> Result, &'static str> { - if listed.started.is_none() { - return Err(reason::PROCESS_UNREADABLE); - } - let usage = match procs.usage(child) { - Ok(usage) if !usage.exited => usage, - Ok(_) | Err(ReadError::Gone) => return Ok(None), - Err(ReadError::Denied) => return Err(reason::PROCESS_UNREADABLE), - }; - match procs.parent(child) { - Ok(now) if now == listed => Ok(Some(usage)), - Ok(now) if now.ppid == listed.ppid && now.started.is_none() => { - Err(reason::PROCESS_UNREADABLE) - } - Ok(_) | Err(ReadError::Gone) => Ok(None), - Err(ReadError::Denied) => Err(reason::PROCESS_UNREADABLE), - } + let timebase = timebase().ok_or(PROCESS_UNREADABLE)?; + read_tree_with(&Live { timebase }, root, identity) } } @@ -748,6 +675,13 @@ mod platform { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + #[cfg(any(target_os = "linux", target_os = "macos"))] + use std::time::{Duration, Instant}; + + use super::tree::{ + Children, HOST_EXITED, PROCESS_UNREADABLE, Process, ProcessTable, ReadError, read_tree_with, + }; use super::*; #[test] @@ -822,472 +756,329 @@ mod tests { .expect("spawn child"); let host = LocalHostProcess::new(child.id(), SystemTime::now()); child.kill().expect("kill child"); - std::thread::sleep(std::time::Duration::from_millis(200)); - let error = snapshot_local_host("runtime-1", Some(&host)).expect_err("zombie host"); + let deadline = Instant::now() + Duration::from_secs(5); + let mut result = snapshot_local_host("runtime-1", Some(&host)); + while result.is_ok() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + result = snapshot_local_host("runtime-1", Some(&host)); + } child.wait().expect("reap child"); assert!(matches!( - error, - FabricError::ResourceObservationUnavailable { .. } + result, + Err(FabricError::ResourceObservationUnavailable { ref reason, .. }) + if reason == HOST_EXITED )); } - #[cfg(target_os = "linux")] + /// The host is `sleep`, which never waits for children. Before its `exec`, + /// the shell starts a child that burns CPU and exits, so the child stays + /// unreaped under the host. + #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] - fn linux_stat_parse_reads_cpu_with_waited_children_start_time_and_ticks() { - let stat = "42 (a (weird) name) S 7 42 42 0 -1 4194560 100 0 0 0 30 20 5 4 20 0 1 0 \ - 9001 1000000 250 18446744073709551615"; + fn snapshot_local_host_counts_cpu_of_exited_unreaped_child() { + let mut host_process = std::process::Command::new("/bin/sh") + .args([ + "-c", + "(i=0; while [ $i -lt 100000 ]; do i=$((i+1)); done) & exec sleep 30", + ]) + .spawn() + .expect("spawn host"); + let host = LocalHostProcess::new(host_process.id(), SystemTime::now()); + let deadline = Instant::now() + Duration::from_secs(30); + let snapshot_until = |done: fn(u64) -> bool| loop { + let snapshot = snapshot_local_host("runtime-1", Some(&host)).expect("snapshot"); + if done(snapshot.process_count) || Instant::now() > deadline { + return snapshot; + } + std::thread::sleep(Duration::from_millis(20)); + }; - let parsed = platform::parse_stat(stat).expect("stat parses"); + let running = snapshot_until(|count| count >= 2); + let exited = snapshot_until(|count| count == 1); + host_process.kill().expect("kill host"); + host_process.wait().expect("reap host"); - assert_eq!(parsed.ppid, 7); - assert_eq!(parsed.cpu_ticks, 30 + 20 + 5 + 4); - assert_eq!(parsed.start, 9001); - assert_eq!(parsed.rss_pages, 250); - assert!(!parsed.zombie); - assert_eq!(platform::ticks_to_nanos(150, 100), 1_500_000_000); + assert_eq!(running.process_count, 2, "child never seen: {running:?}"); + assert_eq!(exited.process_count, 1, "child still running: {exited:?}"); + assert!( + exited.cpu_time_seconds >= 0.1, + "exited child's CPU missing: {exited:?}" + ); } - /// Scripted `libproc` answers for the macOS tree walk. - #[cfg(target_os = "macos")] + /// When a scripted table read the listed children. + const LISTED_AT: u64 = 100; + /// The scripted host: PID 10, started at time 5. + const HOST: u32 = 10; + const HOST_START: u64 = 5; + + /// Scripted answers for the platform-neutral tree walk. A PID without + /// answers has exited. #[derive(Default)] - struct ScriptedLibproc { - pids: Option>, - /// Answers to `parent`, first the listing's, then the later checks'. - parents: std::collections::BTreeMap< - u32, - std::cell::RefCell>>, - >, - usages: std::collections::BTreeMap< - u32, - std::result::Result, - >, + struct ScriptedTable { + children: BTreeMap, ReadError>>, + reads: BTreeMap>, } - #[cfg(target_os = "macos")] - impl ScriptedLibproc { + impl ScriptedTable { fn process( mut self, pid: u32, - parents: Vec>, - usage: std::result::Result, + children: std::result::Result, ReadError>, + read: std::result::Result, ) -> Self { - self.pids.get_or_insert_with(Vec::new).push(pid); - self.parents.insert(pid, std::cell::RefCell::new(parents)); - self.usages.insert(pid, usage); + self.children.insert(pid, children); + self.reads.insert(pid, read); self } - } - #[cfg(target_os = "macos")] - impl platform::Libproc for ScriptedLibproc { - fn pids(&self) -> Option> { - self.pids.clone() - } - - fn parent(&self, pid: u32) -> std::result::Result { - let mut answers = self.parents.get(&pid).expect("scripted pid").borrow_mut(); - if answers.len() > 1 { - answers.remove(0) - } else { - answers[0] - } - } - - fn usage(&self, pid: u32) -> std::result::Result { - self.usages[&pid] + fn read_host_tree(&self) -> std::result::Result { + read_tree_with(self, HOST, HOST_START) } } - #[cfg(target_os = "macos")] - fn mac_parent(ppid: u32, started: u64) -> platform::Parent { - platform::Parent { - ppid, - started: Some((started, 0)), + impl ProcessTable for ScriptedTable { + fn children(&self, pid: u32) -> std::result::Result { + let pids = self + .children + .get(&pid) + .cloned() + .unwrap_or(Err(ReadError::Gone))?; + Ok(Children { + pids, + listed_at: LISTED_AT, + }) } - } - #[cfg(target_os = "macos")] - fn mac_usage(start: u64, cpu_units: u64) -> platform::Usage { - platform::Usage { - cpu_units, - resident_bytes: 1000, - start, - exited: false, + fn read(&self, pid: u32) -> std::result::Result { + self.reads + .get(&pid) + .copied() + .unwrap_or(Err(ReadError::Gone)) } } - /// The host (PID 10) has one listed child (PID 11). After the listing the - /// child exits and an unrelated, newer process takes PID 11. - #[cfg(target_os = "macos")] - #[test] - fn macos_tree_read_skips_child_pid_reused_after_listing() { - let procs = ScriptedLibproc::default() - .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) - .process( - 11, - vec![Ok(mac_parent(10, 200)), Ok(mac_parent(1, 300))], - Ok(mac_usage(9, 1_000_000)), - ); - - let read = platform::read_tree_with(&procs, 10, 5, (1, 1)).expect("tree read"); - - assert_eq!(read.process_count, 1); - assert_eq!(read.cpu_nanos, 7); + /// How a platform shows that a listed child is still the listed process. + #[derive(Debug, Clone, Copy)] + enum Lineage { + /// Linux: the parent PID in the same record. + ParentPid, + /// macOS: a start time before the list was read. + StartedBefore, } - #[cfg(target_os = "macos")] - #[test] - fn macos_tree_read_counts_child_that_is_still_the_listed_process() { - let procs = ScriptedLibproc::default() - .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) - .process(11, vec![Ok(mac_parent(10, 200))], Ok(mac_usage(9, 3))); - - let read = platform::read_tree_with(&procs, 10, 5, (1, 1)).expect("tree read"); - - assert_eq!(read.process_count, 2); - assert_eq!(read.cpu_nanos, 10); - assert_eq!(read.resident_bytes, 2000); - } - - #[cfg(target_os = "macos")] - fn mac_tree_with_child( - child_parents: Vec>, - child_usage: std::result::Result, - ) -> std::result::Result { - let procs = ScriptedLibproc::default() - .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) - .process(11, child_parents, child_usage); - platform::read_tree_with(&procs, 10, 5, (1, 1)) - } - - #[cfg(target_os = "macos")] - #[test] - fn macos_tree_read_with_unreadable_live_child_is_unavailable() { - let denied_usage = mac_tree_with_child( - vec![Ok(mac_parent(10, 200))], - Err(platform::ReadError::Denied), - ); - let other_user = mac_tree_with_child( - vec![Ok(platform::Parent { - ppid: 10, - started: None, - })], - Ok(mac_usage(9, 3)), - ); - let denied_recheck = mac_tree_with_child( - vec![Ok(mac_parent(10, 200)), Err(platform::ReadError::Denied)], - Ok(mac_usage(9, 3)), - ); - - assert_eq!(denied_usage.unwrap_err(), reason::PROCESS_UNREADABLE); - assert_eq!(other_user.unwrap_err(), reason::PROCESS_UNREADABLE); - assert_eq!(denied_recheck.unwrap_err(), reason::PROCESS_UNREADABLE); - } - - /// The child is listed, then exits before its usage read or before the - /// identity check that follows it. - #[cfg(target_os = "macos")] - #[test] - fn macos_tree_read_skips_child_reaped_between_reads() { - let before_usage = mac_tree_with_child( - vec![Ok(mac_parent(10, 200))], - Err(platform::ReadError::Gone), - ) - .expect("tree read"); - let before_check = mac_tree_with_child( - vec![Ok(mac_parent(10, 200)), Err(platform::ReadError::Gone)], - Ok(mac_usage(9, 3)), - ) - .expect("tree read"); - let exited = mac_tree_with_child( - vec![Ok(mac_parent(10, 200))], - Ok(platform::Usage { - exited: true, - ..mac_usage(9, 3) - }), - ) - .expect("tree read"); - - for read in [before_usage, before_check, exited] { - assert_eq!(read.process_count, 1); - assert_eq!(read.cpu_nanos, 7); - } - } - - #[cfg(target_os = "macos")] - #[test] - fn macos_tree_read_with_incomplete_process_table_is_unavailable() { - let unlisted = ScriptedLibproc::default(); - let unreadable_other = ScriptedLibproc::default() - .process(10, vec![Ok(mac_parent(1, 100))], Ok(mac_usage(5, 7))) - .process( - 50, - vec![Err(platform::ReadError::Denied)], - Err(platform::ReadError::Denied), - ); - - assert_eq!( - platform::read_tree_with(&unlisted, 10, 5, (1, 1)).unwrap_err(), - reason::TABLE_UNREADABLE - ); - assert_eq!( - platform::read_tree_with(&unreadable_other, 10, 5, (1, 1)).unwrap_err(), - reason::TABLE_UNREADABLE - ); - } - - #[cfg(target_os = "macos")] - #[test] - fn macos_tree_read_of_exited_or_unreadable_host_is_unavailable() { - let host = |usage| { - let procs = ScriptedLibproc::default().process(10, vec![Ok(mac_parent(1, 100))], usage); - platform::read_tree_with(&procs, 10, 5, (1, 1)).unwrap_err() - }; - - assert_eq!(host(Err(platform::ReadError::Gone)), reason::HOST_EXITED); - assert_eq!(host(Ok(mac_usage(6, 7))), reason::HOST_EXITED); - assert_eq!( - host(Ok(platform::Usage { - exited: true, - ..mac_usage(5, 7) - })), - reason::HOST_EXITED - ); - assert_eq!( - host(Err(platform::ReadError::Denied)), - reason::PROCESS_UNREADABLE - ); - } - - /// Scripted `/proc` answers for the Linux tree walk. - #[cfg(target_os = "linux")] - #[derive(Default)] - struct ScriptedProcfs { - /// Children by parent from children files, or `None` for a kernel - /// without them. - children: Option< - std::collections::BTreeMap, platform::ReadError>>, - >, - pids: Vec, - unlisted: bool, - /// Answers to `stat`, first the listing's, then the walk's. - stats: std::collections::BTreeMap< - u32, - std::cell::RefCell>>, - >, - } - - #[cfg(target_os = "linux")] - impl ScriptedProcfs { - fn process( - mut self, - pid: u32, - stats: Vec>, - ) -> Self { - self.pids.push(pid); - self.stats.insert(pid, std::cell::RefCell::new(stats)); - self - } - } - - #[cfg(target_os = "linux")] - impl platform::Procfs for ScriptedProcfs { - fn has_children_files(&self) -> bool { - self.children.is_some() - } - - fn children(&self, pid: u32) -> std::result::Result, platform::ReadError> { - let children = self.children.as_ref().expect("children files"); - children.get(&pid).cloned().unwrap_or(Ok(Vec::new())) - } - - fn pids(&self) -> std::result::Result, platform::ReadError> { - if self.unlisted { - return Err(platform::ReadError::Unreadable); + impl Lineage { + /// A live process listed under `parent`, with 1000 resident bytes. + fn child(self, parent: u32, cpu_nanos: u64) -> Process { + let (ppid, start) = match self { + Self::ParentPid => (Some(parent), LISTED_AT + 1), + Self::StartedBefore => (None, LISTED_AT - 1), + }; + Process { + ppid, + start, + exited: false, + cpu_nanos, + resident_bytes: 1000, } - Ok(self.pids.clone()) } - fn stat(&self, pid: u32) -> std::result::Result { - let mut answers = self.stats.get(&pid).expect("scripted pid").borrow_mut(); - if answers.len() > 1 { - answers.remove(0) - } else { - answers[0] + /// A process that took the PID of a child of `parent` after the list + /// was read. + fn replacement(self, parent: u32) -> Process { + match self { + Self::ParentPid => Process { + ppid: Some(1), + ..self.child(parent, 900) + }, + Self::StartedBefore => Process { + start: LISTED_AT, + ..self.child(parent, 900) + }, } } - } - #[cfg(target_os = "linux")] - fn linux_stat(ppid: u32, start: u64, cpu_ticks: u64) -> platform::Stat { - platform::Stat { - zombie: false, - ppid, - cpu_ticks, - start, - rss_pages: 1, + fn host(self, cpu_nanos: u64) -> Process { + Process { + start: HOST_START, + ..self.child(1, cpu_nanos) + } } } - /// The host is PID 10 started at tick 5; its child is PID 11. - #[cfg(target_os = "linux")] - fn linux_tree_with_child( - child: Vec>, - ) -> std::result::Result { - let procfs = ScriptedProcfs::default() - .process(10, vec![Ok(linux_stat(1, 5, 7))]) - .process(11, child); - platform::read_tree_with(&procfs, 10, 5, 1, 1) - } + const LINEAGES: [Lineage; 2] = [Lineage::ParentPid, Lineage::StartedBefore]; - #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_with_unreadable_live_child_is_unavailable() { - let read = linux_tree_with_child(vec![ - Ok(linux_stat(10, 9, 3)), - Err(platform::ReadError::Unreadable), - ]); - - assert_eq!(read.unwrap_err(), reason::PROCESS_UNREADABLE); + fn tree_read_counts_live_descendants_and_cpu_of_exited_ones() { + for lineage in LINEAGES { + let table = ScriptedTable::default() + .process(HOST, Ok(vec![11]), Ok(lineage.host(7))) + .process(11, Ok(vec![12, 13]), Ok(lineage.child(HOST, 3))) + .process(12, Ok(vec![]), Ok(lineage.child(11, 2))) + .process( + 13, + Ok(vec![]), + Ok(Process { + exited: true, + ..lineage.child(11, 4) + }), + ); + + let read = table.read_host_tree().expect("tree read"); + + assert_eq!(read.process_count, 3, "{lineage:?}"); + assert_eq!(read.cpu_nanos, 7 + 3 + 2 + 4, "{lineage:?}"); + assert_eq!(read.resident_bytes, 3000, "{lineage:?}"); + } } - #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_skips_child_reaped_or_replaced_between_reads() { - let reaped = linux_tree_with_child(vec![ - Ok(linux_stat(10, 9, 3)), - Err(platform::ReadError::Gone), - ]) - .expect("tree read"); - let replaced = - linux_tree_with_child(vec![Ok(linux_stat(10, 9, 3)), Ok(linux_stat(1, 12, 900))]) - .expect("tree read"); - let counted = linux_tree_with_child(vec![Ok(linux_stat(10, 9, 3))]).expect("tree read"); - - for read in [reaped, replaced] { - assert_eq!(read.process_count, 1); - assert_eq!(read.cpu_nanos, 7_000_000_000); + fn tree_read_skips_child_reaped_or_replaced_after_listing() { + for lineage in LINEAGES { + let with_child = |children, read| { + ScriptedTable::default() + .process(HOST, Ok(vec![11]), Ok(lineage.host(7))) + .process(11, children, read) + .read_host_tree() + .expect("tree read") + }; + let reaped = with_child(Ok(vec![]), Err(ReadError::Gone)); + let reaped_before_its_list = + with_child(Err(ReadError::Gone), Ok(lineage.child(HOST, 3))); + let replaced = with_child(Ok(vec![]), Ok(lineage.replacement(HOST))); + + for read in [reaped, reaped_before_its_list, replaced] { + assert_eq!(read.process_count, 1, "{lineage:?}"); + assert_eq!(read.cpu_nanos, 7, "{lineage:?}"); + } } - assert_eq!(counted.process_count, 2); - assert_eq!(counted.cpu_nanos, 10_000_000_000); } - #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_with_incomplete_process_table_is_unavailable() { - let unlisted = ScriptedProcfs { - unlisted: true, - ..ScriptedProcfs::default() + fn tree_read_with_unreadable_live_child_is_unavailable() { + let with_child = |children, read| { + ScriptedTable::default() + .process(HOST, Ok(vec![11]), Ok(Lineage::ParentPid.host(7))) + .process(11, children, read) + .read_host_tree() }; - let unreadable_other = ScriptedProcfs::default() - .process(10, vec![Ok(linux_stat(1, 5, 7))]) - .process(50, vec![Err(platform::ReadError::Unreadable)]); - assert_eq!( - platform::read_tree_with(&unlisted, 10, 5, 1, 1).unwrap_err(), - reason::TABLE_UNREADABLE - ); - assert_eq!( - platform::read_tree_with(&unreadable_other, 10, 5, 1, 1).unwrap_err(), - reason::TABLE_UNREADABLE + let unreadable = with_child(Ok(vec![]), Err(ReadError::Unavailable(PROCESS_UNREADABLE))); + let unlisted = with_child( + Err(ReadError::Unavailable("children cannot be listed")), + Ok(Lineage::ParentPid.child(HOST, 3)), ); + + assert_eq!(unreadable.unwrap_err(), PROCESS_UNREADABLE); + assert_eq!(unlisted.unwrap_err(), "children cannot be listed"); } - #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_of_exited_or_unreadable_host_is_unavailable() { - let host = |walk| { - let procfs = ScriptedProcfs::default().process(10, vec![Ok(linux_stat(1, 5, 7)), walk]); - platform::read_tree_with(&procfs, 10, 5, 1, 1).unwrap_err() + fn tree_read_of_exited_replaced_or_unreadable_host_is_unavailable() { + let host = |children, read| { + ScriptedTable::default() + .process(HOST, children, read) + .read_host_tree() + .unwrap_err() }; + let live = Lineage::ParentPid.host(7); - assert_eq!(host(Err(platform::ReadError::Gone)), reason::HOST_EXITED); - assert_eq!(host(Ok(linux_stat(1, 6, 7))), reason::HOST_EXITED); + assert_eq!(host(Ok(vec![]), Err(ReadError::Gone)), HOST_EXITED); + assert_eq!(host(Err(ReadError::Gone), Ok(live)), HOST_EXITED); assert_eq!( - host(Ok(platform::Stat { - zombie: true, - ..linux_stat(1, 5, 7) - })), - reason::HOST_EXITED + host( + Ok(vec![]), + Ok(Process { + start: HOST_START + 1, + ..live + }) + ), + HOST_EXITED ); assert_eq!( - host(Err(platform::ReadError::Unreadable)), - reason::PROCESS_UNREADABLE + host( + Ok(vec![]), + Ok(Process { + exited: true, + ..live + }) + ), + HOST_EXITED + ); + assert_eq!( + host(Ok(vec![]), Err(ReadError::Unavailable(PROCESS_UNREADABLE))), + PROCESS_UNREADABLE ); - } - - /// A kernel with children files: the host is PID 10 started at tick 5 and - /// lists child PID 11. The process table is never listed. - #[cfg(target_os = "linux")] - fn linux_children_tree( - host_children: std::result::Result, platform::ReadError>, - child: std::result::Result, - ) -> std::result::Result { - let mut procfs = ScriptedProcfs { - children: Some(std::collections::BTreeMap::from([(10, host_children)])), - unlisted: true, - ..ScriptedProcfs::default() - } - .process(10, vec![Ok(linux_stat(1, 5, 7))]) - .process(11, vec![child]); - procfs.pids.clear(); - platform::read_tree_with(&procfs, 10, 5, 1, 1) } #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_with_children_files_reads_only_the_tree() { - let read = linux_children_tree(Ok(vec![11]), Ok(linux_stat(10, 9, 3))).expect("tree read"); + fn linux_stat_parse_reads_cpu_with_waited_children_start_time_and_ticks() { + let stat = "42 (a (weird) name) S 7 42 42 0 -1 4194560 100 0 0 0 30 20 5 4 20 0 1 0 \ + 9001 1000000 250 18446744073709551615"; + + let parsed = platform::parse_stat(stat).expect("stat parses"); - assert_eq!(read.process_count, 2); - assert_eq!(read.cpu_nanos, 10_000_000_000); - assert_eq!(read.resident_bytes, 2); + assert_eq!(parsed.ppid, 7); + assert_eq!(parsed.cpu_ticks, 30 + 20 + 5 + 4); + assert_eq!(parsed.start, 9001); + assert_eq!(parsed.rss_pages, 250); + assert!(!parsed.zombie); + assert_eq!(platform::ticks_to_nanos(150, 100), 1_500_000_000); } + /// The second read finds a child the first skipped; the third agrees. #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_with_children_files_skips_reaped_or_replaced_child() { - let reaped = - linux_children_tree(Ok(vec![11]), Err(platform::ReadError::Gone)).expect("tree read"); - let replaced = - linux_children_tree(Ok(vec![11]), Ok(linux_stat(1, 12, 900))).expect("tree read"); - - for read in [reaped, replaced] { - assert_eq!(read.process_count, 1); - assert_eq!(read.cpu_nanos, 7_000_000_000); - } + fn linux_children_read_waits_for_two_reads_that_agree() { + let mut reads = vec![vec![11, 13], vec![11, 12, 13], vec![11, 12, 13]].into_iter(); + let mut changing = (0..).map(|n| vec![n]); + + let stable = platform::read_until_stable(|| Ok(reads.next().expect("read"))); + let unstable = platform::read_until_stable(|| Ok(changing.next().expect("read"))); + + assert_eq!(stable, Ok(vec![11, 12, 13])); + assert_eq!( + unstable, + Err(ReadError::Unavailable(platform::TREE_CHANGING)) + ); } #[cfg(target_os = "linux")] #[test] - fn linux_tree_read_with_children_files_and_unreadable_process_is_unavailable() { - let child = linux_children_tree(Ok(vec![11]), Err(platform::ReadError::Unreadable)); - let host_children = linux_children_tree( - Err(platform::ReadError::Unreadable), - Ok(linux_stat(10, 9, 3)), - ); - let host_gone = - linux_children_tree(Err(platform::ReadError::Gone), Ok(linux_stat(10, 9, 3))); + fn linux_process_scan_groups_children_and_needs_every_parent() { + let parents = BTreeMap::from([ + (10, Ok(1)), + (11, Ok(10)), + (12, Ok(10)), + (13, Err(ReadError::Gone)), + ]); + let parent_of = |pid| parents[&pid]; + + let scanned = platform::children_by_parent(vec![10, 11, 12, 13], parent_of).expect("scan"); + let incomplete = platform::children_by_parent(vec![10, 50], |pid| match pid { + 50 => Err(ReadError::Unavailable(PROCESS_UNREADABLE)), + _ => Ok(1), + }); - assert_eq!(child.unwrap_err(), reason::PROCESS_UNREADABLE); - assert_eq!(host_children.unwrap_err(), reason::PROCESS_UNREADABLE); - assert_eq!(host_gone.unwrap_err(), reason::HOST_EXITED); + assert_eq!(scanned, BTreeMap::from([(1, vec![10]), (10, vec![11, 12])])); + assert_eq!(incomplete.unwrap_err(), platform::TABLE_UNREADABLE); } - /// The second read finds a child the first skipped; the third agrees. + /// The fallback for a kernel without children files reads the tree too. #[cfg(target_os = "linux")] #[test] - fn linux_children_read_waits_for_two_reads_that_agree() { - let mut reads = vec![vec![11, 13], vec![11, 12, 13], vec![11, 12, 13]].into_iter(); - let mut changing = (0..).map(|n| vec![n]); + fn linux_tree_read_with_process_scan_reads_this_process_tree() { + let host = LocalHostProcess::new(std::process::id(), SystemTime::now()); + let children = platform::scan_children().expect("scan"); - let stable = platform::read_until_stable(|| Ok(reads.next().expect("read"))); - let unstable = platform::read_until_stable(|| Ok(changing.next().expect("read"))); + let read = platform::read_tree_from(children, host.pid, host.identity.expect("identity")) + .expect("tree read"); - assert_eq!(stable, Ok(vec![11, 12, 13])); - assert_eq!(unstable, Err(platform::ReadError::Unreadable)); + assert!(read.process_count >= 1); + assert!(read.resident_bytes > 0); } #[cfg(target_os = "macos")] diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index ae9926b55..cec4f2607 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -698,20 +698,20 @@ print(snapshot.process_count, snapshot.cpu_time_seconds) | `observed_at` | When NeMo Fabric read the values, as an RFC 3339 UTC timestamp. | | `resident_memory_bytes` | Sum of the resident set sizes of the live processes. Pages shared between processes count once per process. | | `process_count` | Live processes, including the adapter host. Exited processes that are not yet reaped are not counted. | -| `cpu_time_seconds` | CPU time used since the adapter host started, including exited descendants (on macOS, only once they are waited for). It can drop when a descendant leaves the tree. | +| `cpu_time_seconds` | CPU time used since the adapter host started, including exited descendants. It can drop when a descendant leaves the tree. | The scope is fixed: the local adapter host process that NeMo Fabric started for the runtime, plus its descendants. For a runtime started on a shared service, the service process is not included. A descendant that starts a new session or process group still counts while its parent chain leads to the host. A process whose parent exited is reparented outside the tree and no longer counts. The -snapshot reads `/proc` on Linux and `proc_pid_rusage` on macOS. On Linux it -finds the tree from each process's `/proc//task//children` list, so it -reads only the runtime's own processes. On macOS, and on a Linux kernel without -those lists, it reads the parent ID of every process on the machine to find the -tree, and macOS also reads each process's start time to detect a reused PID. -The snapshot exposes only the fields above, never command lines, environment -variables, credentials, or processes outside the runtime's tree. +snapshot finds the tree from each process's list of children: Linux reads +`/proc//task//children` and `/proc//stat`, and macOS reads +`proc_listchildpids` and `proc_pid_rusage`. Both read only the runtime's own +processes. On a Linux kernel without those children lists, the snapshot reads +the parent ID of every process on the machine to find the tree. The snapshot +exposes only the fields above, never command lines, environment variables, +credentials, or processes outside the runtime's tree. When NeMo Fabric cannot read the values, it raises an error instead of returning zeros or leaving a process out: @@ -728,8 +728,8 @@ once after the adapter returned the result. It is `None` when the read failed; a failed read never fails the invocation. To estimate one invocation's CPU time, take a snapshot before it and subtract its `cpu_time_seconds` from the result's snapshot. CPU time can drop when a descendant is reparented out of the -tree, and on macOS while an exited descendant is not yet waited for, so treat a -difference as an estimate and a negative difference as unknown. +tree, so treat a difference as an estimate and a negative difference as +unknown. Snapshots observe resources only. They do not enforce limits, stop runtimes, or queue requests. Use the operating system or execution environment, such as diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 072be7097..504d6b129 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -283,7 +283,7 @@ "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", "properties": { "cpu_time_seconds": { - "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants (on macOS, only once they are waited for).\nIt can drop when a descendant leaves the tree, and on macOS while an\nexited descendant is not yet waited for.", + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", "format": "double", "minimum": 0.0, "type": "number" diff --git a/schemas/sdk/runtime-resource-snapshot.schema.json b/schemas/sdk/runtime-resource-snapshot.schema.json index 8a5bf94ad..0b3ac2916 100644 --- a/schemas/sdk/runtime-resource-snapshot.schema.json +++ b/schemas/sdk/runtime-resource-snapshot.schema.json @@ -3,7 +3,7 @@ "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", "properties": { "cpu_time_seconds": { - "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants (on macOS, only once they are waited for).\nIt can drop when a descendant leaves the tree, and on macOS while an\nexited descendant is not yet waited for.", + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", "format": "double", "minimum": 0.0, "type": "number" diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index c1ddf0323..c6a3cb682 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1849,8 +1849,8 @@ class RuntimeResourceSnapshot(FabricMapping): Pages shared between processes count once per process. process_count: Live processes in the tree, including the adapter host. cpu_time_seconds: CPU time used since the adapter host started, - including exited descendants (on macOS, only once they are - waited for). It can drop when a descendant leaves the tree. + including exited descendants. It can drop when a descendant + leaves the tree. """ runtime_id: str From ced92e81553734430c18bade5c59352aa13f408f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:08:54 -0700 Subject: [PATCH 011/159] docs: explain unavailable and unsupported resource snapshots On macOS, a descendant that runs as another user, including the setuid programs ps, top, and sudo, cannot be read, so the snapshot is unavailable while it runs. The docs now name those programs. Unsupported is also returned for the http and native_plugin adapter kinds, which the error docs left out. RunResult.resource_snapshot is None when the final read fails, and the docs now say that the snapshot call returns the error behind it. The SDK API inventory lists Runtime.resource_snapshot(). Signed-off-by: mnajafian-nv --- crates/fabric-core/src/error.rs | 3 ++- crates/fabric-core/src/runtime.rs | 7 ++++--- crates/fabric-python/src/lib.rs | 2 +- docs/sdk/python.mdx | 15 +++++++++------ schemas/sdk/run-result.schema.json | 2 +- .../nemo-fabric-runtime/src/nemo_fabric/errors.py | 6 +++++- .../src/nemo_fabric/runtime.py | 5 +++-- .../nemo-fabric-runtime/src/nemo_fabric/types.py | 3 ++- .../references/results-and-errors.md | 4 ++-- .../references/sdk-api-inventory.md | 1 + 10 files changed, 30 insertions(+), 18 deletions(-) diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index fb4075af0..2665c8a61 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -287,7 +287,8 @@ pub enum FabricError { /// Why the runtime cannot be observed. reason: String, }, - /// The runtime's environment provider or operating system has no resource reader. + /// The runtime's adapter kind, environment provider, or operating system has + /// no resource reader. #[error("resource observation is unsupported for runtime `{runtime_id}`: {reason}")] ResourceObservationUnsupported { /// Runtime handle id. diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 589cdbd78..2ea389429 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -142,7 +142,8 @@ pub struct RunResult { #[serde(default, skip_serializing_if = "Option::is_none")] pub usage: Option, /// Resource snapshot of the runtime, read once after the adapter returned this - /// result. Absent when the runtime's resources could not be observed. + /// result. Absent when the runtime's resources could not be observed; + /// `runtime_resource_snapshot` returns the error that explains it. #[serde(default, skip_serializing_if = "Option::is_none")] pub resource_snapshot: Option, /// Artifacts produced or collected by this run. @@ -1179,8 +1180,8 @@ pub fn invoke_runtime( /// [`FabricError::ResourceObservationUnavailable`] when the runtime stopped, is /// stopping, its host exited, or a live process in its tree or the process /// table cannot be read, and -/// [`FabricError::ResourceObservationUnsupported`] when the environment provider -/// or operating system has no resource reader. +/// [`FabricError::ResourceObservationUnsupported`] when the adapter kind, +/// environment provider, or operating system has no resource reader. pub fn runtime_resource_snapshot( plan: &RunPlan, runtime: &RuntimeHandle, diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index c068a184a..0d65c3d0b 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -34,7 +34,7 @@ pyo3::create_exception!( _native, ResourceObservationUnsupportedError, PyRuntimeError, - "Raised when a runtime's environment provider or operating system has no resource reader." + "Raised when a runtime's adapter kind, environment provider, or operating system has no resource reader." ); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index cec4f2607..21a34673f 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -718,14 +718,17 @@ returning zeros or leaving a process out: - `ResourceObservationUnavailableError` when the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the - process table cannot be read, for example a macOS descendant that runs as - another user. -- `ResourceObservationUnsupportedError` when the runtime's environment provider - or operating system has no resource reader. Windows raises this error. + process table cannot be read. For example, on macOS, the snapshot is + unavailable while a descendant runs as another user, including setuid + programs such as `ps`, `top`, and `sudo`. +- `ResourceObservationUnsupportedError` when the runtime's adapter kind, + environment provider, or operating system has no resource reader. The `http` + and `native_plugin` adapter kinds and Windows raise this error. Each `RunResult` from a local runtime also carries `resource_snapshot`, read once after the adapter returned the result. It is `None` when the read failed; -a failed read never fails the invocation. To estimate one invocation's CPU time, +a failed read never fails the invocation, and `Runtime.resource_snapshot()` +raises the error that explains it. To estimate one invocation's CPU time, take a snapshot before it and subtract its `cpu_time_seconds` from the result's snapshot. CPU time can drop when a descendant is reparented out of the tree, so treat a difference as an estimate and a negative difference as @@ -1121,7 +1124,7 @@ All public SDK errors inherit from `FabricError`. | `FabricStateError` | Invalid runtime state transition, such as invoking after stop or starting overlapping invocations. | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | | `ResourceObservationUnavailableError` | `Runtime.resource_snapshot()` cannot read the runtime because it stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read. Subclass of `FabricRuntimeError`. | -| `ResourceObservationUnsupportedError` | The runtime's environment provider or operating system has no resource reader. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnsupportedError` | The runtime's adapter kind, environment provider, or operating system has no resource reader. Subclass of `FabricRuntimeError`. | Consumers own job-level retries and rollout-level failure policy. Core host-operation deadlines (`host_timeout`) become Python `TimeoutError`, then diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 504d6b129..78d09c973 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -415,7 +415,7 @@ "type": "null" } ], - "description": "Resource snapshot of the runtime, read once after the adapter returned this\nresult. Absent when the runtime's resources could not be observed." + "description": "Resource snapshot of the runtime, read once after the adapter returned this\nresult. Absent when the runtime's resources could not be observed;\n`runtime_resource_snapshot` returns the error that explains it." }, "runtime_id": { "description": "Runtime handle id.", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index 6f6b8e0c9..85671e577 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -90,4 +90,8 @@ class ResourceObservationUnavailableError(FabricRuntimeError): class ResourceObservationUnsupportedError(FabricRuntimeError): - """The runtime's environment provider or operating system has no resource reader.""" + """Runtime resources have no reader. + + The runtime's adapter kind, environment provider, or operating system has + no resource reader. + """ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index a421eb3ff..723084da0 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -751,8 +751,9 @@ async def resource_snapshot(self) -> RuntimeResourceSnapshot: ResourceObservationUnavailableError: If the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. - ResourceObservationUnsupportedError: If the runtime's environment - provider or operating system has no resource reader. + ResourceObservationUnsupportedError: If the runtime's adapter + kind, environment provider, or operating system has no + resource reader. FabricNativeUnavailableError: If the native extension is missing. FabricRuntimeError: If the native snapshot call fails otherwise. """ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index c6a3cb682..f98c4e4f8 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1921,7 +1921,8 @@ class RunResult(FabricMapping): usage: Normalized invocation usage, or ``None`` when unavailable. resource_snapshot: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's - resources could not be observed. + resources could not be observed. ``Runtime.resource_snapshot()`` + raises the error that explains a ``None``. artifacts: Normalized artifact manifest. telemetry: Ordered telemetry references. events: Ordered lifecycle and invocation events. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 2500fa488..93321c87d 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -17,7 +17,7 @@ Every invocation that reaches the adapter boundary returns a normalized | `error` | Structured failure metadata when available. Branch on `status`, not on this field. | | `output` | Harness output normalized to the configured output schema. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | -| `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed. | +| `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed; `runtime.resource_snapshot()` raises the error that explains it. | | `artifacts` | Output files, logs, patches, and other materialized references. | | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | @@ -84,7 +84,7 @@ Refer to the [errors reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/ | `FabricStateError` | Invalid runtime state transition (invoking after stop, overlapping invocations). | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | | `ResourceObservationUnavailableError` | A resource snapshot cannot be read because the runtime stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read. Subclass of `FabricRuntimeError`. | -| `ResourceObservationUnsupportedError` | The runtime's environment provider or operating system has no resource reader. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnsupportedError` | The runtime's adapter kind, environment provider, or operating system has no resource reader. Subclass of `FabricRuntimeError`. | ## Cleanup And Resilience diff --git a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md index 3e2f09c99..b184af10d 100644 --- a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md +++ b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md @@ -47,6 +47,7 @@ The following table lists the `Runtime` members for driving a stateful runtime. | `invoke(*, input=... \| request=...)` | Yes | One turn on an active runtime. One active invocation at a time; overlap raises `FabricStateError`. | | `invoke_openai_stream(*, input=... \| request=...)` | No | Start exactly one descriptor-gated native invocation and return an async `OpenAIInvokeStream` of `chat.completion.chunk` mappings. Await `stream.result()` for the separate terminal `RunResult`. | | `invoke_stream(*, input=... \| request=...)` | No | Start one NeMo Relay turn and return an async `InvokeStream` of raw ATOF records. Await `stream.result()` for the terminal `RunResult`. | +| `resource_snapshot()` | Yes | Timestamped memory, process count, and CPU time of the local adapter host and its descendants. Does not wait for an active invocation. | | `stop()` | Yes | Stop the runtime. Called automatically by `async with`. | | `status` | No | `RuntimeStatus`: `ACTIVE`, `STOPPED`, or `FAILED`. | | `supports_openai_streaming` | No | `True` when the selected descriptor declares `capabilities.streaming` for native OpenAI Chat Completions chunks. | From 8e52556e491f97940061855be8eb2323e1aec47c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:08:55 -0700 Subject: [PATCH 012/159] test: name the snapshot mapping test for both checks it makes The test checks that a snapshot mapping round-trips and that invalid fields raise FabricConfigError, so its name now says both. Signed-off-by: mnajafian-nv --- tests/python/test_runtime_resource_snapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/test_runtime_resource_snapshot.py b/tests/python/test_runtime_resource_snapshot.py index db63e546a..d14f4354a 100644 --- a/tests/python/test_runtime_resource_snapshot.py +++ b/tests/python/test_runtime_resource_snapshot.py @@ -91,7 +91,7 @@ def _runtime_with_native(native_module: Any) -> Runtime: ) -def test_runtime_resource_snapshot_with_invalid_field_raises_config_error(): +def test_runtime_resource_snapshot_round_trips_and_rejects_invalid_fields(): snapshot = RuntimeResourceSnapshot.from_mapping(_SNAPSHOT) assert snapshot.cpu_time_seconds == 1.25 assert snapshot.to_mapping() == _SNAPSHOT From 2e9cfed8502b2e5fbf9234ba1eccd237f2a13836 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:09:33 -0700 Subject: [PATCH 013/159] docs: regenerate API references for resource snapshot docs Picks up the macOS reader, Unsupported adapter kind, and RunResult snapshot wording from the Rust and Python docstrings. Signed-off-by: mnajafian-nv --- docs/reference/api/python-library-reference/index.md | 2 +- .../api/python-library-reference/nemo_fabric.errors.md | 4 +++- .../api/python-library-reference/nemo_fabric.runtime.md | 2 +- .../api/python-library-reference/nemo_fabric.types.md | 4 ++-- .../nemo-fabric-core/error/enum-fabricerror.mdx | 2 +- .../nemo-fabric-core/resource-snapshot/index.mdx | 6 +++--- .../resource-snapshot/struct-runtimeresourcesnapshot.mdx | 2 +- .../runtime/fn-runtime-resource-snapshot.mdx | 2 +- .../nemo-fabric-core/runtime/struct-runresult.mdx | 2 +- 9 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index 24fb1d403..3ab015d41 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -86,7 +86,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`errors.FabricRuntimeError`](./nemo_fabric.errors.md#class-fabricruntimeerror): Failure while starting, invoking, stopping, or otherwise driving a runtime. - [`errors.FabricStateError`](./nemo_fabric.errors.md#class-fabricstateerror): Operation rejected because a local runtime is in the wrong state. - [`errors.ResourceObservationUnavailableError`](./nemo_fabric.errors.md#class-resourceobservationunavailableerror): Runtime resources cannot be read now. -- [`errors.ResourceObservationUnsupportedError`](./nemo_fabric.errors.md#class-resourceobservationunsupportederror): The runtime's environment provider or operating system has no resource reader. +- [`errors.ResourceObservationUnsupportedError`](./nemo_fabric.errors.md#class-resourceobservationunsupportederror): Runtime resources have no reader. ## Functions diff --git a/docs/reference/api/python-library-reference/nemo_fabric.errors.md b/docs/reference/api/python-library-reference/nemo_fabric.errors.md index e3acafc12..8acf76f75 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.errors.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.errors.md @@ -321,7 +321,9 @@ Initialize a structured NeMo Fabric exception. ## class `ResourceObservationUnsupportedError` -The runtime's environment provider or operating system has no resource reader. +Runtime resources have no reader. + +The runtime's adapter kind, environment provider, or operating system has no resource reader. ### Inheritance diff --git a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md index 98ed49750..73fc8cd5b 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md @@ -198,7 +198,7 @@ The snapshot covers the local adapter host process and its live descendants. It **Raises:** - `ResourceObservationUnavailableError`: If the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. - - `ResourceObservationUnsupportedError`: If the runtime's environment provider or operating system has no resource reader. + - `ResourceObservationUnsupportedError`: If the runtime's adapter kind, environment provider, or operating system has no resource reader. - `FabricNativeUnavailableError`: If the native extension is missing. - `FabricRuntimeError`: If the native snapshot call fails otherwise. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 623c34593..f009db3cf 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1259,7 +1259,7 @@ Memory, process count, and CPU time cover the adapter host process and its live - `observed_at`: When Fabric read these values, as RFC 3339 UTC. - `resident_memory_bytes`: Sum of the live processes' resident set sizes. Pages shared between processes count once per process. - `process_count`: Live processes in the tree, including the adapter host. - - `cpu_time_seconds`: CPU time used since the adapter host started, including exited descendants (on macOS, only once they are waited for). It can drop when a descendant leaves the tree. + - `cpu_time_seconds`: CPU time used since the adapter host started, including exited descendants. It can drop when a descendant leaves the tree. ### Fields @@ -1348,7 +1348,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - `error`: Structured failure, or ``None`` on success. - `usage`: Normalized invocation usage, or ``None`` when unavailable. - - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. + - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. ``Runtime.resource_snapshot()`` raises the error that explains a ``None``. - `artifacts`: Normalized artifact manifest. - `telemetry`: Ordered telemetry references. - `events`: Ordered lifecycle and invocation events. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index d4082937d..805b8e7f7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -517,7 +517,7 @@ Why the runtime cannot be observed.
-The runtime's environment provider or operating system has no resource reader. +The runtime's adapter kind, environment provider, or operating system has no resource reader. #### Fields diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index eefcd4bb4..2ba0dad07 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -12,11 +12,11 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Resource snapshots of a runtime's local adapter host and its descendants. -A snapshot reads the process tree rooted at the adapter host process that NeMo Fabric started for a runtime. Each read visits a parent before its children. CPU time adds every process's own CPU time and the CPU time of the children it has already waited for, so work done by finished descendants still counts. A child reaped between the read of its parent and its own read is missed by that one snapshot and is never counted twice. CPU time can drop between snapshots when a descendant is reparented out of the tree, and on macOS while an exited descendant is not yet waited for, because macOS cannot list an unreaped process. +A snapshot reads the process tree rooted at the adapter host process that NeMo Fabric started for a runtime, each parent before its children. CPU time adds every process's own CPU time and the CPU time of the children it has already waited for. An exited child that is not yet waited for counts through its own CPU time, so work done by finished descendants still counts. A child reaped between the read of its parent and its own read is missed by that one snapshot and is never counted twice. CPU time can drop between snapshots when a descendant is reparented out of the tree. -A live process in the tree that cannot be read, or a process table that cannot be listed in full, makes the snapshot unavailable instead of leaving the process out. +Linux reads `/proc//task//children` and `/proc//stat`. On a kernel without those children files, it reads every process's parent ID once per snapshot to find the tree. macOS reads `proc_listchildpids` and `proc_pid_rusage`. A snapshot exposes only its own fields: never command lines, environment variables, open files, or processes outside the tree. -Linux reads `/proc//stat` and macOS reads `proc_pid_rusage`. To find the tree, the macOS reader, and the Linux reader on a kernel without `/proc//task//children`, also read other processes' parent IDs. A snapshot exposes only its own fields: never command lines, environment variables, open files, or processes outside the tree. +A live process in the tree that cannot be read, or a process table that the Linux fallback cannot read in full, makes the snapshot unavailable instead of leaving the process out. ## Structs diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx index ba0517771..aff244954 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx @@ -40,7 +40,7 @@ Live processes in the tree, including the adapter host. Exited processes that ar ### `cpu_time_seconds: f64` -CPU time consumed by the tree since the adapter host started, in seconds, including exited descendants (on macOS, only once they are waited for). It can drop when a descendant leaves the tree, and on macOS while an exited descendant is not yet waited for. +CPU time consumed by the tree since the adapter host started, in seconds, including exited descendants. It can drop when a descendant leaves the tree. ## Trait Implementations diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx index 745457bd3..1bf68862e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx @@ -13,4 +13,4 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Read a timestamped resource snapshot of a started runtime. -The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, or a live process in its tree or the process table cannot be read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the environment provider or operating system has no resource reader. +The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, or a live process in its tree or the process table cannot be read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the adapter kind, environment provider, or operating system has no resource reader. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 3e05964a5..e8aba8ae4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -61,7 +61,7 @@ Normalized invocation usage when reported by the adapter target. ### `resource_snapshot: Option` -Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed. +Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed; `runtime_resource_snapshot` returns the error that explains it. ### `artifacts: ArtifactManifest` From e46f641b9eff15fd5b931186e5a5a3360c2d5da8 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:58:42 -0700 Subject: [PATCH 014/159] docs: stop saying a later snapshot explains a missing result snapshot RunResult.resource_snapshot docs said that runtime_resource_snapshot returns the error behind a None. It does not: the final read's error is not kept, run_plan stops the runtime before a caller can ask, and a transient cause can be gone by the next read. The field now says only that it is absent when the runtime's resources could not be observed. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 3 +-- docs/sdk/python.mdx | 3 +-- schemas/sdk/run-result.schema.json | 2 +- sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py | 3 +-- skills/nemo-fabric-integrate/references/results-and-errors.md | 2 +- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 2ea389429..ec9daf329 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -142,8 +142,7 @@ pub struct RunResult { #[serde(default, skip_serializing_if = "Option::is_none")] pub usage: Option, /// Resource snapshot of the runtime, read once after the adapter returned this - /// result. Absent when the runtime's resources could not be observed; - /// `runtime_resource_snapshot` returns the error that explains it. + /// result. Absent when the runtime's resources could not be observed. #[serde(default, skip_serializing_if = "Option::is_none")] pub resource_snapshot: Option, /// Artifacts produced or collected by this run. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 21a34673f..829ab85e5 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -727,8 +727,7 @@ returning zeros or leaving a process out: Each `RunResult` from a local runtime also carries `resource_snapshot`, read once after the adapter returned the result. It is `None` when the read failed; -a failed read never fails the invocation, and `Runtime.resource_snapshot()` -raises the error that explains it. To estimate one invocation's CPU time, +a failed read never fails the invocation. To estimate one invocation's CPU time, take a snapshot before it and subtract its `cpu_time_seconds` from the result's snapshot. CPU time can drop when a descendant is reparented out of the tree, so treat a difference as an estimate and a negative difference as diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 78d09c973..504d6b129 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -415,7 +415,7 @@ "type": "null" } ], - "description": "Resource snapshot of the runtime, read once after the adapter returned this\nresult. Absent when the runtime's resources could not be observed;\n`runtime_resource_snapshot` returns the error that explains it." + "description": "Resource snapshot of the runtime, read once after the adapter returned this\nresult. Absent when the runtime's resources could not be observed." }, "runtime_id": { "description": "Runtime handle id.", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index f98c4e4f8..c6a3cb682 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1921,8 +1921,7 @@ class RunResult(FabricMapping): usage: Normalized invocation usage, or ``None`` when unavailable. resource_snapshot: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's - resources could not be observed. ``Runtime.resource_snapshot()`` - raises the error that explains a ``None``. + resources could not be observed. artifacts: Normalized artifact manifest. telemetry: Ordered telemetry references. events: Ordered lifecycle and invocation events. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 93321c87d..def96b269 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -17,7 +17,7 @@ Every invocation that reaches the adapter boundary returns a normalized | `error` | Structured failure metadata when available. Branch on `status`, not on this field. | | `output` | Harness output normalized to the configured output schema. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | -| `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed; `runtime.resource_snapshot()` raises the error that explains it. | +| `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. `None` when the read failed. | | `artifacts` | Output files, logs, patches, and other materialized references. | | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | From f4cba230e693d497ff38b82f2e655f0c392d29c9 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:58:50 -0700 Subject: [PATCH 015/159] docs: list a changing process tree and hidepid among snapshot limits The Linux reader returns ResourceObservationUnavailable when 4 reads of a process's children never give two equal results in a row, because the runtime's process tree kept changing during the read. The public error docs listed only a stopped runtime, an exited host, and unreadable processes. They now list the changing tree too. On Linux with /proc mounted hidepid=invisible, another user's descendant reads as gone, so the snapshot leaves it out instead of reporting it unavailable. The module doc and the Python guide now say so. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/error.rs | 5 +++-- crates/fabric-core/src/resource_snapshot.rs | 4 +++- crates/fabric-core/src/runtime.rs | 4 ++-- crates/fabric-python/src/lib.rs | 2 +- docs/sdk/python.mdx | 13 ++++++++----- .../nemo-fabric-runtime/src/nemo_fabric/errors.py | 5 +++-- .../nemo-fabric-runtime/src/nemo_fabric/runtime.py | 5 +++-- .../references/results-and-errors.md | 2 +- 8 files changed, 24 insertions(+), 16 deletions(-) diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index 2665c8a61..c8bda1957 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -278,8 +278,9 @@ pub enum FabricError { adapter_kind: AdapterKind, }, /// A runtime's resources cannot be read now: the runtime stopped, is - /// stopping, its adapter host process exited, or a live process in its - /// tree or the process table cannot be read. + /// stopping, its adapter host process exited, a live process in its tree + /// or the process table cannot be read, or the process tree kept changing + /// during the read. #[error("resource observation is unavailable for runtime `{runtime_id}`: {reason}")] ResourceObservationUnavailable { /// Runtime handle id. diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 3c4582cb6..09175db45 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -20,7 +20,9 @@ //! //! A live process in the tree that cannot be read, or a process table that //! the Linux fallback cannot read in full, makes the snapshot unavailable -//! instead of leaving the process out. +//! instead of leaving the process out. On Linux with `/proc` mounted +//! `hidepid=invisible`, another user's descendant is invisible and is not +//! counted. use std::time::{SystemTime, UNIX_EPOCH}; diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index ec9daf329..b4039f16b 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1177,8 +1177,8 @@ pub fn invoke_runtime( /// The snapshot covers the runtime's local adapter host process and its live /// descendants. It does not wait for a running invocation. It returns /// [`FabricError::ResourceObservationUnavailable`] when the runtime stopped, is -/// stopping, its host exited, or a live process in its tree or the process -/// table cannot be read, and +/// stopping, its host exited, a live process in its tree or the process table +/// cannot be read, or the process tree kept changing during the read, and /// [`FabricError::ResourceObservationUnsupported`] when the adapter kind, /// environment provider, or operating system has no resource reader. pub fn runtime_resource_snapshot( diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 0d65c3d0b..780ae2f08 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -28,7 +28,7 @@ pyo3::create_exception!( _native, ResourceObservationUnavailableError, PyRuntimeError, - "Raised when a runtime stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read." + "Raised when a runtime stopped, is stopping, its adapter host exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read." ); pyo3::create_exception!( _native, diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 829ab85e5..13b5b4d38 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -717,14 +717,17 @@ When NeMo Fabric cannot read the values, it raises an error instead of returning zeros or leaving a process out: - `ResourceObservationUnavailableError` when the runtime stopped, is stopping, - its adapter host process exited, or a live process in its tree or the - process table cannot be read. For example, on macOS, the snapshot is - unavailable while a descendant runs as another user, including setuid - programs such as `ps`, `top`, and `sudo`. + its adapter host process exited, a live process in its tree or the process + table cannot be read, or the process tree kept changing during the read. For + example, on macOS, the snapshot is unavailable while a descendant runs as + another user, including setuid programs such as `ps`, `top`, and `sudo`. - `ResourceObservationUnsupportedError` when the runtime's adapter kind, environment provider, or operating system has no resource reader. The `http` and `native_plugin` adapter kinds and Windows raise this error. +On Linux with `/proc` mounted `hidepid=invisible`, a descendant owned by +another user is invisible and is not counted. + Each `RunResult` from a local runtime also carries `resource_snapshot`, read once after the adapter returned the result. It is `None` when the read failed; a failed read never fails the invocation. To estimate one invocation's CPU time, @@ -1122,7 +1125,7 @@ All public SDK errors inherit from `FabricError`. | `FabricRuntimeError` | Runtime startup, invocation, or shutdown failed before a normalized result could be returned. | | `FabricStateError` | Invalid runtime state transition, such as invoking after stop or starting overlapping invocations. | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | -| `ResourceObservationUnavailableError` | `Runtime.resource_snapshot()` cannot read the runtime because it stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnavailableError` | `Runtime.resource_snapshot()` cannot read the runtime because it stopped, is stopping, its adapter host exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read. Subclass of `FabricRuntimeError`. | | `ResourceObservationUnsupportedError` | The runtime's adapter kind, environment provider, or operating system has no resource reader. Subclass of `FabricRuntimeError`. | Consumers own job-level retries and rollout-level failure policy. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index 85671e577..fae1fdfb0 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -84,8 +84,9 @@ class FabricNativeUnavailableError(FabricRuntimeError): class ResourceObservationUnavailableError(FabricRuntimeError): """Runtime resources cannot be read now. - The runtime stopped, is stopping, its adapter host process exited, or a - live process in its tree or the process table cannot be read. + The runtime stopped, is stopping, its adapter host process exited, a live + process in its tree or the process table cannot be read, or the process + tree kept changing during the read. """ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 723084da0..dc01e75c7 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -749,8 +749,9 @@ async def resource_snapshot(self) -> RuntimeResourceSnapshot: Raises: ResourceObservationUnavailableError: If the runtime stopped, is - stopping, its adapter host process exited, or a live process - in its tree or the process table cannot be read. + stopping, its adapter host process exited, a live process in + its tree or the process table cannot be read, or the process + tree kept changing during the read. ResourceObservationUnsupportedError: If the runtime's adapter kind, environment provider, or operating system has no resource reader. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index def96b269..0c790fc78 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -83,7 +83,7 @@ Refer to the [errors reference](https://github.com/NVIDIA/NeMo-Fabric/blob/main/ | `FabricRuntimeError` | Startup, invocation, or shutdown failed before a normalized result. | | `FabricStateError` | Invalid runtime state transition (invoking after stop, overlapping invocations). | | `FabricNativeUnavailableError` | Native extension is not installed or importable. | -| `ResourceObservationUnavailableError` | A resource snapshot cannot be read because the runtime stopped, is stopping, its adapter host exited, or a live process in its tree or the process table cannot be read. Subclass of `FabricRuntimeError`. | +| `ResourceObservationUnavailableError` | A resource snapshot cannot be read because the runtime stopped, is stopping, its adapter host exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read. Subclass of `FabricRuntimeError`. | | `ResourceObservationUnsupportedError` | The runtime's adapter kind, environment provider, or operating system has no resource reader. Subclass of `FabricRuntimeError`. | ## Cleanup And Resilience From 0371c19a5ec5c0a09ca028a4362cd994e8155117 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 02:58:50 -0700 Subject: [PATCH 016/159] docs: regenerate API references for resource snapshot error docs Picks up the corrected RunResult.resource_snapshot field doc, the changing-tree unavailable reason, and the hidepid note from the Rust and Python docstrings. Signed-off-by: mnajafian-nv --- .../api/python-library-reference/nemo_fabric.errors.md | 2 +- .../api/python-library-reference/nemo_fabric.runtime.md | 2 +- .../reference/api/python-library-reference/nemo_fabric.types.md | 2 +- .../nemo-fabric-core/error/enum-fabricerror.mdx | 2 +- .../nemo-fabric-core/resource-snapshot/index.mdx | 2 +- .../nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx | 2 +- .../nemo-fabric-core/runtime/struct-runresult.mdx | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/reference/api/python-library-reference/nemo_fabric.errors.md b/docs/reference/api/python-library-reference/nemo_fabric.errors.md index 8acf76f75..413945d97 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.errors.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.errors.md @@ -281,7 +281,7 @@ Initialize a structured NeMo Fabric exception. Runtime resources cannot be read now. -The runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. +The runtime stopped, is stopping, its adapter host process exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read. ### Inheritance diff --git a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md index 73fc8cd5b..cb58196dd 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md @@ -197,7 +197,7 @@ The snapshot covers the local adapter host process and its live descendants. It **Raises:** - - `ResourceObservationUnavailableError`: If the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. + - `ResourceObservationUnavailableError`: If the runtime stopped, is stopping, its adapter host process exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read. - `ResourceObservationUnsupportedError`: If the runtime's adapter kind, environment provider, or operating system has no resource reader. - `FabricNativeUnavailableError`: If the native extension is missing. - `FabricRuntimeError`: If the native snapshot call fails otherwise. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index f009db3cf..59c7e7c06 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1348,7 +1348,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - `error`: Structured failure, or ``None`` on success. - `usage`: Normalized invocation usage, or ``None`` when unavailable. - - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. ``Runtime.resource_snapshot()`` raises the error that explains a ``None``. + - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. - `artifacts`: Normalized artifact manifest. - `telemetry`: Ordered telemetry references. - `events`: Ordered lifecycle and invocation events. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 805b8e7f7..ae0e4009e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -501,7 +501,7 @@ Adapter kind.
-A runtime's resources cannot be read now: the runtime stopped, is stopping, its adapter host process exited, or a live process in its tree or the process table cannot be read. +A runtime's resources cannot be read now: the runtime stopped, is stopping, its adapter host process exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read. #### Fields diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index 2ba0dad07..e55480dcc 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -16,7 +16,7 @@ A snapshot reads the process tree rooted at the adapter host process that NeMo F Linux reads `/proc//task//children` and `/proc//stat`. On a kernel without those children files, it reads every process's parent ID once per snapshot to find the tree. macOS reads `proc_listchildpids` and `proc_pid_rusage`. A snapshot exposes only its own fields: never command lines, environment variables, open files, or processes outside the tree. -A live process in the tree that cannot be read, or a process table that the Linux fallback cannot read in full, makes the snapshot unavailable instead of leaving the process out. +A live process in the tree that cannot be read, or a process table that the Linux fallback cannot read in full, makes the snapshot unavailable instead of leaving the process out. On Linux with `/proc` mounted `hidepid=invisible`, another user's descendant is invisible and is not counted. ## Structs diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx index 1bf68862e..4275ce2f6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx @@ -13,4 +13,4 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Read a timestamped resource snapshot of a started runtime. -The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, or a live process in its tree or the process table cannot be read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the adapter kind, environment provider, or operating system has no resource reader. +The snapshot covers the runtime's local adapter host process and its live descendants. It does not wait for a running invocation. It returns [`FabricError::ResourceObservationUnavailable`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnavailable) when the runtime stopped, is stopping, its host exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read, and [`FabricError::ResourceObservationUnsupported`](../error/enum-fabricerror.mdx#variant.ResourceObservationUnsupported) when the adapter kind, environment provider, or operating system has no resource reader. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index e8aba8ae4..3e05964a5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -61,7 +61,7 @@ Normalized invocation usage when reported by the adapter target. ### `resource_snapshot: Option` -Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed; `runtime_resource_snapshot` returns the error that explains it. +Resource snapshot of the runtime, read once after the adapter returned this result. Absent when the runtime's resources could not be observed. ### `artifacts: ArtifactManifest` From e7665eb0247762799394bc9ae19c438ab2857778 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 04:05:20 -0700 Subject: [PATCH 017/159] refactor: move the RFC 3339 formatter into a shared time module The run record needs the same millisecond RFC 3339 timestamps as resource snapshots. Moving the formatter into crate::time gives core one copy of the civil date algorithm, so a later feature imports it instead of copying it. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/lib.rs | 1 + crates/fabric-core/src/resource_snapshot.rs | 66 +-------------- crates/fabric-core/src/time.rs | 90 +++++++++++++++++++++ 3 files changed, 95 insertions(+), 62 deletions(-) create mode 100644 crates/fabric-core/src/time.rs diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index b20d70a09..72da8560d 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -12,6 +12,7 @@ pub mod error; pub mod resource_snapshot; pub mod runtime; pub mod schema; +mod time; pub use adapter_contract::{ADAPTER_CONTRACT_VERSION, AdapterExtensionPoint}; pub use agent_execution::{ diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 09175db45..a3a3be622 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -24,14 +24,14 @@ //! `hidepid=invisible`, another user's descendant is invisible and is not //! counted. -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::SystemTime; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::error::{FabricError, Result}; +use crate::time::rfc3339_millis; -const MILLIS_PER_DAY: i64 = 86_400_000; const NANOS_PER_SECOND: f64 = 1_000_000_000.0; /// Timestamped resource usage of one runtime's local adapter host process tree. @@ -79,7 +79,7 @@ impl LocalHostProcess { Self { pid, identity: platform::start_identity(pid), - started_at: rfc3339_millis(unix_millis(started_at)), + started_at: rfc3339_millis(started_at), } } } @@ -113,7 +113,7 @@ pub(crate) fn snapshot_local_host( Ok(RuntimeResourceSnapshot { runtime_id: runtime_id.to_string(), started_at: host.started_at.clone(), - observed_at: rfc3339_millis(unix_millis(SystemTime::now())), + observed_at: rfc3339_millis(SystemTime::now()), resident_memory_bytes: read.resident_bytes, process_count: read.process_count, cpu_time_seconds: read.cpu_nanos as f64 / NANOS_PER_SECOND, @@ -255,49 +255,6 @@ mod tree { } } -fn unix_millis(time: SystemTime) -> i64 { - match time.duration_since(UNIX_EPOCH) { - Ok(after) => i64::try_from(after.as_millis()).unwrap_or(i64::MAX), - Err(before) => -i64::try_from(before.duration().as_millis()).unwrap_or(i64::MAX), - } -} - -/// Format Unix milliseconds as RFC 3339 UTC with three fraction digits. -/// -/// Core has no date crate, so this converts days to a civil date with Howard -/// Hinnant's `civil_from_days` algorithm. -fn rfc3339_millis(unix_millis: i64) -> String { - let days = unix_millis.div_euclid(MILLIS_PER_DAY); - let millis_of_day = unix_millis.rem_euclid(MILLIS_PER_DAY); - let (year, month, day) = civil_from_days(days); - let seconds_of_day = millis_of_day / 1000; - format!( - "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{:03}Z", - seconds_of_day / 3600, - seconds_of_day / 60 % 60, - seconds_of_day % 60, - millis_of_day % 1000, - ) -} - -fn civil_from_days(days: i64) -> (i64, i64, i64) { - let z = days + 719_468; - let era = z.div_euclid(146_097); - let day_of_era = z.rem_euclid(146_097); - let year_of_era = - (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; - let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); - let month_index = (5 * day_of_year + 2) / 153; - let day = day_of_year - (153 * month_index + 2) / 5 + 1; - let month = if month_index < 10 { - month_index + 3 - } else { - month_index - 9 - }; - let year = year_of_era + era * 400 + i64::from(month <= 2); - (year, month, day) -} - #[cfg(target_os = "linux")] mod platform { //! `/proc` reads. Each process's children come from @@ -686,21 +643,6 @@ mod tests { }; use super::*; - #[test] - fn rfc3339_millis_formats_fixed_vectors() { - assert_eq!(rfc3339_millis(0), "1970-01-01T00:00:00.000Z"); - assert_eq!( - rfc3339_millis(1_792_000_870_205), - "2026-10-14T18:01:10.205Z" - ); - assert_eq!(rfc3339_millis(951_782_400_000), "2000-02-29T00:00:00.000Z"); - assert_eq!( - rfc3339_millis(4_107_542_399_999), - "2100-02-28T23:59:59.999Z" - ); - assert_eq!(rfc3339_millis(-1), "1969-12-31T23:59:59.999Z"); - } - #[test] fn snapshot_local_host_without_active_host_is_unavailable_or_unsupported() { let error = snapshot_local_host("runtime-1", None).expect_err("no host"); diff --git a/crates/fabric-core/src/time.rs b/crates/fabric-core/src/time.rs new file mode 100644 index 000000000..06ace9edf --- /dev/null +++ b/crates/fabric-core/src/time.rs @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! RFC 3339 UTC timestamps with millisecond precision. +//! +//! Core has no date crate, so this converts Unix milliseconds to a civil date +//! with Howard Hinnant's `civil_from_days` algorithm. + +use std::time::{SystemTime, UNIX_EPOCH}; + +const MILLIS_PER_DAY: i64 = 86_400_000; + +/// Format `time` as RFC 3339 UTC with three fraction digits, such as +/// `2026-10-14T18:01:10.205Z`. +pub(crate) fn rfc3339_millis(time: SystemTime) -> String { + format_unix_millis(unix_millis(time)) +} + +/// Milliseconds since the Unix epoch; negative before it. +fn unix_millis(time: SystemTime) -> i64 { + match time.duration_since(UNIX_EPOCH) { + Ok(after) => i64::try_from(after.as_millis()).unwrap_or(i64::MAX), + Err(before) => -i64::try_from(before.duration().as_millis()).unwrap_or(i64::MAX), + } +} + +fn format_unix_millis(unix_millis: i64) -> String { + let days = unix_millis.div_euclid(MILLIS_PER_DAY); + let millis_of_day = unix_millis.rem_euclid(MILLIS_PER_DAY); + let (year, month, day) = civil_from_days(days); + let seconds_of_day = millis_of_day / 1000; + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{:03}Z", + seconds_of_day / 3600, + seconds_of_day / 60 % 60, + seconds_of_day % 60, + millis_of_day % 1000, + ) +} + +fn civil_from_days(days: i64) -> (i64, i64, i64) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + (year, month, day) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn rfc3339_millis_formats_fixed_vectors() { + assert_eq!(format_unix_millis(0), "1970-01-01T00:00:00.000Z"); + assert_eq!( + format_unix_millis(1_792_000_870_205), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!( + format_unix_millis(951_782_400_000), + "2000-02-29T00:00:00.000Z" + ); + assert_eq!( + format_unix_millis(4_107_542_399_999), + "2100-02-28T23:59:59.999Z" + ); + assert_eq!(format_unix_millis(-1), "1969-12-31T23:59:59.999Z"); + assert_eq!( + rfc3339_millis(UNIX_EPOCH + Duration::from_millis(1_792_000_870_205)), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!( + rfc3339_millis(UNIX_EPOCH - Duration::from_millis(1)), + "1969-12-31T23:59:59.999Z" + ); + } +} From 5e4c29de1584e5c7d61724bd36adc380c7636c9c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 15 Oct 2026 23:35:13 -0700 Subject: [PATCH 018/159] feat: write an opt-in run record for each invocation Core already writes adapter-invocation.json before each request, so a killed run shows which invocation started. Nothing on disk says how long it ran, how it ended, or why it stopped, and RunResult exists only in memory. Add an opt-in, append-only run-record.jsonl per invocation under /.fabric///. runtime.record.enabled turns it on; an artifact root alone does not. Planning rejects an enabled record without an artifact root, with a non-local environment provider, or with an adapter that is not a process or python adapter. Core writes a start line right before it sends the request to the host and an end line after the invocation returns or raises, then syncs the file once after the host lock is released. Both lines are built from typed, allowlisted fields: other context keys appear by name only, override_keys lists override names without values, and the end line keeps the error code, stage and retryable flag but never the message or adapter metadata. A failed write adds a record_write_failed event and never fails the run. The record artifact gets a unique name, as adapter and Relay artifacts do. RecordConfig derives Default (recording off), so a caller can turn recording on without spelling out every record field. The record module is crate-private; the line format is published as the v1alpha1 run-record-line schema. The RFC 3339 formatter lives in a crate-private time module so other core features can share it. Signed-off-by: mnajafian-nv --- crates/fabric-cli/src/presets.rs | 1 + crates/fabric-cli/templates/rust/main.rs.tmpl | 1 + crates/fabric-core/src/config.rs | 161 ++++++ crates/fabric-core/src/lib.rs | 11 +- crates/fabric-core/src/record/invocation.rs | 477 ++++++++++++++++++ crates/fabric-core/src/record/lines.rs | 264 ++++++++++ crates/fabric-core/src/record/mod.rs | 19 + crates/fabric-core/src/record/writer.rs | 104 ++++ crates/fabric-core/src/runtime.rs | 301 ++++++++++- crates/fabric-core/src/schema.rs | 9 +- crates/fabric-core/src/time.rs | 90 ++++ .../api/python-library-reference/index.md | 1 + .../nemo_fabric.models.md | 73 +++ .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../config/enum-capabilitykind.mdx | 2 +- .../config/enum-capabilitytarget.mdx | 2 +- .../config/enum-relayatifstorageconfig.mdx | 2 +- .../config/enum-relayatofmode.mdx | 2 +- .../config/enum-relayatofsinkconfig.mdx | 2 +- .../enum-relayatofstreamfieldnamepolicy.mdx | 2 +- .../config/enum-relayatofstreamtransport.mdx | 2 +- .../enum-relayopentelemetryendpointtype.mdx | 2 +- .../enum-relayopentelemetrymarkprojection.mdx | 2 +- .../config/enum-relayotlptransport.mdx | 2 +- .../config/enum-relayunsupportedbehavior.mdx | 2 +- .../config/enum-resolutionstrategy.mdx | 2 +- .../config/enum-telemetryprovider.mdx | 2 +- .../config/fn-load-adapter-descriptor.mdx | 2 +- .../fn-load-adapter-target-descriptor.mdx | 2 +- .../fn-resolve-run-plan-from-config.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 3 +- .../config/struct-recordconfig.mdx | 108 ++++ .../config/struct-relayatifconfig.mdx | 2 +- .../config/struct-relayatofconfig.mdx | 2 +- .../config/struct-relaycomponentconfig.mdx | 2 +- .../config/struct-relayconfig.mdx | 2 +- .../config/struct-relayconfigpolicy.mdx | 2 +- .../struct-relayobservabilityconfig.mdx | 2 +- .../struct-relayopentelemetryconfig.mdx | 2 +- ...truct-relayopentelemetryendpointconfig.mdx | 2 +- .../config/struct-resolvecontext.mdx | 2 +- .../struct-resolvedadapterdescriptor.mdx | 2 +- ...struct-resolvedadaptertargetdescriptor.mdx | 2 +- .../config/struct-runplan.mdx | 2 +- .../config/struct-runtimecapabilities.mdx | 2 +- .../config/struct-runtimeconfig.mdx | 8 +- .../config/struct-skillconfig.mdx | 2 +- .../config/struct-telemetryconfig.mdx | 2 +- .../config/struct-telemetryplan.mdx | 2 +- .../config/struct-telemetryproviderconfig.mdx | 2 +- .../config/struct-tooldefinitionconfig.mdx | 2 +- .../config/struct-toolsconfig.mdx | 2 +- .../config/struct-toolsplan.mdx | 2 +- .../config/struct-workflowconfig.mdx | 2 +- .../struct-workflowentrypointconfig.mdx | 2 +- .../config/struct-workflowtargetspec.mdx | 2 +- .../doctor/enum-doctorstatus.mdx | 2 +- .../doctor/fn-doctor-plan.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../doctor/struct-doctorcheck.mdx | 2 +- .../doctor/struct-doctorreport.mdx | 2 +- .../error/enum-fabricerror.mdx | 2 +- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/error/type-result.mdx | 2 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/index.mdx | 1 + .../nemo-fabric-core/runtime/index.mdx | 2 +- .../schema/enum-schemaname.mdx | 9 +- .../nemo-fabric-core/schema/index.mdx | 2 +- docs/sdk/python.mdx | 74 ++- schemas/SCHEMA.md | 3 + schemas/sdk/agent.schema.json | 25 + schemas/sdk/run-plan.schema.json | 25 + schemas/sdk/run-record-line.schema.json | 429 ++++++++++++++++ .../src/nemo_fabric/__init__.py | 2 + .../src/nemo_fabric/models.py | 16 + .../src/nemo_fabric/types.py | 19 +- .../references/results-and-errors.md | 15 + tests/python/test_sdk_consumer.py | 15 +- tests/python/test_sdk_contract.py | 23 + 82 files changed, 2321 insertions(+), 72 deletions(-) create mode 100644 crates/fabric-core/src/record/invocation.rs create mode 100644 crates/fabric-core/src/record/lines.rs create mode 100644 crates/fabric-core/src/record/mod.rs create mode 100644 crates/fabric-core/src/record/writer.rs create mode 100644 crates/fabric-core/src/time.rs create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx create mode 100644 schemas/sdk/run-record-line.schema.json diff --git a/crates/fabric-cli/src/presets.rs b/crates/fabric-cli/src/presets.rs index 803c6667e..5ebaa3202 100644 --- a/crates/fabric-cli/src/presets.rs +++ b/crates/fabric-cli/src/presets.rs @@ -287,6 +287,7 @@ fn config( artifacts: None, timeout_seconds: None, max_turns: None, + record: None, extensions: BTreeMap::new(), }, environment: Some(EnvironmentConfig { diff --git a/crates/fabric-cli/templates/rust/main.rs.tmpl b/crates/fabric-cli/templates/rust/main.rs.tmpl index e7f0ccfd3..be1c9771f 100644 --- a/crates/fabric-cli/templates/rust/main.rs.tmpl +++ b/crates/fabric-cli/templates/rust/main.rs.tmpl @@ -38,6 +38,7 @@ fn build_config() -> FabricConfig { artifacts: None, timeout_seconds: {{TIMEOUT_SECONDS}}, max_turns: {{MAX_TURNS}}, + record: None, extensions: BTreeMap::new(), }, environment: Some(EnvironmentConfig { diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index bef84b454..1cb2d7dcf 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -1057,11 +1057,34 @@ pub struct RuntimeConfig { #[serde(default, skip_serializing_if = "Option::is_none")] #[schemars(range(min = 1, max = u32::MAX))] pub max_turns: Option, + /// Opt-in per-invocation run record. Core-only; never sent to adapters. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub record: Option, /// Additive normalized runtime fields. #[serde(default, flatten)] pub extensions: BTreeMap, } +impl RuntimeConfig { + /// Whether `runtime.record.enabled` is set. + pub(crate) fn record_enabled(&self) -> bool { + self.record.as_ref().is_some_and(|record| record.enabled) + } +} + +/// Per-invocation run record settings. +/// +/// When enabled, core appends a start line and an end line for each local +/// invocation to `/.fabric///run-record.jsonl`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RecordConfig { + /// Whether to write a run record for each invocation. Requires an + /// artifact root, the `local` environment provider, and a `process` or + /// `python` adapter. + pub enabled: bool, +} + fn default_input_schema() -> String { "text".to_string() } @@ -1905,6 +1928,7 @@ pub(crate) fn validate_config(config: &FabricConfig) -> Result<()> { "must be a finite number greater than zero", ); } + validate_record_config(config)?; if let Some(system) = config .instructions .as_ref() @@ -2337,6 +2361,52 @@ fn validate_names(field: &str, names: &[String]) -> Result<()> { Ok(()) } +/// A record needs a host-side artifact root and a local adapter host, so an +/// enabled record that could never be written fails validation instead of +/// silently writing nothing. +fn validate_record_config(config: &FabricConfig) -> Result<()> { + if !config.runtime.record_enabled() { + return Ok(()); + } + let environment = config.environment.as_ref(); + if environment.is_some_and(|environment| environment.provider != "local") { + return invalid_config( + "runtime.record.enabled", + "supported only with the local environment provider", + ); + } + let has_artifact_root = config.runtime.artifacts.is_some() + || environment.is_some_and(|environment| environment.artifacts.is_some()); + if !has_artifact_root { + return invalid_config( + "runtime.record.enabled", + "requires runtime.artifacts or an environment artifact root", + ); + } + Ok(()) +} + +/// Core writes the record from its local adapter host path, so an adapter +/// that runs elsewhere would get an enabled record that is never written. +fn validate_record_adapter( + config: &FabricConfig, + descriptor: Option<&AdapterDescriptor>, +) -> Result<()> { + let local_host = descriptor.is_none_or(|descriptor| { + matches!( + descriptor.adapter_kind, + AdapterKind::Process | AdapterKind::Python + ) + }); + if config.runtime.record_enabled() && !local_host { + return invalid_config( + "runtime.record.enabled", + "supported only with a process or python adapter", + ); + } + Ok(()) +} + fn invalid_config(field: impl Into, reason: impl Into) -> Result { Err(FabricError::InvalidConfig { field: field.into(), @@ -2537,6 +2607,7 @@ fn resolve_selected_descriptors( let resolution = resolve_resolution(&config, descriptor)?; let environment_plan = resolve_environment_plan(&config, &base_dir); validate_control_location(descriptor, environment_plan.as_ref())?; + validate_record_adapter(&config, descriptor)?; let capability_plan = resolve_capability_plan(&config, &base_dir, adapter_descriptor.as_ref()); if enforce_compatibility { validate_capability_plan_compatibility(&capability_plan, descriptor)?; @@ -7342,4 +7413,94 @@ mod tests { && paths.len() == 2 )); } + + fn record_error(config: &FabricConfig) -> Option<(String, String)> { + match validate_config(config) { + Err(FabricError::InvalidConfig { field, reason }) => Some((field, reason)), + Err(error) => panic!("unexpected error: {error}"), + Ok(()) => None, + } + } + + #[test] + fn record_enabled_without_an_artifact_root_fails_validation() { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.record = Some(RecordConfig { enabled: true }); + + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "requires runtime.artifacts or an environment artifact root".to_string() + )) + ); + config.environment.as_mut().expect("environment").artifacts = Some(PathBuf::from("runs")); + assert_eq!(record_error(&config), None); + config.environment.as_mut().expect("environment").artifacts = None; + config.runtime.artifacts = Some(PathBuf::from("runs")); + assert_eq!(record_error(&config), None); + } + + #[test] + fn record_enabled_with_a_non_local_provider_fails_validation() { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + config.environment.as_mut().expect("environment").provider = "docker".to_string(); + + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "supported only with the local environment provider".to_string() + )) + ); + } + + #[test] + fn record_enabled_with_an_adapter_outside_the_local_host_fails_validation() { + let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); + let mut descriptor = load_adapter_descriptor(&path).expect("Claude descriptor"); + let mut config = typed_config("nvidia.fabric.claude"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); + + for kind in [AdapterKind::Http, AdapterKind::NativePlugin] { + descriptor.adapter_kind = kind; + assert!(matches!( + validate_record_adapter(&config, Some(&descriptor)), + Err(FabricError::InvalidConfig { field, reason }) + if field == "runtime.record.enabled" + && reason == "supported only with a process or python adapter" + )); + } + config.runtime.record = None; + assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); + } + + #[test] + fn record_disabled_or_absent_needs_no_artifact_root() { + let mut config = typed_config("nvidia.fabric.hermes"); + assert_eq!(record_error(&config), None); + config.runtime.record = Some(RecordConfig { enabled: false }); + assert_eq!(record_error(&config), None); + } + + #[test] + fn record_block_requires_enabled_and_rejects_unknown_fields() { + for runtime in [ + serde_json::json!({"record": {}}), + serde_json::json!({"record": {"enabled": true, "durable": true}}), + serde_json::json!({"record": true}), + ] { + let mut value = + serde_json::to_value(typed_config("nvidia.fabric.hermes")).expect("config JSON"); + value["runtime"] = runtime.clone(); + assert!( + serde_json::from_value::(value).is_err(), + "{runtime}" + ); + } + } } diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index 2dc251c9f..2e7f01458 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -9,8 +9,10 @@ pub mod agent_execution; pub mod config; pub mod doctor; pub mod error; +mod record; pub mod runtime; pub mod schema; +mod time; pub use adapter_contract::{ADAPTER_CONTRACT_VERSION, AdapterExtensionPoint}; pub use agent_execution::{ @@ -28,10 +30,11 @@ pub use config::{ EnvironmentOwnership, EnvironmentPlan, FabricConfig, HarnessConfig, InstructionConfig, InstructionMode, InstructionsConfig, McpAuthenticationConfig, McpConfig, McpExposure, McpServerPlan, McpTransport, MetadataConfig, ModelConfig, OAuthTokenEndpointAuthMethod, - ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, ResolvedAdapterTargetDescriptor, - RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, TelemetryConfig, TelemetryPlan, - TelemetryProvider, TelemetryProviderConfig, ToolDefinitionConfig, ToolsConfig, WorkflowConfig, - WorkflowEntrypointConfig, WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, + RecordConfig, ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, + ResolvedAdapterTargetDescriptor, RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, + TelemetryConfig, TelemetryPlan, TelemetryProvider, TelemetryProviderConfig, + ToolDefinitionConfig, ToolsConfig, WorkflowConfig, WorkflowEntrypointConfig, + WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, load_adapter_target_descriptor, resolve_diagnostic_plan_from_config, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config, resolve_run_plan_from_config_with_adapter_directories, diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs new file mode 100644 index 000000000..7d8a2595a --- /dev/null +++ b/crates/fabric-core/src/record/invocation.rs @@ -0,0 +1,477 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One invocation's record: the start line before the request, the end line +//! after the invocation. +//! +//! Recording never changes the run. A failed create, write or sync stops the +//! record for that invocation and adds a `record_write_failed` event to the +//! result; nothing here returns an error to the caller. + +use std::collections::BTreeMap; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime}; + +use serde_json::Value; + +use super::lines::{ + ContextValue, EndArtifact, EndError, EndLine, EndUsage, LineBody, ModelName, NotRecorded, + RecordLine, RecordSchema, RecordVersion, StartLine, cut_text, recorded_code, +}; +use super::writer::{RECORD_FILE_NAME, RecordWriter}; +use crate::config::RunPlan; +use crate::error::{FabricError, Result}; +use crate::runtime::{ + ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, INVALID_AGENT_RUN_RESULT, + InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, RuntimeHandle, + event_with_metadata, unique_artifact_name, +}; +use crate::time::rfc3339_millis; + +/// Name of the result artifact that points at the record. +const RECORD_ARTIFACT_NAME: &str = "record"; +/// Kind of the result artifact that points at the record. +const RECORD_ARTIFACT_KIND: &str = "run_record"; +/// Event added to the result when the record could not be written. +const RECORD_WRITE_FAILED: &str = "record_write_failed"; +/// `RunRequest.context` keys whose string or integer values the start line +/// keeps: task identity and the Harbor trial link. +const RECORDED_CONTEXT_KEYS: [&str; 7] = [ + "task_id", + "task_revision", + "attempt", + "step", + "harbor_session_id", + "harbor_context_id", + "harbor_trial_name", +]; +/// End-line error code for an artifact write that failed after the adapter returned. +const ARTIFACT_WRITE_FAILED: &str = "artifact_write_failed"; +/// End-line error code for any other Fabric error after the request was sent. +const FABRIC_ERROR: &str = "fabric_error"; + +/// What the start line needs from core's invocation path. +pub(crate) struct RecordStart<'a> { + pub plan: &'a RunPlan, + pub runtime: &'a RuntimeHandle, + pub invocation: &'a InvocationHandle, + pub request: &'a RunRequest, + /// The invocation folder, `/.fabric//`. + pub fabric_home: &'a Path, + /// The host timeout that applies to this invocation. + pub timeout: Duration, +} + +/// The record of one running invocation. +#[derive(Debug)] +pub(crate) struct InvocationRecorder { + runtime_id: String, + invocation_id: String, + request_id: String, + path: PathBuf, + started: Instant, + writer: Option, + wrote_start: bool, + failure: Option, +} + +impl InvocationRecorder { + /// Create the record and write its start line. + pub(crate) fn start(start: RecordStart<'_>) -> Self { + let path = start.fabric_home.join(RECORD_FILE_NAME); + let mut recorder = Self { + runtime_id: start.invocation.runtime_id.clone(), + invocation_id: start.invocation.invocation_id.clone(), + request_id: start.invocation.request_id.clone(), + started: Instant::now(), + writer: None, + wrote_start: false, + failure: None, + path, + }; + match RecordWriter::create(&recorder.path) { + Ok(writer) => { + recorder.writer = Some(writer); + recorder.append(LineBody::Start(start_line(&start))); + recorder.wrote_start = recorder.failure.is_none(); + } + Err(error) => recorder.failure = Some(error), + } + recorder + } + + /// Write and sync the end line for `outcome`, then return it. A result + /// gains the `record` artifact when the start line was written, and a + /// `record_write_failed` event when any write or the sync failed. + pub(crate) fn finish(mut self, outcome: Result) -> Result { + let elapsed_ms = u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX); + let end = match &outcome { + Ok(result) => result_end_line(result, elapsed_ms), + Err(error) => error_end_line(error, elapsed_ms), + }; + self.append(LineBody::End(end)); + if let Some(writer) = &self.writer + && let Err(error) = writer.sync() + { + self.failure = Some(error); + } + let mut result = outcome?; + if self.wrote_start { + let name = unique_artifact_name(&result.artifacts, RECORD_ARTIFACT_NAME); + result.artifacts.artifacts.push(ArtifactRef { + name, + kind: RECORD_ARTIFACT_KIND.to_string(), + path: self.path.clone(), + media_type: Some("application/jsonl".to_string()), + metadata: BTreeMap::new(), + }); + } + if let Some(error) = &self.failure { + result.events.push(event_with_metadata( + RECORD_WRITE_FAILED, + format!("run record write failed; the record stops here: {error}"), + BTreeMap::from([ + ( + "runtime_id".to_string(), + Value::from(self.runtime_id.as_str()), + ), + ( + "invocation_id".to_string(), + Value::from(self.invocation_id.as_str()), + ), + ]), + )); + } + Ok(result) + } + + /// Append one line. After the first failure the record stops. + fn append(&mut self, body: LineBody) { + let Some(writer) = self.writer.as_mut() else { + return; + }; + let line = RecordLine { + schema: RecordSchema::RunRecord, + version: RecordVersion::V1Alpha1, + runtime_id: self.runtime_id.clone(), + invocation_id: self.invocation_id.clone(), + request_id: self.request_id.clone(), + time: rfc3339_millis(SystemTime::now()), + body, + }; + if let Err(error) = writer.append(&line) { + self.failure = Some(error); + self.writer = None; + } + } +} + +fn start_line(start: &RecordStart<'_>) -> StartLine { + StartLine { + agent_name: cut_text(&start.runtime.agent_name), + harness: cut_text(&start.runtime.harness), + adapter_id: start.runtime.adapter_id.as_deref().map(cut_text), + models: start + .plan + .config + .models + .iter() + .map(|(role, model)| { + ( + cut_text(role), + ModelName { + provider: cut_text(&model.provider), + model: cut_text(&model.model), + }, + ) + }) + .collect(), + fabric_version: crate::version().to_string(), + timeout_ms: u64::try_from(start.timeout.as_millis()).unwrap_or(u64::MAX), + context: start + .request + .context + .iter() + .map(|(key, value)| (cut_text(key), context_value(key, value))) + .collect(), + override_keys: override_keys(start.request.overrides.as_ref()), + } +} + +/// The sorted top-level key names of `overrides`; empty unless it is an object. +fn override_keys(overrides: Option<&Value>) -> Vec { + let Some(Value::Object(overrides)) = overrides else { + return Vec::new(); + }; + let mut keys: Vec = overrides.keys().map(|key| cut_text(key)).collect(); + keys.sort(); + keys +} + +/// An allowlisted key keeps a string (cut) or integer value; any other key, +/// or an allowlisted key with another value type, is recorded by name only. +fn context_value(key: &str, value: &Value) -> ContextValue { + if !RECORDED_CONTEXT_KEYS.contains(&key) { + return ContextValue::NotRecorded(NotRecorded::VALUE); + } + match value { + Value::String(text) => ContextValue::Text(cut_text(text)), + Value::Number(number) => number + .as_i64() + .map_or(ContextValue::NotRecorded(NotRecorded::VALUE), |integer| { + ContextValue::Integer(integer) + }), + _ => ContextValue::NotRecorded(NotRecorded::VALUE), + } +} + +fn result_end_line(result: &RunResult, elapsed_ms: u64) -> EndLine { + EndLine { + status: result.status, + elapsed_ms, + error: result.error.as_ref().map(result_error), + usage: result.usage.as_ref().map(end_usage), + artifacts: end_artifacts(&result.artifacts), + } +} + +fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { + let (code, stage) = match error { + FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), + // After the record starts, only the check of the adapter's result + // extensions raises this, the same fault as a rejected result. + FabricError::AdapterCompatibility { .. } => (INVALID_AGENT_RUN_RESULT, ErrorStage::Invoke), + FabricError::Write { .. } => (ARTIFACT_WRITE_FAILED, ErrorStage::Artifact), + _ => (FABRIC_ERROR, ErrorStage::Invoke), + }; + EndLine { + status: RunStatus::Failed, + elapsed_ms, + error: Some(EndError { + code: recorded_code(code), + stage, + retryable: false, + }), + usage: None, + artifacts: Vec::new(), + } +} + +fn result_error(error: &ErrorInfo) -> EndError { + EndError { + code: recorded_code(&error.code), + stage: error.stage, + retryable: error.retryable, + } +} + +fn end_usage(usage: &RunUsage) -> EndUsage { + EndUsage { + input_tokens: usage.input_tokens, + cached_input_tokens: usage.cached_input_tokens, + input_tokens_include_cache: usage.input_tokens_include_cache, + output_tokens: usage.output_tokens, + total_tokens: usage.total_tokens, + cost_usd: usage.cost_usd, + } +} + +fn end_artifacts(manifest: &ArtifactManifest) -> Vec { + manifest + .artifacts + .iter() + .map(|artifact| EndArtifact { + name: cut_text(&artifact.name), + kind: cut_text(&artifact.kind), + media_type: artifact.media_type.as_deref().map(cut_text), + path: manifest + .root + .as_deref() + .and_then(|root| artifact.path.strip_prefix(root).ok()) + .map(|relative| relative.to_string_lossy().into_owned()), + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; + + fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { + let config: FabricConfig = serde_json::from_value(serde_json::json!({ + "schema_version": "fabric.agent/v1alpha1", + "metadata": {"name": "demo"}, + "harness": {"adapter_id": "nvidia.fabric.hermes", "resolution": "preinstalled"}, + "runtime": {}, + "models": {"default": {"provider": "nvidia", "model": "nemotron"}}, + })) + .expect("config"); + let plan = resolve_run_plan_from_config(config, ResolveContext::new("/tmp/fabric-base")) + .expect("plan"); + let runtime: RuntimeHandle = serde_json::from_value(serde_json::json!({ + "runtime_id": "rt", "runtime_binding": "binding", "agent_name": "demo", + "harness": "nvidia.fabric.hermes", "adapter_kind": "python", + "adapter_id": "nvidia.fabric.hermes", + "environment": { + "environment_id": "env", "provider": "local", + "control_location": "external_control", "ownership": "caller_owned", + }, + })) + .expect("runtime handle"); + let invocation = InvocationHandle { + invocation_id: "inv".to_string(), + request_id: "req".to_string(), + runtime_id: "rt".to_string(), + }; + (plan, runtime, invocation) + } + + #[test] + fn start_line_keeps_override_and_unknown_context_key_names_only() { + let (plan, runtime, invocation) = record_test_fixture(); + let mut request = RunRequest::text("input"); + request.context = BTreeMap::from([ + ("task_id".to_string(), Value::from("t".repeat(300))), + ("task_revision".to_string(), Value::from(1.5)), + ( + "headers".to_string(), + serde_json::json!({"authorization": "Bearer abc"}), + ), + ]); + request.overrides = Some(serde_json::json!({"zeta": 1, "acme": {"api_key": "sk-123"}})); + let start = RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: Path::new("/unused"), + timeout: Duration::from_secs(5), + }; + + let line = serde_json::to_value(RecordLine { + schema: RecordSchema::RunRecord, + version: RecordVersion::V1Alpha1, + runtime_id: "rt".to_string(), + invocation_id: "inv".to_string(), + request_id: "req".to_string(), + time: "2026-10-14T18:01:10.205Z".to_string(), + body: LineBody::Start(start_line(&start)), + }) + .expect("start line"); + + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunRecordLine) + .expect("record line schema"); + let validator = jsonschema::validator_for(&schema).expect("valid schema"); + assert!(validator.is_valid(&line), "{line}"); + assert_eq!(line["override_keys"], serde_json::json!(["acme", "zeta"])); + assert_eq!( + line["context"]["headers"], + serde_json::json!({"recorded": false}) + ); + assert_eq!( + line["context"]["task_revision"], + serde_json::json!({"recorded": false}) + ); + assert_eq!( + line["context"]["task_id"] + .as_str() + .expect("task id") + .chars() + .count(), + 256 + ); + assert_eq!(line["timeout_ms"], 5_000); + assert_eq!( + line["models"], + serde_json::json!({"default": {"provider": "nvidia", "model": "nemotron"}}) + ); + let text = line.to_string(); + assert!( + !text.contains("sk-123") && !text.contains("Bearer"), + "{text}" + ); + } + + fn temp_folder(name: &str) -> PathBuf { + let folder = + std::env::temp_dir().join(format!("fabric-recorder-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&folder); + std::fs::create_dir_all(&folder).expect("folder"); + folder + } + + fn succeeded_result(invocation: &InvocationHandle) -> RunResult { + serde_json::from_value(serde_json::json!({ + "agent_name": "a", "harness": "h", "adapter_kind": "python", + "runtime_id": invocation.runtime_id, "invocation_id": invocation.invocation_id, + "request_id": invocation.request_id, "status": "succeeded", "output": "done", + })) + .expect("result") + } + + #[test] + fn finish_names_the_record_artifact_apart_from_an_adapter_artifact_named_record() { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder("unique-name"); + let request = RunRequest::text("input"); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + let mut result = succeeded_result(&invocation); + result.artifacts.artifacts.push(ArtifactRef { + name: RECORD_ARTIFACT_NAME.to_string(), + kind: "report".to_string(), + path: folder.join("report.json"), + media_type: None, + metadata: BTreeMap::new(), + }); + + let finished = recorder.finish(Ok(result)).expect("result"); + + let names: Vec<&str> = finished + .artifacts + .artifacts + .iter() + .map(|artifact| artifact.name.as_str()) + .collect(); + assert_eq!(names, ["record", "record_2"]); + assert_eq!(finished.artifacts.artifacts[1].kind, RECORD_ARTIFACT_KIND); + let _ = std::fs::remove_dir_all(folder); + } + + #[test] + fn finish_after_a_failed_create_adds_the_event_and_no_record_artifact() { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder("create-failure"); + std::fs::write(folder.join(RECORD_FILE_NAME), "planted").expect("planted file"); + let request = RunRequest::text("input"); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + let result = succeeded_result(&invocation); + + let finished = recorder.finish(Ok(result.clone())).expect("result"); + + assert_eq!(finished.artifacts, result.artifacts); + assert_eq!(finished.events.len(), 1); + assert_eq!(finished.events[0].kind, RECORD_WRITE_FAILED); + assert_eq!(finished.status, RunStatus::Succeeded); + assert_eq!( + std::fs::read_to_string(folder.join(RECORD_FILE_NAME)).expect("read"), + "planted" + ); + let _ = std::fs::remove_dir_all(folder); + } +} diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs new file mode 100644 index 000000000..ebe9df282 --- /dev/null +++ b/crates/fabric-core/src/record/lines.rs @@ -0,0 +1,264 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Line types of `run-record.jsonl`, schema `nemo.fabric.run_record` version +//! `v1alpha1`. +//! +//! Every field is typed. A line holds identifiers, names, numbers, closed codes +//! and the values of a few allowlisted request context keys. It never holds a +//! value outside that allowlist: no prompt, output, error message or metadata. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::Serialize; + +use crate::runtime::{ErrorStage, RunStatus}; + +/// Longest string a line keeps from config or the request, in characters. +const MAX_VALUE_CHARS: usize = 256; +/// Longest adapter error code a line keeps. +const MAX_CODE_CHARS: usize = 128; +/// Error code written in place of a code outside `^[a-z0-9_.:-]{1,128}$`. +const UNRECOGNIZED_CODE: &str = "unrecognized"; + +/// One line of a run record. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +#[schemars(title = "RunRecordLine")] +pub(crate) struct RecordLine { + /// Always `nemo.fabric.run_record`. + pub schema: RecordSchema, + /// Line format version. `v1alpha1` may change until a record reader ships. + pub version: RecordVersion, + /// Runtime id. + pub runtime_id: String, + /// Invocation id. + pub invocation_id: String, + /// Request id. + pub request_id: String, + /// UTC wall-clock time the line was written, RFC 3339 with milliseconds. + #[schemars(extend("format" = "date-time"))] + pub time: String, + /// Line kind and its fields. + #[serde(flatten)] + pub body: LineBody, +} + +/// The record's schema name. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) enum RecordSchema { + /// `nemo.fabric.run_record`. + #[serde(rename = "nemo.fabric.run_record")] + RunRecord, +} + +/// The record's line format version. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) enum RecordVersion { + /// `v1alpha1`. + #[serde(rename = "v1alpha1")] + V1Alpha1, +} + +/// A line's kind, with the fields that kind carries. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub(crate) enum LineBody { + /// Written once, before core sends the request to the adapter host. + Start(StartLine), + /// Written once, after the invocation returned a result or failed. + End(EndLine), +} + +/// Fields of the start line. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub(crate) struct StartLine { + /// Config name (`metadata.name`). + pub agent_name: String, + /// Harness identifier. + pub harness: String, + /// Adapter implementation id. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub adapter_id: Option, + /// Configured models by role: provider and model name only. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub models: BTreeMap, + /// NeMo Fabric version that wrote the record. + pub fabric_version: String, + /// Host invocation timeout that applied, in milliseconds. + pub timeout_ms: u64, + /// `RunRequest.context`. Allowlisted identity and Harbor link keys keep a + /// string or integer value; every other key is listed by name only. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub context: BTreeMap, + /// Top-level key names of `RunRequest.overrides`, sorted. Values are never + /// recorded. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub override_keys: Vec, +} + +/// One configured model. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct ModelName { + /// Model provider name. + pub provider: String, + /// Provider model identifier. + pub model: String, +} + +/// A request context value as the start line keeps it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(untagged)] +pub(crate) enum ContextValue { + /// An allowlisted key's string value, cut to 256 characters. + Text(String), + /// An allowlisted key's integer value. + Integer(i64), + /// Any other key or value: the key name only. + NotRecorded(NotRecorded), +} + +/// Marks a key whose value the record leaves out. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct NotRecorded { + /// Always `false`. + #[schemars(extend("const" = false))] + pub recorded: bool, +} + +impl NotRecorded { + pub(crate) const VALUE: Self = Self { recorded: false }; +} + +/// Fields of the end line. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub(crate) struct EndLine { + /// Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`. + pub status: RunStatus, + /// Monotonic time from the start line to the end line, in milliseconds. + pub elapsed_ms: u64, + /// Error code, stage and retryable flag, without the message. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Numeric usage counters, without adapter metadata. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, + /// Result artifacts, with paths relative to the artifact root. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub artifacts: Vec, +} + +/// The end line's error. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct EndError { + /// Error code matching `^[a-z0-9_.:-]{1,128}$`, else `unrecognized`. + #[schemars(regex(pattern = r"^[a-z0-9_.:-]{1,128}$"))] + pub code: String, + /// Lifecycle stage where the failure surfaced. + pub stage: ErrorStage, + /// Whether a consumer-level retry is considered safe. + pub retryable: bool, +} + +/// The end line's usage: the numeric `RunUsage` fields. +#[derive(Debug, Clone, Default, PartialEq, Serialize, JsonSchema)] +pub(crate) struct EndUsage { + /// Input tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens: Option, + /// Cached input tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cached_input_tokens: Option, + /// Whether `input_tokens` includes `cached_input_tokens`; absent means unknown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens_include_cache: Option, + /// Output tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_tokens: Option, + /// Total tokens reported by the provider. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_tokens: Option, + /// Reported cost in US dollars. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 0.0))] + pub cost_usd: Option, +} + +/// One result artifact on the end line. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct EndArtifact { + /// Logical artifact name. + pub name: String, + /// Artifact kind. + pub kind: String, + /// Media type. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub media_type: Option, + /// Path relative to the artifact root; absent for a path outside it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, +} + +/// Cut `text` to [`MAX_VALUE_CHARS`] characters: 255 plus "…". +pub(crate) fn cut_text(text: &str) -> String { + if text.chars().count() <= MAX_VALUE_CHARS { + return text.to_string(); + } + let mut cut: String = text.chars().take(MAX_VALUE_CHARS - 1).collect(); + cut.push('…'); + cut +} + +/// An error code as the end line keeps it: unchanged when it matches +/// `^[a-z0-9_.:-]{1,128}$`, else [`UNRECOGNIZED_CODE`], because a code outside +/// that pattern may be a message. +pub(crate) fn recorded_code(code: &str) -> String { + let valid = (1..=MAX_CODE_CHARS).contains(&code.len()) + && code.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"_.:-".contains(&byte) + }); + if valid { + code.to_string() + } else { + UNRECOGNIZED_CODE.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cut_text_over_the_limit_keeps_255_chars_and_an_ellipsis() { + let cut = cut_text(&"é".repeat(1000)); + + assert_eq!(cut.chars().count(), MAX_VALUE_CHARS); + assert!(cut.ends_with('…')); + assert_eq!(cut_text("short"), "short"); + assert_eq!(cut_text(&"a".repeat(256)), "a".repeat(256)); + } + + #[test] + fn recorded_code_keeps_only_codes_in_the_pattern() { + for code in [ + "timeout", + "host_timeout", + "provider.rate_limit", + "http:429", + "a-b", + ] { + assert_eq!(recorded_code(code), code); + } + for code in [ + "", + "Timeout", + "rate limit", + "error: key sk-123", + "über", + &"a".repeat(129), + ] { + assert_eq!(recorded_code(code), UNRECOGNIZED_CODE, "{code}"); + } + assert_eq!(recorded_code(&"a".repeat(128)), "a".repeat(128)); + } +} diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs new file mode 100644 index 000000000..32f867bb6 --- /dev/null +++ b/crates/fabric-core/src/record/mod.rs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The opt-in per-invocation run record, `run-record.jsonl`. +//! +//! When `runtime.record.enabled` is set, core appends one start line before it +//! sends the request to the adapter host and one end line after the invocation +//! returns or fails. Recording never changes the run: a failed write stops the +//! record and becomes a `record_write_failed` event on the result. +//! +//! Everything here is crate-private. The line format is public only through +//! the `run-record-line` JSON Schema, version `v1alpha1`. + +mod invocation; +mod lines; +mod writer; + +pub(crate) use invocation::{InvocationRecorder, RecordStart}; +pub(crate) use lines::RecordLine; diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs new file mode 100644 index 000000000..1a849ac60 --- /dev/null +++ b/crates/fabric-core/src/record/writer.rs @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The run record file: created once, owner-only, appended one line per write. +//! +//! Each line is one `write_all` on an unbuffered file, so a SIGKILL of Fabric +//! or the adapter host keeps every line whose write returned. Only the end +//! line is synced, once, so the record survives a crash of the machine only +//! up to its last synced line. + +use std::fs::{File, OpenOptions}; +use std::io::{self, Write}; +use std::path::Path; + +use super::lines::RecordLine; + +/// File name of the record inside the invocation folder. +pub(crate) const RECORD_FILE_NAME: &str = "run-record.jsonl"; + +/// Appends lines to one invocation's record. +#[derive(Debug)] +pub(crate) struct RecordWriter { + file: File, +} + +impl RecordWriter { + /// Create the record at `path`. `create_new` refuses an existing file or + /// symbolic link at `path`, and on Unix the file is mode 0600. + pub(crate) fn create(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.append(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + Ok(Self { + file: options.open(path)?, + }) + } + + /// Append `line` and its newline with one write. + pub(crate) fn append(&mut self, line: &RecordLine) -> io::Result<()> { + let mut bytes = serde_json::to_vec(line).map_err(io::Error::other)?; + bytes.push(b'\n'); + self.file.write_all(&bytes) + } + + /// Flush the file's data and metadata to stable storage. + pub(crate) fn sync(&self) -> io::Result<()> { + self.file.sync_all() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_folder(name: &str) -> std::path::PathBuf { + let folder = std::env::temp_dir().join(format!( + "fabric-record-writer-{name}-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&folder); + std::fs::create_dir_all(&folder).expect("create temp folder"); + folder + } + + #[test] + fn record_writer_refuses_an_existing_file_or_a_symlink() { + let folder = temp_folder("excl"); + let path = folder.join(RECORD_FILE_NAME); + std::fs::write(&path, "old").expect("seed file"); + + let error = RecordWriter::create(&path).expect_err("existing file"); + + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!(std::fs::read_to_string(&path).expect("read"), "old"); + #[cfg(unix)] + { + let linked = folder.join("linked"); + std::fs::create_dir_all(&linked).expect("linked folder"); + let target = folder.join("target"); + std::os::unix::fs::symlink(&target, linked.join(RECORD_FILE_NAME)).expect("symlink"); + assert!(RecordWriter::create(&linked.join(RECORD_FILE_NAME)).is_err()); + assert!(!target.exists()); + } + let _ = std::fs::remove_dir_all(folder); + } + + #[cfg(unix)] + #[test] + fn record_writer_new_file_is_owner_only() { + use std::os::unix::fs::PermissionsExt; + let folder = temp_folder("mode"); + let path = folder.join(RECORD_FILE_NAME); + + RecordWriter::create(&path).expect("create"); + let mode = std::fs::metadata(&path).expect("stat").permissions().mode(); + + assert_eq!(mode & 0o777, 0o600); + let _ = std::fs::remove_dir_all(folder); + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index ab16add07..fce59395e 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -31,9 +31,12 @@ use crate::config::{ validate_tool_definitions, validate_workflow, }; use crate::error::{FabricError, Result}; +use crate::record::{InvocationRecorder, RecordStart}; static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; +/// Lifecycle error code for an adapter result that core rejects. +pub(crate) const INVALID_AGENT_RUN_RESULT: &str = "invalid_agent_run_result"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; const LOCAL_HOST_START_TIMEOUT: Duration = Duration::from_secs(90); // Protocol liveness backstop. Adapters remain responsible for normal request @@ -1726,11 +1729,38 @@ impl LocalHostInvocation { } fn run_local_host_invocation_with_timeout( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, + invocation_kind: LocalHostInvocation, + invoke_timeout: Duration, +) -> Result { + let mut recorder = None; + let outcome = local_host_invocation_result( + plan, + runtime, + request, + invocation_kind, + invoke_timeout, + &mut recorder, + ); + match recorder { + // The end line and its sync run here, after the host lock is released. + Some(recorder) => recorder.finish(outcome), + None => outcome, + } +} + +/// Run one invocation on the local host. When the plan turns the run record +/// on, `recorder` is set right before core sends the request to the host, so +/// every invocation that core tried to send has a record. +fn local_host_invocation_result( plan: &RunPlan, runtime: &RuntimeHandle, mut request: RunRequest, invocation_kind: LocalHostInvocation, invoke_timeout: Duration, + recorder: &mut Option, ) -> Result { let operation = invocation_kind.operation(); if request.request_id.is_empty() { @@ -1806,6 +1836,16 @@ fn run_local_host_invocation_with_timeout( } }; let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; + if plan.config.runtime.record_enabled() && artifacts.root.is_some() { + *recorder = Some(InvocationRecorder::start(RecordStart { + plan, + runtime, + invocation: &invocation, + request: &request, + fabric_home: &fabric_home, + timeout: invoke_timeout, + })); + } match exchange_lifecycle_message( &mut host_guard, &runtime.runtime_id, @@ -1851,7 +1891,7 @@ fn run_local_host_invocation_with_timeout( lifecycle_error( operation, &runtime.runtime_id, - "invalid_agent_run_result", + INVALID_AGENT_RUN_RESULT, format!("adapter returned an invalid AgentRunResult: {error}"), &stderr, ) @@ -1860,7 +1900,7 @@ fn run_local_host_invocation_with_timeout( lifecycle_error( operation, &runtime.runtime_id, - "invalid_agent_run_result", + INVALID_AGENT_RUN_RESULT, format!("adapter returned an invalid AgentRunResult: {error}"), &stderr, ) @@ -1870,7 +1910,7 @@ fn run_local_host_invocation_with_timeout( return Err(lifecycle_error( operation, &runtime.runtime_id, - "invalid_agent_run_result", + INVALID_AGENT_RUN_RESULT, "adapter returned artifacts without a configured runtime artifact root", &stderr, )); @@ -3297,7 +3337,7 @@ fn relay_artifact_media_type(kind: &str) -> Option<&'static str> { } } -fn unique_artifact_name(manifest: &ArtifactManifest, base: &str) -> String { +pub(crate) fn unique_artifact_name(manifest: &ArtifactManifest, base: &str) -> String { if !manifest .artifacts .iter() @@ -3658,7 +3698,7 @@ fn telemetry_ref( }) } -fn event_with_metadata( +pub(crate) fn event_with_metadata( kind: impl Into, message: impl Into, metadata: BTreeMap, @@ -3947,12 +3987,16 @@ for line in sys.stdin: "context_env": invocation["runtime_context"]["environment"].get("env", {}), "deadline_millis": invocation["runtime_context"].get("deadline_millis"), } - if MODE in {"adapter_reported_failure", "blank_error_code"}: + if MODE in {"adapter_reported_failure", "blank_error_code", "free_text_error_code"}: + codes = { + "blank_error_code": " \t", + "free_text_error_code": "Rate limit for key sk-live-123 exceeded", + } result = { "status": "failed", "output": output, "error": { - "code": " \t" if MODE == "blank_error_code" else "fake_adapter_failure", + "code": codes.get(MODE, "fake_adapter_failure"), "message": "adapter rejected the invocation", "retryable": True, "extensions": {"source": "fake-host"}, @@ -5458,4 +5502,247 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + + fn record_plan(mode: &str) -> (PathBuf, RunPlan) { + let (root, mut plan) = local_host_plan(mode); + plan.config.runtime.record = Some(crate::config::RecordConfig { enabled: true }); + (root, plan) + } + + fn record_files(root: &Path) -> Vec { + let mut records = Vec::new(); + let mut folders = vec![root.join("artifacts/.fabric")]; + while let Some(folder) = folders.pop() { + let Ok(entries) = fs::read_dir(&folder) else { + continue; + }; + for path in entries.flatten().map(|entry| entry.path()) { + if path.is_dir() { + folders.push(path); + } else if path.ends_with("run-record.jsonl") { + records.push(path); + } + } + } + records + } + + fn record_line_validator() -> jsonschema::Validator { + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunRecordLine) + .expect("record line schema"); + jsonschema::validator_for(&schema).expect("valid schema") + } + + /// The only record under `root`, with every line checked against the + /// published line schema. + fn only_record(root: &Path) -> (String, Vec) { + let records = record_files(root); + assert_eq!(records.len(), 1, "{records:?}"); + let text = fs::read_to_string(&records[0]).expect("read record"); + let validator = record_line_validator(); + let lines: Vec = text + .lines() + .map(|line| serde_json::from_str(line).expect("record line is JSON")) + .collect(); + for line in &lines { + assert!(validator.is_valid(line), "{line}"); + } + (text, lines) + } + + /// The one line of `kind`. Lookup by kind keeps these tests independent + /// of other line kinds written between the start and end lines. + fn line_of_kind<'a>(lines: &'a [Value], kind: &str) -> &'a Value { + let mut matching = lines.iter().filter(|line| line["kind"] == kind); + let line = matching.next().unwrap_or_else(|| panic!("no {kind} line")); + assert!(matching.next().is_none(), "more than one {kind} line"); + line + } + + #[test] + fn record_off_with_an_artifact_root_writes_no_record() { + let (root, plan) = local_host_plan("typed_result"); + assert!(plan.config.runtime.artifacts.is_some()); + + let result = run_plan(&plan, RunRequest::text("no record")).expect("run"); + + assert!(record_files(&root).is_empty()); + assert!( + result + .artifacts + .artifacts + .iter() + .all(|artifact| artifact.kind != "run_record") + ); + assert!( + result + .events + .iter() + .all(|event| event.kind != "record_write_failed") + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_on_writes_a_start_line_and_an_end_line() { + let (root, plan) = record_plan("typed_result"); + let mut request = RunRequest::text("record"); + request.context = BTreeMap::from([ + ("task_id".to_string(), serde_json::json!("repo-io-fix")), + ("attempt".to_string(), serde_json::json!(2)), + ( + "harbor_trial_name".to_string(), + serde_json::json!("trial-1"), + ), + ("step".to_string(), serde_json::json!({"nested": "value"})), + ("api_key".to_string(), serde_json::json!("sk-live-SECRET")), + ]); + + let result = run_plan(&plan, request).expect("run"); + let (text, lines) = only_record(&root); + + let (start, end) = (line_of_kind(&lines, "start"), line_of_kind(&lines, "end")); + assert_eq!(lines.first(), Some(start), "{text}"); + assert_eq!(lines.last(), Some(end), "{text}"); + for line in &lines { + assert_eq!(line["schema"], "nemo.fabric.run_record"); + assert_eq!(line["version"], "v1alpha1"); + assert_eq!(line["runtime_id"], result.runtime_id.as_str()); + assert_eq!(line["invocation_id"], result.invocation_id.as_str()); + assert_eq!(line["request_id"], result.request_id.as_str()); + } + assert_eq!(start["agent_name"], "local-host-test-agent"); + assert_eq!(start["adapter_id"], "acme.fabric.local-host"); + assert_eq!(start["timeout_ms"], 3_600_000); + assert_eq!( + start["context"], + serde_json::json!({ + "api_key": {"recorded": false}, + "attempt": 2, + "harbor_trial_name": "trial-1", + "step": {"recorded": false}, + "task_id": "repo-io-fix", + }) + ); + assert!(!text.contains("SECRET"), "{text}"); + assert_eq!(end["status"], "succeeded"); + assert!(end["elapsed_ms"].is_u64()); + assert_eq!( + end["usage"], + serde_json::json!({ + "input_tokens": 3, + "cached_input_tokens": 1, + "input_tokens_include_cache": true, + "output_tokens": 5, + "total_tokens": 8, + "cost_usd": 0.25, + }) + ); + let report = end["artifacts"] + .as_array() + .expect("artifacts") + .iter() + .find(|artifact| artifact["name"] == "report") + .expect("report artifact"); + assert_eq!( + report, + &serde_json::json!({ + "name": "report", + "kind": "report", + "media_type": "application/json", + "path": "reports/result.json", + }) + ); + for marker in ["fake-host", "\"fake\"", "trace-1"] { + assert!(!text.contains(marker), "{marker} in {text}"); + } + let record = result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.name == "record") + .expect("record artifact"); + assert_eq!(record.kind, "run_record"); + assert_eq!(record.media_type.as_deref(), Some("application/jsonl")); + assert_eq!(record.path, record_files(&root)[0]); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_end_line_keeps_the_error_code_and_drops_the_message() { + let (root, plan) = record_plan("adapter_reported_failure"); + + let result = run_plan(&plan, RunRequest::text("fail")).expect("failed result"); + let (text, lines) = only_record(&root); + + assert_eq!(result.status, RunStatus::Failed); + assert_eq!( + line_of_kind(&lines, "end")["error"], + serde_json::json!({"code": "fake_adapter_failure", "stage": "invoke", "retryable": true}) + ); + assert!(!text.contains("adapter rejected"), "{text}"); + let mut wrong_kind = line_of_kind(&lines, "end").clone(); + wrong_kind["kind"] = serde_json::json!("start"); + assert!(!record_line_validator().is_valid(&wrong_kind)); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_end_line_replaces_a_free_text_error_code() { + let (root, plan) = record_plan("free_text_error_code"); + + let result = run_plan(&plan, RunRequest::text("fail")).expect("failed result"); + let (text, lines) = only_record(&root); + + assert!(result.error.expect("error").code.contains("sk-live-123")); + assert_eq!(line_of_kind(&lines, "end")["error"]["code"], "unrecognized"); + assert!(!text.contains("sk-live-123"), "{text}"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_lifecycle_error_writes_a_failed_end_line() { + let (root, plan) = record_plan("invoke_timeout"); + let runtime = start_runtime(&plan).expect("start local host"); + + let error = run_local_host_adapter_with_timeout( + &plan, + &runtime, + RunRequest::text("hang"), + Duration::from_millis(100), + ) + .expect_err("timeout raises"); + let (_, lines) = only_record(&root); + + assert!(error.to_string().contains("host_timeout"), "{error}"); + let end = line_of_kind(&lines, "end"); + assert_eq!(line_of_kind(&lines, "start")["timeout_ms"], 100); + assert_eq!(end["status"], "failed"); + assert_eq!( + end["error"], + serde_json::json!({"code": "host_timeout", "stage": "invoke", "retryable": false}) + ); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_warm_runtime_writes_one_record_per_invocation() { + let (root, plan) = record_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + + let first = invoke_runtime(&plan, &runtime, RunRequest::text("one")).expect("first"); + let second = invoke_runtime(&plan, &runtime, RunRequest::text("two")).expect("second"); + stop_runtime(&plan, &runtime).expect("stop"); + + let records = record_files(&root); + assert_eq!(records.len(), 2); + for result in [&first, &second] { + assert!(records.iter().any(|path| path.ends_with(format!( + ".fabric/{}/{}/run-record.jsonl", + result.runtime_id, result.invocation_id + )))); + } + let _ = fs::remove_dir_all(root); + } } diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index ee15fda12..aefc27b27 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -13,6 +13,7 @@ use crate::config::{ AdapterDescriptor, AdapterTargetDescriptor, AgentConfig, FabricConfig, RunPlan, }; use crate::error::{FabricError, Result}; +use crate::record::RecordLine; use crate::runtime::{ AdapterInvocation, ArtifactManifest, EnvironmentHandle, ErrorInfo, FabricEvent, InvocationHandle, OpenAiStreamInvocation, OpenAiStreamRecord, RunRequest, RunResult, @@ -65,11 +66,13 @@ pub enum SchemaName { ErrorInfo, /// NeMo Fabric lifecycle event schema. FabricEvent, + /// One line of an opt-in per-invocation run record, version `v1alpha1`. + RunRecordLine, } impl SchemaName { /// All public schemas in stable output order. - pub const ALL: [Self; 21] = [ + pub const ALL: [Self; 22] = [ Self::Agent, Self::AgentConfig, Self::AgentRunRequest, @@ -91,6 +94,7 @@ impl SchemaName { Self::ArtifactManifest, Self::ErrorInfo, Self::FabricEvent, + Self::RunRecordLine, ]; /// Stable file stem for this schema. @@ -117,6 +121,7 @@ impl SchemaName { Self::ArtifactManifest => "artifact-manifest", Self::ErrorInfo => "error-info", Self::FabricEvent => "fabric-event", + Self::RunRecordLine => "run-record-line", } } @@ -170,6 +175,7 @@ impl SchemaName { "artifact-manifest" | "artifact_manifest" => Ok(Self::ArtifactManifest), "error-info" | "error_info" => Ok(Self::ErrorInfo), "fabric-event" | "fabric_event" => Ok(Self::FabricEvent), + "run-record-line" | "run_record_line" => Ok(Self::RunRecordLine), _ => Err(FabricError::UnknownSchema { schema: value.to_string(), available: Self::ALL @@ -205,6 +211,7 @@ pub fn generate_schema(schema: SchemaName) -> Result { SchemaName::ArtifactManifest => to_value(schema_for!(ArtifactManifest)), SchemaName::ErrorInfo => to_value(schema_for!(ErrorInfo)), SchemaName::FabricEvent => to_value(schema_for!(FabricEvent)), + SchemaName::RunRecordLine => to_value(schema_for!(RecordLine)), } } diff --git a/crates/fabric-core/src/time.rs b/crates/fabric-core/src/time.rs new file mode 100644 index 000000000..06ace9edf --- /dev/null +++ b/crates/fabric-core/src/time.rs @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! RFC 3339 UTC timestamps with millisecond precision. +//! +//! Core has no date crate, so this converts Unix milliseconds to a civil date +//! with Howard Hinnant's `civil_from_days` algorithm. + +use std::time::{SystemTime, UNIX_EPOCH}; + +const MILLIS_PER_DAY: i64 = 86_400_000; + +/// Format `time` as RFC 3339 UTC with three fraction digits, such as +/// `2026-10-14T18:01:10.205Z`. +pub(crate) fn rfc3339_millis(time: SystemTime) -> String { + format_unix_millis(unix_millis(time)) +} + +/// Milliseconds since the Unix epoch; negative before it. +fn unix_millis(time: SystemTime) -> i64 { + match time.duration_since(UNIX_EPOCH) { + Ok(after) => i64::try_from(after.as_millis()).unwrap_or(i64::MAX), + Err(before) => -i64::try_from(before.duration().as_millis()).unwrap_or(i64::MAX), + } +} + +fn format_unix_millis(unix_millis: i64) -> String { + let days = unix_millis.div_euclid(MILLIS_PER_DAY); + let millis_of_day = unix_millis.rem_euclid(MILLIS_PER_DAY); + let (year, month, day) = civil_from_days(days); + let seconds_of_day = millis_of_day / 1000; + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{:03}Z", + seconds_of_day / 3600, + seconds_of_day / 60 % 60, + seconds_of_day % 60, + millis_of_day % 1000, + ) +} + +fn civil_from_days(days: i64) -> (i64, i64, i64) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + (year, month, day) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn rfc3339_millis_formats_fixed_vectors() { + assert_eq!(format_unix_millis(0), "1970-01-01T00:00:00.000Z"); + assert_eq!( + format_unix_millis(1_792_000_870_205), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!( + format_unix_millis(951_782_400_000), + "2000-02-29T00:00:00.000Z" + ); + assert_eq!( + format_unix_millis(4_107_542_399_999), + "2100-02-28T23:59:59.999Z" + ); + assert_eq!(format_unix_millis(-1), "1969-12-31T23:59:59.999Z"); + assert_eq!( + rfc3339_millis(UNIX_EPOCH + Duration::from_millis(1_792_000_870_205)), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!( + rfc3339_millis(UNIX_EPOCH - Duration::from_millis(1)), + "1969-12-31T23:59:59.999Z" + ); + } +} diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index c1bdd6e0c..06d04bb04 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -42,6 +42,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`models.McpServerConfig`](./nemo_fabric.models.md#class-mcpserverconfig): MCP server configuration. - [`models.MetadataConfig`](./nemo_fabric.models.md#class-metadataconfig): Human-readable agent identity. - [`models.ModelConfig`](./nemo_fabric.models.md#class-modelconfig): Configuration for one model role. +- [`models.RecordConfig`](./nemo_fabric.models.md#class-recordconfig): Opt-in per-invocation run record. - [`models.RelayAtifConfig`](./nemo_fabric.models.md#class-relayatifconfig): NeMo Relay ATIF export configuration. - [`models.RelayAtofConfig`](./nemo_fabric.models.md#class-relayatofconfig): NeMo Relay ATOF export configuration. - [`models.RelayAtofFileSinkConfig`](./nemo_fabric.models.md#class-relayatoffilesinkconfig): NeMo Relay ATOF file sink configuration. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index db3a42208..703cff7b4 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -472,6 +472,78 @@ Returns the set of fields that have been explicitly set on this model instance. +**Returns:** + A set of strings representing the fields that have been set, i.e. that were not filled from defaults. + + + +--- + + +### classmethod `from_mapping` + +```python +def from_mapping(value: Mapping[str, Any]) -> Self +``` + +Validate a mapping using this Pydantic model. + +--- + + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached JSON-compatible mapping for Rust/core calls. + + +--- + + +## class `RecordConfig` + +Opt-in per-invocation run record. + +When ``enabled`` is true, NeMo Fabric writes ``/.fabric///run-record.jsonl`` for each invocation. It requires an artifact root, the ``local`` environment provider, and a ``process`` or ``python`` adapter. + + + +### Fields + +The model defines the following fields: + +| Field | Type | Required | Default | Constraints | Description | +| --- | --- | --- | --- | --- | --- | +| `enabled` | `StrictBool` | Yes | — | `Strict(strict=True)` | — | + +--- + +### property extra_fields + +Return fields preserved by the extension point for this model. + +--- + +### property model_extra + +Get extra fields set during validation. + + + +**Returns:** + A dictionary of extra fields, or `None` if `config.extra` is not set to `"allow"`. + +--- + +### property model_fields_set + +Returns the set of fields that have been explicitly set on this model instance. + + + **Returns:** A set of strings representing the fields that have been set, i.e. that were not filled from defaults. @@ -520,6 +592,7 @@ The model defines the following fields: | `artifacts` | `str \| Path \| None` | No | `None` | — | — | | `timeout_seconds` | `float \| None` | No | `None` | `Gt(gt=0), _PydanticGeneralMetadata(allow_inf_nan=False)` | — | | `max_turns` | `int \| None` | No | `None` | `Gt(gt=0), Le(le=4294967295)` | — | +| `record` | `RecordConfig \| None` | No | `None` | — | — | --- diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 52a3b6f57..feabb8df6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 125 +position: 126 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 26a286a0c..4c4e21b44 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 126 +position: 127 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 2c89a0e39..de4682271 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 127 +position: 128 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx index 40e387ace..65c0f3eaf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Kind" sidebar-title: "CapabilityKind" description: "Capability kind." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx index e121e6b73..010851195 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Target" sidebar-title: "CapabilityTarget" description: "Component responsible for executing a configured capability." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx index 74dc754f8..b1e6d19ac 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atif Storage Config" sidebar-title: "RelayAtifStorageConfig" description: "Relay ATIF remote storage configuration." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx index bbf8ff9e5..2e4357780 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Mode" sidebar-title: "RelayAtofMode" description: "Relay ATOF file mode." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx index d53f00f8d..d699b5db0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Sink Config" sidebar-title: "RelayAtofSinkConfig" description: "Relay ATOF sink configuration." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx index 892413a9e..9ee176668 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Field Name Policy" sidebar-title: "RelayAtofStreamFieldNamePolicy" description: "Relay ATOF stream field-name policy." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx index ea48753dd..a90cec5f3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Transport" sidebar-title: "RelayAtofStreamTransport" description: "Relay ATOF stream transport." -position: 80 +position: 81 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx index beb3da6eb..ba9321f6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Endpoint Type" sidebar-title: "RelayOpenTelemetryEndpointType" description: "Relay OpenTelemetry span projection." -position: 81 +position: 82 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx index da13057aa..9b78e9a7e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Mark Projection" sidebar-title: "RelayOpenTelemetryMarkProjection" description: "Relay OpenTelemetry mark projection policy." -position: 82 +position: 83 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx index 8db6e0872..a73f2ad1b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Otlp Transport" sidebar-title: "RelayOtlpTransport" description: "Relay OTLP transport." -position: 83 +position: 84 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx index 95ec26616..506a2c506 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Unsupported Behavior" sidebar-title: "RelayUnsupportedBehavior" description: "Relay unsupported/unknown config handling." -position: 84 +position: 85 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx index f130d1764..7e6f001b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx @@ -2,7 +2,7 @@ title: "Enum Resolution Strategy" sidebar-title: "ResolutionStrategy" description: "Adapter install or availability strategy." -position: 54 +position: 55 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx index 8ce5c9908..d1bfc189f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx @@ -2,7 +2,7 @@ title: "Enum Telemetry Provider" sidebar-title: "TelemetryProvider" description: "Telemetry runtime provider." -position: 64 +position: 65 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx index fc493bae7..de34aedc1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx @@ -2,7 +2,7 @@ title: "Function load_adapter_descriptor" sidebar-title: "load_adapter_descriptor" description: "Load an adapter descriptor from JSON package metadata." -position: 71 +position: 72 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx index 142fdba30..bd5cd97d5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx @@ -2,7 +2,7 @@ title: "Function load_adapter_target_descriptor" sidebar-title: "load_adapter_target_descriptor" description: "Load an Adapter Target Descriptor from declarative JSON metadata." -position: 72 +position: 73 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx index 6bc8a973b..a6d607703 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx @@ -2,7 +2,7 @@ title: "Function resolve_run_plan_from_config" sidebar-title: "resolve_run_plan_from_config" description: "Resolve a typed NVIDIA NeMo Fabric config into a runnable plan." -position: 73 +position: 74 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index fa7255361..7f45f6f6e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 128 +position: 129 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -53,6 +53,7 @@ NeMo Fabric config models and loading helpers. - [McpServerPlan](struct-mcpserverplan.mdx): Resolved MCP server exposure. - [MetadataConfig](struct-metadataconfig.mdx): Human-readable metadata. - [ModelConfig](struct-modelconfig.mdx): Model configuration. +- [RecordConfig](struct-recordconfig.mdx): Per-invocation run record settings. - [RelayAtifConfig](struct-relayatifconfig.mdx): Relay ATIF export configuration. - [RelayAtofConfig](struct-relayatofconfig.mdx): Relay ATOF export configuration. - [RelayComponentConfig](struct-relaycomponentconfig.mdx): Generic NeMo Relay plugin component configuration. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx new file mode 100644 index 000000000..d33e42889 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx @@ -0,0 +1,108 @@ +--- +title: "Struct Record Config" +sidebar-title: "RecordConfig" +description: "Per-invocation run record settings." +position: 54 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
bool,\n}"}} />
+ +Per-invocation run record settings. + +When enabled, core appends a start line and an end line for each local invocation to `/.fabric///run-record.jsonl`. + +## Fields + +### `enabled: bool` + +Whether to write a run record for each invocation. Requires an artifact root, the `local` environment provider, and a `process` or `python` adapter. + +## Trait Implementations + +### `impl Clone for RecordConfig` + +
Clone for RecordConfig"}} />
+ +#### `clone` + +
clone(&self) -> RecordConfig"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RecordConfig` + +
Debug for RecordConfig"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for RecordConfig` + +
Default for RecordConfig"}} />
+ +#### `default` + +
default() -> RecordConfig"}} />
+ +### `impl<'de> Deserialize<'de> for RecordConfig` + +
Deserialize<'de> for RecordConfig"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RecordConfig` + +
RecordConfig"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RecordConfig` + +
PartialEq for RecordConfig"}} />
+ +#### `eq` + +
eq(&self, other: &RecordConfig) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RecordConfig` + +
Serialize for RecordConfig"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for RecordConfig` + +
Eq for RecordConfig"}} />
+ +### `impl StructuralPartialEq for RecordConfig` + +
StructuralPartialEq for RecordConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx index 4a2989ac6..030a4323a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Atif Config" sidebar-title: "RelayAtifConfig" description: "Relay ATIF export configuration." -position: 38 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx index 4ff5d7de9..d595a140a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Atof Config" sidebar-title: "RelayAtofConfig" description: "Relay ATOF export configuration." -position: 39 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx index 56259d7a5..4c490558c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Component Config" sidebar-title: "RelayComponentConfig" description: "Generic NeMo Relay plugin component configuration." -position: 40 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx index 570ef8d8e..ffc52a18c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Config" sidebar-title: "RelayConfig" description: "NeMo Relay integration configuration." -position: 41 +position: 42 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx index 712c72795..8d9e4a386 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Config Policy" sidebar-title: "RelayConfigPolicy" description: "Relay validation policy." -position: 42 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx index a01db8716..4db7528da 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Observability Config" sidebar-title: "RelayObservabilityConfig" description: "NeMo Relay observability component configuration." -position: 43 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx index 4bade1462..922156d9f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Open Telemetry Config" sidebar-title: "RelayOpenTelemetryConfig" description: "Relay OpenTelemetry export section." -position: 44 +position: 45 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx index da8138f4e..6bb243f6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Open Telemetry Endpoint Config" sidebar-title: "RelayOpenTelemetryEndpointConfig" description: "One typed Relay OpenTelemetry OTLP destination." -position: 45 +position: 46 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx index 8e70ade88..bf1537e85 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Resolve Context" sidebar-title: "ResolveContext" description: "Source context used when resolving an in-memory NeMo Fabric config." -position: 55 +position: 56 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx index 7c1e65663..34ad4c681 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx @@ -2,7 +2,7 @@ title: "Struct Resolved Adapter Descriptor" sidebar-title: "ResolvedAdapterDescriptor" description: "Adapter descriptor selected for a run plan." -position: 56 +position: 57 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx index 8adac3efc..b3be2d837 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx @@ -2,7 +2,7 @@ title: "Struct Resolved Adapter Target Descriptor" sidebar-title: "ResolvedAdapterTargetDescriptor" description: "Adapter target descriptor selected for a run plan." -position: 57 +position: 58 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx index e3c43e4e1..b8f75e62d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx @@ -2,7 +2,7 @@ title: "Struct RunPlan" sidebar-title: "RunPlan" description: "Resolved NeMo Fabric run plan." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx index 0d972cb87..5c2282536 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Capabilities" sidebar-title: "RuntimeCapabilities" description: "Lifecycle behavior implemented by a resolved runtime path." -position: 59 +position: 60 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx index 2c89a11b2..d9b6e6015 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Runtime Config" sidebar-title: "RuntimeConfig" description: "Invocation runtime contract." -position: 60 +position: 61 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub record: Option<RecordConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Invocation runtime contract. @@ -35,6 +35,10 @@ Maximum duration of one invocation in seconds. Maximum number of harness turns within one invocation. +### `record: Option` + +Opt-in per-invocation run record. Core-only; never sent to adapters. + ### `extensions: BTreeMap` Additive normalized runtime fields. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx index 47644f228..1f4e2fe76 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Skill Config" sidebar-title: "SkillConfig" description: "Skill capability configuration." -position: 61 +position: 62 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx index 2e1b4af6b..61e3aa7d0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Config" sidebar-title: "TelemetryConfig" description: "Telemetry configuration." -position: 62 +position: 63 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx index f12176331..326bcdbb5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Plan" sidebar-title: "TelemetryPlan" description: "Resolved telemetry plan." -position: 63 +position: 64 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx index f683b0886..bef871340 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Provider Config" sidebar-title: "TelemetryProviderConfig" description: "Provider-specific telemetry configuration." -position: 65 +position: 66 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx index b5e0b460f..771156fff 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Tool Definition Config" sidebar-title: "ToolDefinitionConfig" description: "One named normalized tool or tool-group definition." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx index 64b2d6eec..f9f3714b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Tools Config" sidebar-title: "ToolsConfig" description: "Harness-neutral tool capability configuration." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx index d2b51fe8b..ffe368f07 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx @@ -2,7 +2,7 @@ title: "Struct Tools Plan" sidebar-title: "ToolsPlan" description: "Normalized tool policy for a run." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx index 21beec0df..ab0796f56 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Workflow Config" sidebar-title: "WorkflowConfig" description: "Registered workflow target and immutable construction settings." -position: 68 +position: 69 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx index a002d8fc3..03fecb115 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Workflow Entrypoint Config" sidebar-title: "WorkflowEntrypointConfig" description: "Adapter-owned workflow entry point supplied by an Adapter Target Descriptor." -position: 69 +position: 70 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx index e66041abb..a0e9695a1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx @@ -2,7 +2,7 @@ title: "Struct Workflow Target Spec" sidebar-title: "WorkflowTargetSpec" description: "Workflow-specific Adapter Target Descriptor fields." -position: 70 +position: 71 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx index d8c0693d4..bbae2fe19 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx @@ -2,7 +2,7 @@ title: "Enum Doctor Status" sidebar-title: "DoctorStatus" description: "Diagnostic status." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx index bb8aa23e8..b97461157 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx @@ -2,7 +2,7 @@ title: "Function doctor_plan" sidebar-title: "doctor_plan" description: "Inspect a resolved run plan without mutating the environment." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index ea299ed94..4f6f26b00 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 129 +position: 130 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx index c64d5792e..9ae58d2b6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx @@ -2,7 +2,7 @@ title: "Struct Doctor Check" sidebar-title: "DoctorCheck" description: "Diagnostic check result." -position: 74 +position: 75 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx index c7d65e76f..c015cc391 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx @@ -2,7 +2,7 @@ title: "Struct Doctor Report" sidebar-title: "DoctorReport" description: "Diagnostic report for a resolved run plan." -position: 75 +position: 76 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 95cf2f59a..70e1c1cc4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -2,7 +2,7 @@ title: "Enum Fabric Error" sidebar-title: "FabricError" description: "Errors raised by NeMo Fabric config loading and validation." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index cf7208b32..e389cbb01 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 130 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx index 0a91b0129..6f11a50d8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx @@ -2,7 +2,7 @@ title: "Type Alias Result" sidebar-title: "Result" description: "Core NeMo Fabric result type." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 8af97f312..f9c667fa9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 133 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 6670ab4d8..94890397c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -66,6 +66,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use config::MetadataConfig;` - `pub use config::ModelConfig;` - `pub use config::OAuthTokenEndpointAuthMethod;` +- `pub use config::RecordConfig;` - `pub use config::ResolutionStrategy;` - `pub use config::ResolveContext;` - `pub use config::ResolvedAdapterDescriptor;` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index e4ce802a1..3b8f07380 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 131 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index 9ae563e23..041dbb1fa 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -32,6 +32,7 @@ pub enum SchemaName { ArtifactManifest, ErrorInfo, FabricEvent, + RunRecordLine, } ``` @@ -165,6 +166,12 @@ Structured error metadata schema. NeMo Fabric lifecycle event schema. +### `RunRecordLine` + +
+ +One line of an opt-in per-invocation run record, version `v1alpha1`. + ## Implementations ### `impl SchemaName` @@ -173,7 +180,7 @@ NeMo Fabric lifecycle event schema. #### `ALL` -
21]"}} />
+
22]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 0569b34b2..722c3a849 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 132 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 227e0bd44..555e6ba5c 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -237,7 +237,7 @@ Relay subfields: | `models..top_p`, `.max_tokens` | No | No | Yes | Yes | Yes; passed through LiteLLM | | `instructions.system` | `replace`, `append` | `replace`; maps to Codex base instructions | `replace` | `replace` | `replace` | | `runtime.input_schema`, `.output_schema` | Core | Core | Core | Core | Core | -| `runtime.artifacts`, `.timeout_seconds` | Core | Core | Core | Core | Core | +| `runtime.artifacts`, `.timeout_seconds`, `.record` | Core | Core | Core | Core | Core | | `runtime.max_turns` | Yes | No | Yes; maps to LangGraph supersteps | Yes; maps to Hermes iterations | Yes | | `environment.provider`, `.control_location`, `.ownership` | Core | Core | Core | Core | Core | | `environment.workspace`, `.artifacts`, `.env`, `.env_names` | Core | Core | Core | Core | Core | @@ -941,6 +941,78 @@ If NeMo Fabric cannot resolve config, start a runtime, or obtain a normalized re the SDK raises a `FabricError` subclass instead of returning a partial `RunResult`. +## Run Records + +A run record is an append-only JSON Lines file that NeMo Fabric writes for each +invocation, so a run that is killed still leaves its identity, start time, and +timeout behind. Recording is off by default. Setting `runtime.artifacts` alone +does not turn it on. Turn it on with `runtime.record`: + +```python +from nemo_fabric import RecordConfig, RuntimeConfig + +runtime = RuntimeConfig(artifacts="runs", record=RecordConfig(enabled=True)) +``` + +`enabled` is required inside `record`. The record needs an artifact root +(`runtime.artifacts` or `environment.artifacts`), the `local` environment +provider, and a `process` or `python` adapter. Planning fails with +`runtime.record.enabled` when any of these is missing. + +Each invocation that NeMo Fabric tries to send to the adapter host writes +`/.fabric///run-record.jsonl`, and the +result lists it as the `record` artifact with kind `run_record`: + +```python +record = next(a for a in result.artifacts.artifacts if a.kind == "run_record") +``` + +The file holds a start line and an end line. Every line carries `schema` +(`nemo.fabric.run_record`), `version` (`v1alpha1`), `kind`, `runtime_id`, +`invocation_id`, `request_id`, and `time`. + +- The `start` line is written before the request is sent. It holds the agent + name, harness, adapter ID, each configured model's provider and model name, + the NeMo Fabric version, and the timeout in milliseconds. From + `RunRequest.context`, it keeps the string or integer values of `task_id`, + `task_revision`, `attempt`, `step`, `harbor_session_id`, `harbor_context_id`, + and `harbor_trial_name`, cut to 256 characters. Every other context key + appears by name only as `{"recorded": false}`. `override_keys` lists the + top-level `RunRequest.overrides` key names, sorted, without their values. +- The `end` line is written after the invocation returns or fails. It holds + `status`, `elapsed_ms`, the error's `code`, `stage`, and `retryable` flag, + the numeric `RunUsage` counters, and each result artifact's name, kind, + media type, and path relative to the artifact root. An error code outside + `^[a-z0-9_.:-]{1,128}$` is written as `unrecognized`. When NeMo Fabric + raises instead of returning a result, the end line reads `failed`. An + adapter host error keeps its code, such as `host_timeout`, `host_crashed`, + `host_io`, or `invalid_agent_run_result`. A result extension that the + adapter's descriptor does not declare also reads `invalid_agent_run_result`. + A failed artifact write reads `artifact_write_failed`, and any other error + reads `fabric_error`. + +The end line describes the invocation. A one-shot `Fabric.run` stops the +runtime after the invocation; when that stop fails, the result reads `failed` +with stage `stop`, and the record still holds the invocation's own end line. + +`run-record.jsonl` never holds prompts, model output, error messages, tool +payloads, adapter metadata, or context values outside the keys above. The same +folder also holds `adapter-invocation.json` and `stdout.txt`, which NeMo Fabric +writes for every invocation whether or not recording is on; those files are +not covered by this rule. + +Each line is written with one append, so a killed process keeps every line +whose write returned. NeMo Fabric syncs the file once, after the end line. A +record without an end line belongs to an invocation that is still running or +that never finished; treat its outcome as unknown, not as failed. If a write +fails, the record stops, the result gains a `record_write_failed` event, and +the invocation continues unchanged. A record whose end line failed to write +therefore has no end line, and that event says why. The file is created with +mode `0600` on Linux and macOS, and never through a symbolic link. + +The line format is published as `schemas/sdk/run-record-line.schema.json` at +version `v1alpha1`, which may change until a reader for run records ships. + ## Events Each `RunResult` includes the normalized events collected for that invocation. diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 43867290c..af78b2daf 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -127,6 +127,9 @@ descriptor schema. - `sdk/artifact-manifest`: normalized artifact references. - `sdk/error-info`: structured runtime or adapter error metadata. - `sdk/fabric-event`: NeMo Fabric lifecycle or progress event. +- `sdk/run-record-line`: one line of the opt-in per-invocation run record, + `run-record.jsonl`. The schema is `v1alpha1` and may change until a reader + for run records ships. ### Deferred Core Objects diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index 720929df3..7d6c5e994 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -558,6 +558,20 @@ } ] }, + "RecordConfig": { + "additionalProperties": false, + "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "properties": { + "enabled": { + "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -1329,6 +1343,17 @@ "description": "Output schema label.", "type": "string" }, + "record": { + "anyOf": [ + { + "$ref": "#/$defs/RecordConfig" + }, + { + "type": "null" + } + ], + "description": "Opt-in per-invocation run record. Core-only; never sent to adapters." + }, "timeout_seconds": { "description": "Maximum duration of one invocation in seconds.", "exclusiveMinimum": 0.0, diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index b33341bf2..5d3211b4e 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -1926,6 +1926,20 @@ } ] }, + "RecordConfig": { + "additionalProperties": false, + "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "properties": { + "enabled": { + "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -2772,6 +2786,17 @@ "description": "Output schema label.", "type": "string" }, + "record": { + "anyOf": [ + { + "$ref": "#/$defs/RecordConfig" + }, + { + "type": "null" + } + ], + "description": "Opt-in per-invocation run record. Core-only; never sent to adapters." + }, "timeout_seconds": { "description": "Maximum duration of one invocation in seconds.", "exclusiveMinimum": 0.0, diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json new file mode 100644 index 000000000..7b0f8642e --- /dev/null +++ b/schemas/sdk/run-record-line.schema.json @@ -0,0 +1,429 @@ +{ + "$defs": { + "ContextValue": { + "anyOf": [ + { + "description": "An allowlisted key's string value, cut to 256 characters.", + "type": "string" + }, + { + "description": "An allowlisted key's integer value.", + "format": "int64", + "type": "integer" + }, + { + "$ref": "#/$defs/NotRecorded", + "description": "Any other key or value: the key name only." + } + ], + "description": "A request context value as the start line keeps it." + }, + "EndArtifact": { + "description": "One result artifact on the end line.", + "properties": { + "kind": { + "description": "Artifact kind.", + "type": "string" + }, + "media_type": { + "description": "Media type.", + "type": [ + "string", + "null" + ] + }, + "name": { + "description": "Logical artifact name.", + "type": "string" + }, + "path": { + "description": "Path relative to the artifact root; absent for a path outside it.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "name", + "kind" + ], + "type": "object" + }, + "EndError": { + "description": "The end line's error.", + "properties": { + "code": { + "description": "Error code matching `^[a-z0-9_.:-]{1,128}$`, else `unrecognized`.", + "pattern": "^[a-z0-9_.:-]{1,128}$", + "type": "string" + }, + "retryable": { + "description": "Whether a consumer-level retry is considered safe.", + "type": "boolean" + }, + "stage": { + "$ref": "#/$defs/ErrorStage", + "description": "Lifecycle stage where the failure surfaced." + } + }, + "required": [ + "code", + "stage", + "retryable" + ], + "type": "object" + }, + "EndLine": { + "description": "Fields of the end line.", + "properties": { + "artifacts": { + "description": "Result artifacts, with paths relative to the artifact root.", + "items": { + "$ref": "#/$defs/EndArtifact" + }, + "type": "array" + }, + "elapsed_ms": { + "description": "Monotonic time from the start line to the end line, in milliseconds.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/EndError" + }, + { + "type": "null" + } + ], + "description": "Error code, stage and retryable flag, without the message." + }, + "status": { + "$ref": "#/$defs/RunStatus", + "description": "Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`." + }, + "usage": { + "anyOf": [ + { + "$ref": "#/$defs/EndUsage" + }, + { + "type": "null" + } + ], + "description": "Numeric usage counters, without adapter metadata." + } + }, + "required": [ + "status", + "elapsed_ms" + ], + "type": "object" + }, + "EndUsage": { + "description": "The end line's usage: the numeric `RunUsage` fields.", + "properties": { + "cached_input_tokens": { + "description": "Cached input tokens.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Reported cost in US dollars.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Input tokens.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether `input_tokens` includes `cached_input_tokens`; absent means unknown.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Output tokens.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Total tokens reported by the provider.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "ErrorStage": { + "description": "NeMo Fabric lifecycle stage associated with an error.", + "oneOf": [ + { + "const": "config", + "description": "Configuration resolution failed.", + "type": "string" + }, + { + "const": "plan", + "description": "Effective config planning failed.", + "type": "string" + }, + { + "const": "prepare", + "description": "Environment preparation failed.", + "type": "string" + }, + { + "const": "start", + "description": "Runtime start/connect failed.", + "type": "string" + }, + { + "const": "invoke", + "description": "Runtime invocation failed.", + "type": "string" + }, + { + "const": "stop", + "description": "Runtime stop/detach failed.", + "type": "string" + }, + { + "const": "release", + "description": "Environment release failed.", + "type": "string" + }, + { + "const": "artifact", + "description": "Artifact collection or writing failed.", + "type": "string" + } + ] + }, + "ModelName": { + "description": "One configured model.", + "properties": { + "model": { + "description": "Provider model identifier.", + "type": "string" + }, + "provider": { + "description": "Model provider name.", + "type": "string" + } + }, + "required": [ + "provider", + "model" + ], + "type": "object" + }, + "NotRecorded": { + "description": "Marks a key whose value the record leaves out.", + "properties": { + "recorded": { + "const": false, + "description": "Always `false`.", + "type": "boolean" + } + }, + "required": [ + "recorded" + ], + "type": "object" + }, + "RecordSchema": { + "description": "The record's schema name.", + "oneOf": [ + { + "const": "nemo.fabric.run_record", + "description": "`nemo.fabric.run_record`.", + "type": "string" + } + ] + }, + "RecordVersion": { + "description": "The record's line format version.", + "oneOf": [ + { + "const": "v1alpha1", + "description": "`v1alpha1`.", + "type": "string" + } + ] + }, + "RunStatus": { + "description": "Runtime completion status.", + "oneOf": [ + { + "const": "succeeded", + "description": "The harness completed successfully.", + "type": "string" + }, + { + "const": "failed", + "description": "The harness completed with a non-zero status.", + "type": "string" + }, + { + "const": "cancelled", + "description": "The invocation or runtime was cancelled.", + "type": "string" + } + ] + }, + "StartLine": { + "description": "Fields of the start line.", + "properties": { + "adapter_id": { + "description": "Adapter implementation id.", + "type": [ + "string", + "null" + ] + }, + "agent_name": { + "description": "Config name (`metadata.name`).", + "type": "string" + }, + "context": { + "additionalProperties": { + "$ref": "#/$defs/ContextValue" + }, + "description": "`RunRequest.context`. Allowlisted identity and Harbor link keys keep a\nstring or integer value; every other key is listed by name only.", + "type": "object" + }, + "fabric_version": { + "description": "NeMo Fabric version that wrote the record.", + "type": "string" + }, + "harness": { + "description": "Harness identifier.", + "type": "string" + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelName" + }, + "description": "Configured models by role: provider and model name only.", + "type": "object" + }, + "override_keys": { + "description": "Top-level key names of `RunRequest.overrides`, sorted. Values are never\nrecorded.", + "items": { + "type": "string" + }, + "type": "array" + }, + "timeout_ms": { + "description": "Host invocation timeout that applied, in milliseconds.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "agent_name", + "harness", + "fabric_version", + "timeout_ms" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "One line of a run record.", + "oneOf": [ + { + "$ref": "#/$defs/StartLine", + "description": "Written once, before core sends the request to the adapter host.", + "properties": { + "kind": { + "const": "start", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "$ref": "#/$defs/EndLine", + "description": "Written once, after the invocation returned a result or failed.", + "properties": { + "kind": { + "const": "end", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ], + "properties": { + "invocation_id": { + "description": "Invocation id.", + "type": "string" + }, + "request_id": { + "description": "Request id.", + "type": "string" + }, + "runtime_id": { + "description": "Runtime id.", + "type": "string" + }, + "schema": { + "$ref": "#/$defs/RecordSchema", + "description": "Always `nemo.fabric.run_record`." + }, + "time": { + "description": "UTC wall-clock time the line was written, RFC 3339 with milliseconds.", + "format": "date-time", + "type": "string" + }, + "version": { + "$ref": "#/$defs/RecordVersion", + "description": "Line format version. `v1alpha1` may change until a record reader ships." + } + }, + "required": [ + "schema", + "version", + "runtime_id", + "invocation_id", + "request_id", + "time" + ], + "title": "RunRecordLine", + "type": "object" +} \ No newline at end of file diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index aaa15bfe0..a585aac9d 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -24,6 +24,7 @@ from nemo_fabric.models import McpServerConfig from nemo_fabric.models import MetadataConfig from nemo_fabric.models import ModelConfig +from nemo_fabric.models import RecordConfig from nemo_fabric.models import RelayAtifConfig from nemo_fabric.models import RelayAtofConfig from nemo_fabric.models import RelayAtofFileSinkConfig @@ -94,6 +95,7 @@ "MetadataConfig", "ModelConfig", "OpenAIInvokeStream", + "RecordConfig", "RelayAtifConfig", "RelayAtofConfig", "RelayAtofFileSinkConfig", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index d72220f03..66ca0ba7d 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -25,6 +25,7 @@ from pydantic import ConfigDict from pydantic import Field from pydantic import SerializerFunctionWrapHandler +from pydantic import StrictBool from pydantic import field_validator from pydantic import model_serializer from pydantic import model_validator @@ -187,6 +188,20 @@ class InstructionsConfig(FabricBaseModel): system: InstructionConfig | None = None +class RecordConfig(FabricBaseModel): + """Opt-in per-invocation run record. + + When ``enabled`` is true, NeMo Fabric writes + ``/.fabric///run-record.jsonl`` + for each invocation. It requires an artifact root, the ``local`` + environment provider, and a ``process`` or ``python`` adapter. + """ + + model_config = ConfigDict(extra="forbid") + + enabled: StrictBool + + class RuntimeConfig(FabricBaseModel): """Invocation runtime contract.""" @@ -199,6 +214,7 @@ class RuntimeConfig(FabricBaseModel): allow_inf_nan=False, ) max_turns: int | None = Field(default=None, gt=0, le=(1 << 32) - 1) + record: RecordConfig | None = None class EnvironmentConfig(FabricBaseModel): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 04504e4dc..0456b68ff 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -372,11 +372,19 @@ class _RuntimeConfig(_ConfigMapping): artifacts: Optional artifact-root path. timeout_seconds: Optional invocation deadline in seconds. max_turns: Optional harness turn limit. + record: Optional run record settings, ``{"enabled": bool}``. extra_fields: Preserved extension fields not recognized by this SDK. """ _fields = frozenset( - {"input_schema", "output_schema", "artifacts", "timeout_seconds", "max_turns"} + { + "input_schema", + "output_schema", + "artifacts", + "timeout_seconds", + "max_turns", + "record", + } ) def __init__( @@ -387,6 +395,7 @@ def __init__( artifacts: str | Path | None = None, timeout_seconds: float | None = None, max_turns: int | None = None, + record: Mapping[str, Any] | None = None, extra_fields: Mapping[str, Any] | None = None, ) -> None: values: dict[str, Any] = {} @@ -419,6 +428,13 @@ def __init__( "runtime max_turns must be between 1 and 4294967295" ) values["max_turns"] = max_turns + if record is not None: + record = _mapping(record, "runtime record") + if set(record) != {"enabled"} or not isinstance(record["enabled"], bool): + raise FabricConfigError( + "runtime record must be an object with only a boolean enabled field" + ) + values["record"] = {"enabled": record["enabled"]} super().__init__(values, extra_fields=extra_fields) @classmethod @@ -432,6 +448,7 @@ def from_mapping(cls, value: Mapping[str, Any]) -> "_RuntimeConfig": artifacts=data.get("artifacts"), timeout_seconds=data.get("timeout_seconds"), max_turns=data.get("max_turns"), + record=data.get("record"), extra_fields={ key: item for key, item in data.items() if key not in cls._fields }, diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index f2660d5e3..ebfeb1896 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -46,6 +46,21 @@ if result.usage is not None: Treat missing counters and cost as unknown, not zero. `cached_input_tokens` reports cached prompt tokens; `input_tokens_include_cache` declares whether `input_tokens` includes them (`True`), excludes them (`False`), or has unknown semantics (absent). Do not add cache to an inclusive input count. Adapters can put cache-write, reasoning, peak request, and per-model detail in the descriptor-declared `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` object. NeMo Fabric preserves that object at `result.usage["metadata"]["nemo_fabric.detailed_token_usage.v1"]`. Treat detailed input and output independently: any detailed accounting field for a dimension makes that dimension authoritative, with no fallback to its top-level counters. The views can diverge. Calculate an inclusive total only when the selected detailed counters provide enough information; otherwise leave the total unknown. Keep estimated costs separate in provider metadata; `cost_usd` is reported cost, not an estimate. +## Run Records + +Set `RuntimeConfig(artifacts=..., record=RecordConfig(enabled=True))` to have +NeMo Fabric write `run-record.jsonl` for each invocation under +`/.fabric///`. Recording is off by +default and needs an artifact root, the `local` environment provider, and a +`process` or `python` adapter. The +result lists the file as the artifact with kind `run_record`. The file has a +`start` line, written before the request, and an `end` line with status, +elapsed time, error code, numeric usage, and artifact names. It holds no +prompts, output, or error messages. A record without an end line has an +unknown outcome; do not count it as failed. The format is the `v1alpha1` +[run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) +and may change until a reader ships. + ## Correlation IDs `runtime_id` identifies the runtime lifecycle, `invocation_id` identifies one diff --git a/tests/python/test_sdk_consumer.py b/tests/python/test_sdk_consumer.py index bac8db3e3..a9e5e5727 100644 --- a/tests/python/test_sdk_consumer.py +++ b/tests/python/test_sdk_consumer.py @@ -10,7 +10,7 @@ from unittest.mock import MagicMock import pytest -from nemo_fabric import Fabric, FabricConfigError, RunPlan +from nemo_fabric import Fabric, FabricConfigError, RecordConfig, RunPlan from _utils.configs import hermes_shim_config @@ -148,3 +148,16 @@ async def test_installed_descriptors_reject_invalid_config_before_native_start( base_dir=hermes_shim_agent_dir, ) mock_start.assert_not_called() + + +def test_record_without_an_artifact_root_fails_native_planning(hermes_shim_agent_dir): + config = hermes_shim_config() + config.runtime.artifacts = None + config.environment.artifacts = None + config.runtime.record = RecordConfig(enabled=True) + + with pytest.raises(FabricConfigError, match="runtime.record.enabled"): + Fabric().plan(config, base_dir=hermes_shim_agent_dir) + + config.runtime.artifacts = "./artifacts" + assert Fabric().plan(config, base_dir=hermes_shim_agent_dir).config.runtime.record == {"enabled": True} diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index 5361e4697..e527e819a 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -36,6 +36,7 @@ from nemo_fabric import McpConfig from nemo_fabric import McpServerConfig from nemo_fabric import MetadataConfig +from nemo_fabric import RecordConfig from nemo_fabric import RelayAtifConfig from nemo_fabric import RelayAtofConfig from nemo_fabric import RelayAtofFileSinkConfig @@ -698,6 +699,28 @@ def test_instruction_content_must_be_non_empty(content: str): _FabricConfigSnapshot.from_mapping(raw) +def test_runtime_record_round_trips_and_rejects_other_shapes(): + config = FabricConfig( + metadata=MetadataConfig(name="demo"), + harness=HarnessConfig(adapter_id="test.fabric.shim"), + runtime=RuntimeConfig(artifacts="runs", record=RecordConfig(enabled=True)), + ) + + assert config.to_mapping()["runtime"]["record"] == {"enabled": True} + assert "record" not in RuntimeConfig().to_mapping() + for invalid in ({}, {"enabled": "yes"}, {"enabled": True, "durable": True}): + with pytest.raises(ValidationError): + RecordConfig.model_validate(invalid) + + raw = _plan()["config"] + raw["runtime"] = {"record": {"enabled": True}} + assert _FabricConfigSnapshot.from_mapping(raw).runtime.record == {"enabled": True} + for invalid in ({}, {"enabled": 1}, {"enabled": True, "durable": True}): + raw["runtime"]["record"] = invalid + with pytest.raises(FabricConfigError, match="boolean enabled field"): + _FabricConfigSnapshot.from_mapping(raw) + + def test_max_turns_matches_rust_u32_range(): maximum = (1 << 32) - 1 From 3d0b6edab8886c743ac5a953e51cbdd495c1d68a Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:31:15 -0700 Subject: [PATCH 019/159] fix: check the parent's generation before counting a Linux child The Linux tree walk accepted a child when its stat named the queued parent PID. If the parent and child both exited and other processes reused both PIDs with the same parent-child relation, the walk counted the unrelated process. The walk now reads the parent again after the child and counts the child only while the parent still has the start time it was counted with: a parent that is the same process after the child's read held its PID the whole time, so the child's parent PID names it. A parent that exited no longer shows that, so the child is skipped like a reparented one, and a parent that cannot be read makes the snapshot unavailable. Both Linux listings use the shared walk, so the check covers the children files and the fallback scan. The scripted regressions fail without it with 3 processes and 900 ns of CPU time from the unrelated process. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 185 +++++++++++++++++--- 1 file changed, 157 insertions(+), 28 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index a3a3be622..d362a979d 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -197,15 +197,40 @@ mod tree { fn read(&self, pid: u32) -> Result; } - /// Whether `process`, listed as a child of `parent` at `listed_at` and - /// read afterwards, is still the listed process. A parent PID read with - /// the record shows it directly. Without one, a process that started - /// before the list was read already held its PID then, so it is the - /// listed process. - fn is_listed_child(process: &Process, parent: u32, listed_at: u64) -> bool { + /// The process that listed a child, as the tree read saw it. + #[derive(Debug, Clone, Copy)] + struct Parent { + pid: u32, + /// The parent's start time when the tree read counted it. + start: u64, + /// When the parent's children were listed. + listed_at: u64, + } + + /// Whether `process`, listed as a child of `parent` and read afterwards, + /// is still a child of that same parent process. + /// + /// With a parent PID in the record, the parent is read again after the + /// child. A parent that still has the start time it was counted with + /// held its PID throughout, so the child's parent PID names it and not a + /// later process that reused its PID. A parent that exited no longer + /// shows that, so its child is not counted, like a reparented child. + /// + /// Without a parent PID, a process that started before the list was read + /// already held its PID then, so it is the listed process. + fn is_listed_child( + table: &impl ProcessTable, + process: &Process, + parent: Parent, + ) -> Result { match process.ppid { - Some(ppid) => ppid == parent, - None => process.start < listed_at, + Some(ppid) if ppid != parent.pid => Ok(false), + Some(_) => match table.read(parent.pid) { + Ok(now) => Ok(now.start == parent.start), + Err(ReadError::Gone) => Ok(false), + Err(ReadError::Unavailable(reason)) => Err(reason), + }, + None => Ok(process.start < parent.listed_at), } } @@ -218,38 +243,41 @@ mod tree { identity: u64, ) -> Result { let mut read = TreeRead::default(); - // Each entry is a PID and, for a child, its parent and the time the - // parent's children were listed. + // Each entry is a PID and, for a child, the parent that listed it. let mut pending = VecDeque::from([(root, None)]); - while let Some((pid, listed)) = pending.pop_front() { + while let Some((pid, parent)) = pending.pop_front() { // List the children before reading the process: a read that still // shows the expected process proves the listed children were its own. let found = table .children(pid) .and_then(|children| Ok((table.read(pid)?, children))); - let expected = |process: &Process| match listed { - None => process.start == identity && !process.exited, - Some((parent, listed_at)) => is_listed_child(process, parent, listed_at), - }; - let (process, children) = match found { - Ok((process, children)) if expected(&process) => (process, children), - // A reaped child, or one whose PID now names another process. - Ok(_) | Err(ReadError::Gone) if listed.is_some() => continue, - Ok(_) | Err(ReadError::Gone) => return Err(HOST_EXITED), - Err(ReadError::Unavailable(reason)) => return Err(reason), + let (process, children) = match (found, parent) { + (Ok((process, children)), None) if process.start == identity && !process.exited => { + (process, children) + } + (Ok(_) | Err(ReadError::Gone), None) => return Err(HOST_EXITED), + (Ok((process, children)), Some(parent)) => { + if !is_listed_child(table, &process, parent)? { + // Its PID now names another process, or it left the tree. + continue; + } + (process, children) + } + // A reaped child. + (Err(ReadError::Gone), Some(_)) => continue, + (Err(ReadError::Unavailable(reason)), _) => return Err(reason), }; read.cpu_nanos = read.cpu_nanos.saturating_add(process.cpu_nanos); if !process.exited { read.resident_bytes = read.resident_bytes.saturating_add(process.resident_bytes); read.process_count += 1; } - let listed_at = children.listed_at; - pending.extend( - children - .pids - .into_iter() - .map(|child| (child, Some((pid, listed_at)))), - ); + let parent = Parent { + pid, + start: process.start, + listed_at: children.listed_at, + }; + pending.extend(children.pids.into_iter().map(|child| (child, Some(parent)))); } Ok(read) } @@ -900,6 +928,107 @@ mod tests { } } + /// How the Linux reader lists children. + #[derive(Debug, Clone, Copy)] + enum Listing { + /// `/proc//task//children`, read when the walk reaches the process. + ChildrenFiles, + /// One pass over every process's parent, made before the walk. + Scan, + } + + const LISTINGS: [Listing; 2] = [Listing::ChildrenFiles, Listing::Scan]; + + /// A table in which other processes take some PIDs once the walk has + /// read `reused_after`: `before` answers until then and `after` from then + /// on. A scan lists children from `before`, because it ran before the walk. + struct PidReuseTable { + before: ScriptedTable, + after: ScriptedTable, + reused_after: u32, + reused: std::cell::Cell, + listing: Listing, + } + + impl PidReuseTable { + fn current(&self) -> &ScriptedTable { + if self.reused.get() { + &self.after + } else { + &self.before + } + } + } + + impl ProcessTable for PidReuseTable { + fn children(&self, pid: u32) -> std::result::Result { + match self.listing { + Listing::ChildrenFiles => self.current().children(pid), + Listing::Scan => self.before.children(pid), + } + } + + fn read(&self, pid: u32) -> std::result::Result { + let read = self.current().read(pid); + if pid == self.reused_after { + self.reused.set(true); + } + read + } + } + + /// The host's child 11 and grandchild 12 exit right after the walk reads + /// 11. `after_11` is what PID 11 then reads as; PID 12 is an unrelated + /// process whose parent PID is 11 and which used 900 ns of CPU time. + fn read_tree_after_reuse( + listing: Listing, + after_11: std::result::Result, + ) -> std::result::Result { + let lineage = Lineage::ParentPid; + let before = ScriptedTable::default() + .process(HOST, Ok(vec![11]), Ok(lineage.host(7))) + .process(11, Ok(vec![12]), Ok(lineage.child(HOST, 3))) + .process(12, Ok(vec![]), Ok(lineage.child(11, 2))); + let after = ScriptedTable::default() + .process(HOST, Ok(vec![]), Ok(lineage.host(7))) + .process(11, Ok(vec![12]), after_11) + .process(12, Ok(vec![]), Ok(lineage.child(11, 900))); + let table = PidReuseTable { + before, + after, + reused_after: 11, + reused: std::cell::Cell::new(false), + listing, + }; + read_tree_with(&table, HOST, HOST_START) + } + + #[test] + fn tree_read_skips_child_whose_parent_pid_was_reused() { + for listing in LISTINGS { + let unrelated_parent = Process { + ppid: Some(1), + start: LISTED_AT + 50, + ..Lineage::ParentPid.child(1, 500) + }; + for after_11 in [Ok(unrelated_parent), Err(ReadError::Gone)] { + let read = read_tree_after_reuse(listing, after_11).expect("tree read"); + + assert_eq!(read.process_count, 2, "{listing:?} {after_11:?}"); + assert_eq!(read.cpu_nanos, 7 + 3, "{listing:?} {after_11:?}"); + } + } + } + + #[test] + fn tree_read_with_parent_unreadable_after_child_read_is_unavailable() { + for listing in LISTINGS { + let read = read_tree_after_reuse(listing, Err(ReadError::Unavailable("hidden"))); + + assert_eq!(read.unwrap_err(), "hidden", "{listing:?}"); + } + } + #[test] fn tree_read_with_unreadable_live_child_is_unavailable() { let with_child = |children, read| { From b2e92f20e748d9380740b6ca8096f8504aafc3be Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:34:02 -0700 Subject: [PATCH 020/159] fix: make a Linux snapshot unavailable when /proc hides a descendant With /proc mounted hidepid=invisible or hidepid=ptraceable, another user's process has no /proc entry, so the reader took a live descendant that changed user for an exited one and left it out of a successful snapshot. A missing /proc entry now counts as exited only when no process holds the PID, checked with signal 0; a PID that still names a process makes the snapshot unavailable. The kernel docs name kill -0 as the way to learn that a hidden PID exists. The fallback for a kernel without children files lists every process from /proc, where hidden processes are missing altogether. It now reads the /proc mount options and is unavailable under those two modes. With hidepid=noaccess, other users' processes stay listed and reading them fails, so the scan already reports the table as unreadable. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 121 +++++++++++++++++++- 1 file changed, 116 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index d362a979d..8548b799e 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -21,8 +21,9 @@ //! A live process in the tree that cannot be read, or a process table that //! the Linux fallback cannot read in full, makes the snapshot unavailable //! instead of leaving the process out. On Linux with `/proc` mounted -//! `hidepid=invisible`, another user's descendant is invisible and is not -//! counted. +//! `hidepid=invisible` or `hidepid=ptraceable`, a descendant that `/proc` +//! hides, such as one running as another user, makes the snapshot unavailable, +//! and the fallback for a kernel without children files is always unavailable. use std::time::SystemTime; @@ -302,6 +303,10 @@ mod platform { pub(super) const TABLE_UNREADABLE: &str = "the process table cannot be read in full"; pub(super) const TREE_CHANGING: &str = "the runtime's process tree kept changing during the read"; + pub(super) const PROCESS_HIDDEN: &str = + "/proc hides a process in the runtime's process tree (hidepid)"; + pub(super) const TABLE_HIDDEN: &str = + "/proc hides other users' processes (hidepid), so the process table cannot be read in full"; /// Times a process's children are read while waiting for two reads that agree. const MAX_CHILDREN_READS: usize = 4; @@ -373,7 +378,8 @@ mod platform { } fn stat(pid: u32) -> Result { - let text = std::fs::read_to_string(format!("/proc/{pid}/stat")).map_err(read_error)?; + let text = std::fs::read_to_string(format!("/proc/{pid}/stat")) + .map_err(|error| process_read_error(pid, error))?; parse_stat(&text).ok_or(ReadError::Unavailable(PROCESS_UNREADABLE)) } @@ -381,7 +387,9 @@ mod platform { /// exited during the read is skipped; its children move to another thread. fn children_once(pid: u32) -> Result, ReadError> { let mut children = Vec::new(); - for entry in std::fs::read_dir(format!("/proc/{pid}/task")).map_err(read_error)? { + let tasks = std::fs::read_dir(format!("/proc/{pid}/task")) + .map_err(|error| process_read_error(pid, error))?; + for entry in tasks { let tid = entry.map_err(read_error)?.file_name(); let path = format!("/proc/{pid}/task/{}/children", tid.to_string_lossy()); let text = match std::fs::read_to_string(path).map_err(read_error) { @@ -459,6 +467,52 @@ mod platform { } } + /// A process whose `/proc` entry is missing exited, unless its PID still + /// names a process: then `/proc` is mounted with `hidepid` and hides it. + pub(super) fn process_read_error(pid: u32, error: std::io::Error) -> ReadError { + match read_error(error) { + ReadError::Gone if pid_exists(pid) => ReadError::Unavailable(PROCESS_HIDDEN), + other => other, + } + } + + /// Whether a process holds `pid`. Signal 0 sends nothing: `ESRCH` means + /// no process has the PID, and `EPERM` means another user's process does. + fn pid_exists(pid: u32) -> bool { + // PID 0 would address this process group, so it is never checked. + let Some(pid) = libc::pid_t::try_from(pid).ok().filter(|pid| *pid > 0) else { + return false; + }; + // SAFETY: kill with signal 0 only checks the target; it sends nothing. + let status = unsafe { libc::kill(pid, 0) }; + status == 0 || std::io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) + } + + /// Whether the `/proc` mount in `mountinfo` leaves other users' processes + /// out of its listing: `hidepid=invisible` (2) or `hidepid=ptraceable` + /// (4). With `hidepid=noaccess` (1) they stay listed and reading them + /// fails, so the scan already reports the table as unreadable. The last + /// `/proc` mount listed is the one the path reaches. + pub(super) fn proc_hides_processes(mountinfo: &str) -> bool { + let proc_options = mountinfo.lines().rev().find_map(|line| { + // ` ... - ` + let (mount, filesystem) = line.split_once(" - ")?; + let mount_point = mount.split(' ').nth(4)?; + let mut filesystem = filesystem.split(' '); + let is_proc = filesystem.next()? == "proc"; + let options = filesystem.nth(1)?; + (mount_point == "/proc" && is_proc).then_some(options) + }); + proc_options.is_some_and(|options| { + options.split(',').any(|option| { + matches!( + option, + "hidepid=2" | "hidepid=invisible" | "hidepid=4" | "hidepid=ptraceable" + ) + }) + }) + } + pub(super) fn start_identity(pid: u32) -> Option { stat(pid).ok().map(|stat| stat.start) } @@ -476,8 +530,14 @@ mod platform { }) } - /// Children by parent PID, from one pass over every process. + /// Children by parent PID, from one pass over every process. A `/proc` + /// that leaves processes out of its listing makes the pass incomplete. pub(super) fn scan_children() -> Result { + let mountinfo = + std::fs::read_to_string("/proc/self/mountinfo").map_err(|_| TABLE_UNREADABLE)?; + if proc_hides_processes(&mountinfo) { + return Err(TABLE_HIDDEN); + } let by_parent = children_by_parent(all_pids()?, |pid| stat(pid).map(|stat| stat.ppid))?; Ok(ChildSource::Scanned(by_parent)) } @@ -1140,6 +1200,57 @@ mod tests { assert_eq!(incomplete.unwrap_err(), platform::TABLE_UNREADABLE); } + /// A missing `/proc` entry is a hidden process while its PID still names + /// a process, and an exited one otherwise. + #[cfg(target_os = "linux")] + #[test] + fn linux_missing_proc_entry_is_hidden_while_its_pid_is_live() { + let missing = || std::io::Error::from(std::io::ErrorKind::NotFound); + let mut reaped = std::process::Command::new("true") + .spawn() + .expect("spawn child"); + reaped.wait().expect("reap child"); + + assert_eq!( + platform::process_read_error(std::process::id(), missing()), + ReadError::Unavailable(platform::PROCESS_HIDDEN) + ); + assert_eq!( + platform::process_read_error(reaped.id(), missing()), + ReadError::Gone + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_proc_mount_that_hides_processes_is_detected() { + let proc_mount = |options: &str| { + format!("22 1 0:5 / /proc rw,nosuid,relatime shared:13 - proc proc {options}\n") + }; + let other_mount = "23 22 0:6 / /proc/sys/fs/binfmt_misc rw - binfmt_misc binfmt_misc rw\n"; + let hides = |mountinfo: &str| platform::proc_hides_processes(mountinfo); + + for options in [ + "rw,hidepid=2", + "rw,hidepid=invisible", + "rw,hidepid=4,gid=10", + "rw,hidepid=ptraceable", + ] { + assert!(hides(&(proc_mount(options) + other_mount)), "{options}"); + } + for options in [ + "rw", + "rw,hidepid=off", + "rw,hidepid=1", + "rw,hidepid=noaccess", + ] { + assert!(!hides(&(proc_mount(options) + other_mount)), "{options}"); + } + // A later mount on /proc covers the earlier one. + assert!(!hides(&(proc_mount("rw,hidepid=2") + &proc_mount("rw")))); + assert!(!hides("30 1 0:7 / /tmp/proc rw - proc proc rw,hidepid=2\n")); + } + /// The fallback for a kernel without children files reads the tree too. #[cfg(target_os = "linux")] #[test] From 5e39195c488344d9febe901c6f91206e56e8ba1f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:34:03 -0700 Subject: [PATCH 021/159] docs: state that a hidden descendant makes a snapshot unavailable The SDK guide said a descendant hidden by hidepid=invisible is not counted. The reader now makes the snapshot unavailable instead, and the fallback for a kernel without children lists is unavailable under that mount, so the guide states both, for hidepid=ptraceable too. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 13b5b4d38..f8edd255c 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -725,8 +725,11 @@ returning zeros or leaving a process out: environment provider, or operating system has no resource reader. The `http` and `native_plugin` adapter kinds and Windows raise this error. -On Linux with `/proc` mounted `hidepid=invisible`, a descendant owned by -another user is invisible and is not counted. +On Linux with `/proc` mounted `hidepid=invisible` or `hidepid=ptraceable`, the +snapshot is unavailable while `/proc` hides a descendant, such as one that runs +as another user. On a kernel without children lists, every snapshot is +unavailable under those mount options, because hidden processes are missing +from the process table that the snapshot reads to find the tree. Each `RunResult` from a local runtime also carries `resource_snapshot`, read once after the adapter returned the result. It is `None` when the read failed; From 2f76f8c39ca83c22563472b62a0f1b9089039cca Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:35:14 -0700 Subject: [PATCH 022/159] docs: regenerate the Rust reference for the hidepid module doc The resource_snapshot module doc now says a descendant hidden by hidepid makes a snapshot unavailable. Regenerated with Rust 1.94.0; only this page changes. Signed-off-by: mnajafian-nv --- .../nemo-fabric-core/resource-snapshot/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index e55480dcc..affbe0d0f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -16,7 +16,7 @@ A snapshot reads the process tree rooted at the adapter host process that NeMo F Linux reads `/proc//task//children` and `/proc//stat`. On a kernel without those children files, it reads every process's parent ID once per snapshot to find the tree. macOS reads `proc_listchildpids` and `proc_pid_rusage`. A snapshot exposes only its own fields: never command lines, environment variables, open files, or processes outside the tree. -A live process in the tree that cannot be read, or a process table that the Linux fallback cannot read in full, makes the snapshot unavailable instead of leaving the process out. On Linux with `/proc` mounted `hidepid=invisible`, another user's descendant is invisible and is not counted. +A live process in the tree that cannot be read, or a process table that the Linux fallback cannot read in full, makes the snapshot unavailable instead of leaving the process out. On Linux with `/proc` mounted `hidepid=invisible` or `hidepid=ptraceable`, a descendant that `/proc` hides, such as one running as another user, makes the snapshot unavailable, and the fallback for a kernel without children files is always unavailable. ## Structs From 6655b13eaff210549486a85ae897f146beba4900 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:54:49 -0700 Subject: [PATCH 023/159] test: cover run record write failures The recorder promises that a failed record write never replaces the invocation result, but only a failed create was tested. Add per-thread test hooks to the record writer that delay record creation or fail the start line, the end line or the sync, and use them to check that each failure keeps the result's status, output, error, usage and identity, adds one record_write_failed event, and leaves a raised invocation error unchanged. The hooks compile only in tests. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 143 ++++++++++++++++++++ crates/fabric-core/src/record/mod.rs | 2 + crates/fabric-core/src/record/writer.rs | 45 ++++++ 3 files changed, 190 insertions(+) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 7d8a2595a..276eba78b 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -298,6 +298,7 @@ fn end_artifacts(manifest: &ArtifactManifest) -> Vec { mod tests { use super::*; use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; + use crate::record::test_hooks::{self, Fault}; fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { let config: FabricConfig = serde_json::from_value(serde_json::json!({ @@ -474,4 +475,146 @@ mod tests { ); let _ = std::fs::remove_dir_all(folder); } + + fn failed_result_with_usage(invocation: &InvocationHandle) -> RunResult { + serde_json::from_value(serde_json::json!({ + "agent_name": "a", "harness": "h", "adapter_kind": "python", + "runtime_id": invocation.runtime_id, "invocation_id": invocation.invocation_id, + "request_id": invocation.request_id, "status": "failed", "output": "partial", + "error": { + "stage": "invoke", "code": "adapter_failure", + "message": "adapter gave up", "retryable": true, + }, + "usage": {"input_tokens": 3, "output_tokens": 5, "cost_usd": 0.25}, + })) + .expect("result") + } + + /// Start and finish one record with `fault` injected, and return the + /// finished outcome with the record file's lines. + fn finish_with_fault( + name: &str, + fault: Fault, + outcome: Result, + ) -> (Result, Vec) { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder(name); + let request = RunRequest::text("input"); + test_hooks::FAULT.with(|slot| slot.set(Some(fault))); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + let finished = recorder.finish(outcome); + test_hooks::FAULT.with(|slot| slot.set(None)); + let lines = std::fs::read_to_string(folder.join(RECORD_FILE_NAME)) + .expect("read record") + .lines() + .map(|line| serde_json::from_str(line).expect("record line")) + .collect(); + let _ = std::fs::remove_dir_all(folder); + (finished, lines) + } + + /// `finished` equals `original` apart from the artifacts and events that + /// recording adds, and carries one `record_write_failed` event. + fn assert_result_kept_with_failure_event(finished: &RunResult, original: &RunResult) { + let mut without_record = finished.clone(); + without_record.artifacts = original.artifacts.clone(); + without_record.events = original.events.clone(); + assert_eq!(&without_record, original); + let failures: Vec<_> = finished + .events + .iter() + .filter(|event| event.kind == RECORD_WRITE_FAILED) + .collect(); + assert_eq!(failures.len(), 1, "{:?}", finished.events); + assert_eq!( + failures[0].metadata["runtime_id"], + original.runtime_id.as_str() + ); + assert_eq!( + failures[0].metadata["invocation_id"], + original.invocation_id.as_str() + ); + } + + fn has_record_artifact(result: &RunResult) -> bool { + result + .artifacts + .artifacts + .iter() + .any(|artifact| artifact.kind == RECORD_ARTIFACT_KIND) + } + + #[test] + fn finish_after_a_failed_start_line_keeps_the_result_and_writes_nothing() { + let (_, _, invocation) = record_test_fixture(); + let original = failed_result_with_usage(&invocation); + + let (finished, lines) = + finish_with_fault("start-append", Fault::StartAppend, Ok(original.clone())); + + let finished = finished.expect("result"); + assert_result_kept_with_failure_event(&finished, &original); + assert!(!has_record_artifact(&finished)); + assert!(lines.is_empty(), "{lines:?}"); + } + + #[test] + fn finish_after_a_failed_end_line_keeps_the_result_and_the_start_line() { + let (_, _, invocation) = record_test_fixture(); + let original = failed_result_with_usage(&invocation); + + let (finished, lines) = + finish_with_fault("end-append", Fault::EndAppend, Ok(original.clone())); + + let finished = finished.expect("result"); + assert_result_kept_with_failure_event(&finished, &original); + assert!(has_record_artifact(&finished)); + let kinds: Vec<&Value> = lines.iter().map(|line| &line["kind"]).collect(); + assert_eq!(kinds, ["start"]); + } + + #[test] + fn finish_after_a_failed_sync_keeps_the_result_and_both_lines() { + let (_, _, invocation) = record_test_fixture(); + let original = failed_result_with_usage(&invocation); + + let (finished, lines) = finish_with_fault("sync", Fault::Sync, Ok(original.clone())); + + let finished = finished.expect("result"); + assert_result_kept_with_failure_event(&finished, &original); + assert!(has_record_artifact(&finished)); + let kinds: Vec<&Value> = lines.iter().map(|line| &line["kind"]).collect(); + assert_eq!(kinds, ["start", "end"]); + assert_eq!(lines[1]["usage"]["input_tokens"], 3); + } + + #[test] + fn finish_returns_the_invocation_error_when_recording_also_fails() { + for fault in [Fault::StartAppend, Fault::EndAppend, Fault::Sync] { + let error = FabricError::AdapterLifecycleOperation { + operation: "invoke", + runtime_id: "rt".to_string(), + code: "host_timeout".to_string(), + message: "adapter host did not answer".to_string(), + diagnostics: String::new(), + }; + let expected = error.to_string(); + + let (finished, _) = finish_with_fault("error", fault, Err(error)); + + let error = finished.expect_err("invocation error"); + assert_eq!(error.to_string(), expected, "{fault:?}"); + assert!( + matches!(&error, FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout"), + "{fault:?}: {error}" + ); + } + } } diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 32f867bb6..43a6019db 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -17,3 +17,5 @@ mod writer; pub(crate) use invocation::{InvocationRecorder, RecordStart}; pub(crate) use lines::RecordLine; +#[cfg(test)] +pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs index 1a849ac60..ea95049cf 100644 --- a/crates/fabric-core/src/record/writer.rs +++ b/crates/fabric-core/src/record/writer.rs @@ -27,6 +27,8 @@ impl RecordWriter { /// Create the record at `path`. `create_new` refuses an existing file or /// symbolic link at `path`, and on Unix the file is mode 0600. pub(crate) fn create(path: &Path) -> io::Result { + #[cfg(test)] + std::thread::sleep(test_hooks::CREATE_DELAY.with(std::cell::Cell::get)); let mut options = OpenOptions::new(); options.append(true).create_new(true); #[cfg(unix)] @@ -41,6 +43,8 @@ impl RecordWriter { /// Append `line` and its newline with one write. pub(crate) fn append(&mut self, line: &RecordLine) -> io::Result<()> { + #[cfg(test)] + test_hooks::fail_append(&line.body)?; let mut bytes = serde_json::to_vec(line).map_err(io::Error::other)?; bytes.push(b'\n'); self.file.write_all(&bytes) @@ -48,10 +52,51 @@ impl RecordWriter { /// Flush the file's data and metadata to stable storage. pub(crate) fn sync(&self) -> io::Result<()> { + #[cfg(test)] + test_hooks::fail_if(test_hooks::Fault::Sync)?; self.file.sync_all() } } +/// Per-thread hooks that make record I/O slow or fail in tests. +#[cfg(test)] +pub(crate) mod test_hooks { + use std::cell::Cell; + use std::io; + use std::time::Duration; + + use super::super::lines::LineBody; + + /// A record operation that a test makes fail. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum Fault { + StartAppend, + EndAppend, + Sync, + } + + thread_local! { + /// Delay added before each record file is created on this thread. + pub(crate) static CREATE_DELAY: Cell = const { Cell::new(Duration::ZERO) }; + /// Operation that fails on this thread. + pub(crate) static FAULT: Cell> = const { Cell::new(None) }; + } + + pub(super) fn fail_append(body: &LineBody) -> io::Result<()> { + match body { + LineBody::Start(_) => fail_if(Fault::StartAppend), + LineBody::End(_) => fail_if(Fault::EndAppend), + } + } + + pub(super) fn fail_if(fault: Fault) -> io::Result<()> { + if FAULT.with(Cell::get) == Some(fault) { + return Err(io::Error::other(format!("injected {fault:?} failure"))); + } + Ok(()) + } +} + #[cfg(test)] mod tests { use super::*; From f43809ed73de7c5e3b7608588e3aa5e7ef6dcc0f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:55:03 -0700 Subject: [PATCH 024/159] fix: start the run record before the adapter deadline Core set the adapter's deadline and then created the record and wrote its start line, while the host timeout starts only when the request is sent. Slow record storage therefore shortened the adapter's budget without shortening the host timeout, so recording could turn a successful invocation into a timeout. Create the record and write the start line before core sets the deadline and writes adapter-invocation.json. The start line records the timeout, not the deadline, so the deadline the adapter receives and the one in adapter-invocation.json stay the same value. A failed adapter-invocation.json write now happens after the record starts, so its end line reads artifact_write_failed. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 5 +- crates/fabric-core/src/runtime.rs | 53 ++++++++++++++++----- docs/sdk/python.mdx | 4 +- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 276eba78b..a38538da0 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -46,9 +46,10 @@ const RECORDED_CONTEXT_KEYS: [&str; 7] = [ "harbor_context_id", "harbor_trial_name", ]; -/// End-line error code for an artifact write that failed after the adapter returned. +/// End-line error code for a failed Fabric file write: `adapter-invocation.json` +/// before the request, or a result artifact after the adapter returned. const ARTIFACT_WRITE_FAILED: &str = "artifact_write_failed"; -/// End-line error code for any other Fabric error after the request was sent. +/// End-line error code for any other Fabric error after the record started. const FABRIC_ERROR: &str = "fabric_error"; /// What the start line needs from core's invocation path. diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index fce59395e..51e21dd5f 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1752,8 +1752,9 @@ fn run_local_host_invocation_with_timeout( } /// Run one invocation on the local host. When the plan turns the run record -/// on, `recorder` is set right before core sends the request to the host, so -/// every invocation that core tried to send has a record. +/// on, `recorder` is set before core sets the adapter deadline and writes +/// `adapter-invocation.json`, so every invocation that core tried to send has +/// a record. fn local_host_invocation_result( plan: &RunPlan, runtime: &RuntimeHandle, @@ -1802,6 +1803,18 @@ fn local_host_invocation_result( &host_guard.environment_env, &adapter_invocation.runtime_context.environment.env, )?; + // Record I/O runs before the adapter deadline is set, so slow record + // storage delays the request instead of shortening the adapter's budget. + if plan.config.runtime.record_enabled() && artifacts.root.is_some() { + *recorder = Some(InvocationRecorder::start(RecordStart { + plan, + runtime, + invocation: &invocation, + request: &request, + fabric_home: &fabric_home, + timeout: invoke_timeout, + })); + } adapter_invocation.runtime_context.deadline_millis = adapter_deadline_millis(invoke_timeout); let (lifecycle_request, adapter_payload) = match invocation_kind { @@ -1836,16 +1849,6 @@ fn local_host_invocation_result( } }; let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; - if plan.config.runtime.record_enabled() && artifacts.root.is_some() { - *recorder = Some(InvocationRecorder::start(RecordStart { - plan, - runtime, - invocation: &invocation, - request: &request, - fabric_home: &fabric_home, - timeout: invoke_timeout, - })); - } match exchange_lifecycle_message( &mut host_guard, &runtime.runtime_id, @@ -5726,6 +5729,32 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn record_preparation_does_not_shorten_the_adapter_deadline() { + let (root, mut plan) = record_plan("success"); + plan.config.runtime.timeout_seconds = Some(60.0); + let runtime = start_runtime(&plan).expect("start local host"); + let delay = Duration::from_millis(500); + crate::record::test_hooks::CREATE_DELAY.with(|slot| slot.set(delay)); + + let before = now_millis(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("slow record storage")); + let after = now_millis(); + crate::record::test_hooks::CREATE_DELAY.with(|slot| slot.set(Duration::ZERO)); + + let result = result.expect("invoke"); + let deadline = u128::from(result.output["deadline_millis"].as_u64().expect("deadline")); + // The adapter's 50-second budget starts after the record is created, + // so slow record storage does not eat into it. + assert!( + (before + delay.as_millis() + 50_000..=after + 50_000).contains(&deadline), + "deadline {deadline}, before {before}, after {after}" + ); + assert_eq!(record_files(&root).len(), 1); + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + #[test] fn record_warm_runtime_writes_one_record_per_invocation() { let (root, plan) = record_plan("success"); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 555e6ba5c..6ee3008a4 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -988,8 +988,8 @@ The file holds a start line and an end line. Every line carries `schema` adapter host error keeps its code, such as `host_timeout`, `host_crashed`, `host_io`, or `invalid_agent_run_result`. A result extension that the adapter's descriptor does not declare also reads `invalid_agent_run_result`. - A failed artifact write reads `artifact_write_failed`, and any other error - reads `fabric_error`. + A failed write of `adapter-invocation.json` or a result artifact reads + `artifact_write_failed`, and any other error reads `fabric_error`. The end line describes the invocation. A one-shot `Fabric.run` stops the runtime after the invocation; when that stop fails, the result reads `failed` From 100394a1fd0d8fd2e42b1716f83c8334fc24d1be Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:55:15 -0700 Subject: [PATCH 025/159] fix: keep run record model role and context key names whole The start line cut model role names and request context key names to 256 characters, and both are map keys. Two names that shared their first 255 characters became the same key, so one entry silently replaced the other. Config validation does not bound either name, so keep both names whole and cut only values. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 38 +++++++++++++++++++-- crates/fabric-core/src/record/lines.rs | 3 +- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index a38538da0..645a931cd 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -180,7 +180,7 @@ fn start_line(start: &RecordStart<'_>) -> StartLine { .iter() .map(|(role, model)| { ( - cut_text(role), + role.clone(), ModelName { provider: cut_text(&model.provider), model: cut_text(&model.model), @@ -194,7 +194,7 @@ fn start_line(start: &RecordStart<'_>) -> StartLine { .request .context .iter() - .map(|(key, value)| (cut_text(key), context_value(key, value))) + .map(|(key, value)| (key.clone(), context_value(key, value))) .collect(), override_keys: override_keys(start.request.overrides.as_ref()), } @@ -396,6 +396,40 @@ mod tests { ); } + #[test] + fn start_line_keeps_long_role_and_context_key_names_apart() { + let (mut plan, runtime, invocation) = record_test_fixture(); + let shared = "k".repeat(300); + let model = plan.config.models["default"].clone(); + for suffix in ["a", "b"] { + plan.config + .models + .insert(format!("{shared}{suffix}"), model.clone()); + } + let mut request = RunRequest::text("input"); + request.context = BTreeMap::from([ + (format!("{shared}a"), Value::from(1)), + (format!("{shared}b"), Value::from(2)), + ]); + let start = RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: Path::new("/unused"), + timeout: Duration::from_secs(5), + }; + + let line = start_line(&start); + + assert_eq!(line.models.len(), 3); + assert_eq!(line.context.len(), 2); + for suffix in ["a", "b"] { + assert!(line.models.contains_key(&format!("{shared}{suffix}"))); + assert!(line.context.contains_key(&format!("{shared}{suffix}"))); + } + } + fn temp_folder(name: &str) -> PathBuf { let folder = std::env::temp_dir().join(format!("fabric-recorder-{name}-{}", std::process::id())); diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index ebe9df282..964a79eda 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -15,7 +15,8 @@ use serde::Serialize; use crate::runtime::{ErrorStage, RunStatus}; -/// Longest string a line keeps from config or the request, in characters. +/// Longest value a line keeps from config or the request, in characters. +/// Model role and context key names are kept whole, so map keys stay distinct. const MAX_VALUE_CHARS: usize = 256; /// Longest adapter error code a line keeps. const MAX_CODE_CHARS: usize = 128; From 264e34ba469ebd05951045ab2d854bb0759cc780 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:55:23 -0700 Subject: [PATCH 026/159] docs: describe run recording as synchronous and best-effort The record docs said recording never changes the run and that the file is never created through a symbolic link. Both overstated the contract: record I/O runs on the invocation path, so slow storage delays the request and the return, and create_new refuses only a file or symbolic link at the record path itself. State that record I/O errors do not replace the invocation result, that recording is synchronous and best-effort with no time limit, that the record is created before the adapter deadline is set, and that creation refuses an existing file or symbolic link at run-record.jsonl. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 7 ++++--- crates/fabric-core/src/record/mod.rs | 6 ++++-- docs/sdk/python.mdx | 18 +++++++++++++----- .../references/results-and-errors.md | 5 ++++- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 645a931cd..45451bd2c 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -4,9 +4,10 @@ //! One invocation's record: the start line before the request, the end line //! after the invocation. //! -//! Recording never changes the run. A failed create, write or sync stops the -//! record for that invocation and adds a `record_write_failed` event to the -//! result; nothing here returns an error to the caller. +//! Record I/O errors do not replace the invocation result. A failed create, +//! write or sync stops the record for that invocation and adds a +//! `record_write_failed` event to the result; nothing here returns a record +//! error to the caller. use std::collections::BTreeMap; use std::io; diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 43a6019db..4b4b2deee 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -5,8 +5,10 @@ //! //! When `runtime.record.enabled` is set, core appends one start line before it //! sends the request to the adapter host and one end line after the invocation -//! returns or fails. Recording never changes the run: a failed write stops the -//! record and becomes a `record_write_failed` event on the result. +//! returns or fails. Recording is synchronous and best-effort: record I/O +//! errors do not replace the invocation result. A failed write stops the +//! record and becomes a `record_write_failed` event on the result, and slow +//! storage delays the invocation. //! //! Everything here is crate-private. The line format is public only through //! the `run-record-line` JSON Schema, version `v1alpha1`. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 6ee3008a4..3fa5363a9 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1004,11 +1004,19 @@ not covered by this rule. Each line is written with one append, so a killed process keeps every line whose write returned. NeMo Fabric syncs the file once, after the end line. A record without an end line belongs to an invocation that is still running or -that never finished; treat its outcome as unknown, not as failed. If a write -fails, the record stops, the result gains a `record_write_failed` event, and -the invocation continues unchanged. A record whose end line failed to write -therefore has no end line, and that event says why. The file is created with -mode `0600` on Linux and macOS, and never through a symbolic link. +that never finished; treat its outcome as unknown, not as failed. + +Recording is synchronous and best-effort. If a write or the sync fails, the +record stops. Record I/O errors do not replace the invocation result: a +returned result keeps its status, output, error, and usage and gains a +`record_write_failed` event, and a raised error stays the same error. A record +whose end line failed to write therefore has no end line, and that event says +why. NeMo Fabric creates the record and writes the start line before it sets +the adapter's deadline, so slow storage delays the request without shortening +that deadline. The end line and the sync run before the result returns, so +slow storage delays the return too. Recording sets no time limit on either +delay. The file is created with mode `0600` on Linux and macOS. Creation +refuses an existing file or symbolic link at `run-record.jsonl`. The line format is published as `schemas/sdk/run-record-line.schema.json` at version `v1alpha1`, which may change until a reader for run records ships. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index ebfeb1896..1c06367a3 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -57,7 +57,10 @@ result lists the file as the artifact with kind `run_record`. The file has a `start` line, written before the request, and an `end` line with status, elapsed time, error code, numeric usage, and artifact names. It holds no prompts, output, or error messages. A record without an end line has an -unknown outcome; do not count it as failed. The format is the `v1alpha1` +unknown outcome; do not count it as failed. Recording is synchronous and +best-effort: a failed record write adds a `record_write_failed` event and does +not replace the result, and slow storage delays the invocation. The format is +the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) and may change until a reader ships. From 73ebc0a8b5efdf8c1551f5cb9f14e1e18e07596b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 00:15:35 -0700 Subject: [PATCH 027/159] feat: record why each invocation stopped A run record's end line keeps the error code but no cause. A reader has to know core's and each adapter's codes to separate infrastructure failures from the agent's own, and an out-of-memory kill leaves no trace. The end line gains `stop`: a cause from Fabric's own closed list (fabric_failed, sandbox_failed, agent_exhausted_budget, unclassified, agent_failed), `reason: timeout` with the invoke timeout that applied, the time used and whether Fabric force-stopped the host, and the host's exit code or signal when it exited. Following the adapter contract, only a failed result reads agent_failed; every raised error other than a timeout or a host exit reads fabric_failed, except that an error from the exchange with the host during an out-of-memory rise reads sandbox_failed, because a kill can make the adapter raise. Both lines record the container's cgroup v2 oom_kill count when Fabric owns its cgroup. A rise sets a cause only with runtime.record.exclusive_container, which the start line records; set it only when nothing else runs in the container during an invocation, including other runtimes in the same process. The host's exit status is read only when a record is on. An ignored Linux test drives a real out-of-memory kill of the adapter host; it runs only in a memory-limited cgroup v2 container. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 26 +- crates/fabric-core/src/record/cgroup.rs | 151 +++++++++ crates/fabric-core/src/record/invocation.rs | 295 +++++++++++++++++- crates/fabric-core/src/record/lines.rs | 113 +++++++ crates/fabric-core/src/record/mod.rs | 5 +- crates/fabric-core/src/record/stop.rs | 155 +++++++++ crates/fabric-core/src/runtime.rs | 241 +++++++++++++- .../nemo_fabric.models.md | 1 + .../config/struct-recordconfig.mdx | 6 +- docs/sdk/python.mdx | 56 ++++ schemas/sdk/agent.schema.json | 4 + schemas/sdk/run-plan.schema.json | 4 + schemas/sdk/run-record-line.schema.json | 213 ++++++++++++- .../src/nemo_fabric/models.py | 9 + .../src/nemo_fabric/types.py | 14 +- .../references/results-and-errors.md | 6 +- tests/python/test_sdk_contract.py | 18 ++ 17 files changed, 1279 insertions(+), 38 deletions(-) create mode 100644 crates/fabric-core/src/record/cgroup.rs create mode 100644 crates/fabric-core/src/record/stop.rs diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 1cb2d7dcf..5cf456ae5 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -1083,6 +1083,12 @@ pub struct RecordConfig { /// artifact root, the `local` environment provider, and a `process` or /// `python` adapter. pub enabled: bool, + /// Whether nothing else runs in the container during an invocation, + /// including other runtimes in the same process, so a rise in the + /// container's out-of-memory kill count reads `sandbox_failed` in the + /// record. Unset means false. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub exclusive_container: bool, } fn default_input_schema() -> String { @@ -7425,7 +7431,10 @@ mod tests { #[test] fn record_enabled_without_an_artifact_root_fails_validation() { let mut config = typed_config("nvidia.fabric.hermes"); - config.runtime.record = Some(RecordConfig { enabled: true }); + config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container: false, + }); assert_eq!( record_error(&config), @@ -7445,7 +7454,10 @@ mod tests { fn record_enabled_with_a_non_local_provider_fails_validation() { let mut config = typed_config("nvidia.fabric.hermes"); config.runtime.artifacts = Some(PathBuf::from("runs")); - config.runtime.record = Some(RecordConfig { enabled: true }); + config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container: false, + }); config.environment.as_mut().expect("environment").provider = "docker".to_string(); assert_eq!( @@ -7463,7 +7475,10 @@ mod tests { let mut descriptor = load_adapter_descriptor(&path).expect("Claude descriptor"); let mut config = typed_config("nvidia.fabric.claude"); config.runtime.artifacts = Some(PathBuf::from("runs")); - config.runtime.record = Some(RecordConfig { enabled: true }); + config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container: false, + }); assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); for kind in [AdapterKind::Http, AdapterKind::NativePlugin] { @@ -7483,7 +7498,10 @@ mod tests { fn record_disabled_or_absent_needs_no_artifact_root() { let mut config = typed_config("nvidia.fabric.hermes"); assert_eq!(record_error(&config), None); - config.runtime.record = Some(RecordConfig { enabled: false }); + config.runtime.record = Some(RecordConfig { + enabled: false, + exclusive_container: false, + }); assert_eq!(record_error(&config), None); } diff --git a/crates/fabric-core/src/record/cgroup.rs b/crates/fabric-core/src/record/cgroup.rs new file mode 100644 index 000000000..d3931c32e --- /dev/null +++ b/crates/fabric-core/src/record/cgroup.rs @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The container's out-of-memory kill count: cgroup v2 `memory.events` +//! `oom_kill`, read at the start and end lines of an invocation. +//! +//! The count describes Fabric's container only when Fabric owns its cgroup: +//! `/proc/self/cgroup` has the single v2 line `0::/`, and the cgroup's +//! `memory.events` reads with an `oom_kill` line. The kernel creates +//! `memory.events` only in a non-root cgroup with the memory controller, so a +//! true root cgroup and rootless containers without memory delegation fail +//! that check. Every other case reads unavailable with the failed check's +//! reason. + +use std::path::{Path, PathBuf}; + +use super::lines::{OomKills, OomKillsUnavailable}; + +/// Reads cgroup v2 files under a root, `/` on a real system. +#[derive(Debug, Clone)] +pub(crate) struct CgroupReader { + root: PathBuf, +} + +impl CgroupReader { + /// The running system's `/proc/self/cgroup` and `/sys/fs/cgroup`. + pub(crate) fn system() -> Self { + Self { + root: PathBuf::from("/"), + } + } + + /// A fake cgroupfs: `/proc/self/cgroup` and `/sys/fs/cgroup`. + #[cfg(test)] + pub(crate) fn with_root(root: impl Into) -> Self { + Self { root: root.into() } + } + + /// The owned cgroup's `oom_kill` count, or why it is unavailable. + pub(crate) fn oom_kills(&self) -> OomKills { + match self.read_oom_kills() { + Ok(count) => OomKills::Count(count), + Err(reason) => OomKills::Unavailable { reason }, + } + } + + fn read_oom_kills(&self) -> Result { + if !cfg!(target_os = "linux") && self.root == Path::new("/") { + return Err(OomKillsUnavailable::Platform); + } + let membership = std::fs::read_to_string(self.root.join("proc/self/cgroup")) + .map_err(|_| OomKillsUnavailable::NoOwnedCgroup)?; + let lines: Vec<&str> = membership.lines().filter(|line| !line.is_empty()).collect(); + let [line] = lines.as_slice() else { + return Err(OomKillsUnavailable::CgroupV1); + }; + let Some(path) = line.strip_prefix("0::") else { + return Err(OomKillsUnavailable::CgroupV1); + }; + if path != "/" { + return Err(OomKillsUnavailable::NoOwnedCgroup); + } + let events = std::fs::read_to_string(self.root.join("sys/fs/cgroup/memory.events")) + .map_err(|_| OomKillsUnavailable::NoOwnedCgroup)?; + events + .lines() + .find_map(|line| line.strip_prefix("oom_kill ")) + .and_then(|count| count.trim().parse().ok()) + .ok_or(OomKillsUnavailable::NoOwnedCgroup) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + /// A fake cgroupfs root with `membership` in `/proc/self/cgroup` and, when + /// given, the cgroup's `memory.events`. + pub(crate) fn fake_root(name: &str, membership: &str, memory_events: Option<&str>) -> PathBuf { + let root = + std::env::temp_dir().join(format!("fabric-cgroup-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("proc/self")).expect("proc"); + std::fs::create_dir_all(root.join("sys/fs/cgroup")).expect("cgroup"); + std::fs::write(root.join("proc/self/cgroup"), membership).expect("membership"); + if let Some(events) = memory_events { + std::fs::write(root.join("sys/fs/cgroup/memory.events"), events).expect("events"); + } + root + } + + /// `memory.events` of a cgroup whose `oom_kill` count is `count`. + pub(crate) fn memory_events(count: u64) -> String { + format!("low 0\nhigh 0\nmax 2\noom 1\noom_kill {count}\noom_group_kill 0\n") + } + + fn unavailable(reason: OomKillsUnavailable) -> OomKills { + OomKills::Unavailable { reason } + } + + #[test] + fn oom_kills_owned_cgroup_v2_reads_memory_events() { + let root = fake_root("owned", "0::/\n", Some(&memory_events(1))); + + assert_eq!( + CgroupReader::with_root(&root).oom_kills(), + OomKills::Count(1) + ); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn oom_kills_cgroup_v1_reads_cgroup_v1() { + let root = fake_root( + "v1", + "12:memory:/docker/abc\n11:cpu:/docker/abc\n", + Some(&memory_events(1)), + ); + + assert_eq!( + CgroupReader::with_root(&root).oom_kills(), + unavailable(OomKillsUnavailable::CgroupV1) + ); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn oom_kills_named_cgroup_missing_events_or_no_oom_kill_line_reads_no_owned_cgroup() { + let named = fake_root( + "named", + "0::/user.slice/session-2.scope\n", + Some(&memory_events(1)), + ); + // Rootless containers without memory delegation, and a true root + // cgroup, have no `memory.events`. + let no_events = fake_root("no-events", "0::/\n", None); + let no_oom_kill = fake_root("no-oom-kill", "0::/\n", Some("low 0\nhigh 0\n")); + + for root in [&named, &no_events, &no_oom_kill] { + assert_eq!( + CgroupReader::with_root(root).oom_kills(), + unavailable(OomKillsUnavailable::NoOwnedCgroup), + "{}", + root.display() + ); + } + for root in [named, no_events, no_oom_kill] { + let _ = std::fs::remove_dir_all(root); + } + } +} diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 45451bd2c..2c18fa180 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -16,17 +16,20 @@ use std::time::{Duration, Instant, SystemTime}; use serde_json::Value; +use super::cgroup::CgroupReader; use super::lines::{ - ContextValue, EndArtifact, EndError, EndLine, EndUsage, LineBody, ModelName, NotRecorded, - RecordLine, RecordSchema, RecordVersion, StartLine, cut_text, recorded_code, + ContextValue, EndArtifact, EndError, EndLine, EndUsage, LimitType, LineBody, ModelName, + NotRecorded, OomKills, RecordLine, RecordSchema, RecordVersion, StartLine, Stop, StopLimit, + StopReason, cut_text, recorded_code, }; +use super::stop::{StopFacts, stop_cause}; use super::writer::{RECORD_FILE_NAME, RecordWriter}; use crate::config::RunPlan; use crate::error::{FabricError, Result}; use crate::runtime::{ - ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, INVALID_AGENT_RUN_RESULT, - InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, RuntimeHandle, - event_with_metadata, unique_artifact_name, + ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, HOST_CRASHED, HOST_TIMEOUT, + INVALID_AGENT_RUN_RESULT, InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, + RuntimeHandle, event_with_metadata, unique_artifact_name, }; use crate::time::rfc3339_millis; @@ -52,6 +55,9 @@ const RECORDED_CONTEXT_KEYS: [&str; 7] = [ const ARTIFACT_WRITE_FAILED: &str = "artifact_write_failed"; /// End-line error code for any other Fabric error after the record started. const FABRIC_ERROR: &str = "fabric_error"; +/// Error code an adapter reports when it stops at the deadline core gave it, +/// whether in a failed result or as a lifecycle error. +const ADAPTER_TIMEOUT_CODE: &str = "timeout"; /// What the start line needs from core's invocation path. pub(crate) struct RecordStart<'a> { @@ -63,6 +69,31 @@ pub(crate) struct RecordStart<'a> { pub fabric_home: &'a Path, /// The host timeout that applies to this invocation. pub timeout: Duration, + /// Reads the container's out-of-memory kill count. + pub cgroup: CgroupReader, +} + +/// The adapter host's wait status, when it exited. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct HostExit { + /// Exit code. + pub exit_code: Option, + /// Signal that ended the host (Unix only). + pub exit_signal: Option, +} + +impl HostExit { + /// Read a wait status. + pub(crate) fn from_status(status: std::process::ExitStatus) -> Self { + #[cfg(unix)] + let exit_signal = std::os::unix::process::ExitStatusExt::signal(&status); + #[cfg(not(unix))] + let exit_signal = None; + Self { + exit_code: status.code(), + exit_signal, + } + } } /// The record of one running invocation. @@ -76,12 +107,27 @@ pub(crate) struct InvocationRecorder { writer: Option, wrote_start: bool, failure: Option, + timeout: Duration, + cgroup: CgroupReader, + oom_kills_at_start: OomKills, + exclusive_container: bool, + failed_exchange: Option, +} + +/// What core saw when the exchange with the adapter host failed. +#[derive(Debug, Clone, Copy)] +struct FailedExchange { + /// Time from the start line to the failure, before core's own cleanup. + elapsed_ms: u64, + /// The host's wait status, when it had exited. + host_exit: Option, } impl InvocationRecorder { /// Create the record and write its start line. pub(crate) fn start(start: RecordStart<'_>) -> Self { let path = start.fabric_home.join(RECORD_FILE_NAME); + let oom_kills_at_start = start.cgroup.oom_kills(); let mut recorder = Self { runtime_id: start.invocation.runtime_id.clone(), invocation_id: start.invocation.invocation_id.clone(), @@ -91,11 +137,16 @@ impl InvocationRecorder { wrote_start: false, failure: None, path, + timeout: start.timeout, + cgroup: start.cgroup.clone(), + oom_kills_at_start, + exclusive_container: exclusive_container(start.plan), + failed_exchange: None, }; match RecordWriter::create(&recorder.path) { Ok(writer) => { recorder.writer = Some(writer); - recorder.append(LineBody::Start(start_line(&start))); + recorder.append(LineBody::Start(start_line(&start, oom_kills_at_start))); recorder.wrote_start = recorder.failure.is_none(); } Err(error) => recorder.failure = Some(error), @@ -103,14 +154,26 @@ impl InvocationRecorder { recorder } + /// Keep when the exchange with the adapter host failed and the host's + /// wait status, for the end line's `stop`. Core calls this before its + /// cleanup, such as stopping a timed-out host, so `failed_at` excludes it. + pub(crate) fn exchange_failed(&mut self, failed_at: Instant, host_exit: Option) { + self.failed_exchange = Some(FailedExchange { + elapsed_ms: millis(failed_at.saturating_duration_since(self.started)), + host_exit, + }); + } + /// Write and sync the end line for `outcome`, then return it. A result /// gains the `record` artifact when the start line was written, and a /// `record_write_failed` event when any write or the sync failed. pub(crate) fn finish(mut self, outcome: Result) -> Result { - let elapsed_ms = u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX); + let elapsed_ms = millis(self.started.elapsed()); + let oom_kills = self.cgroup.oom_kills(); + let stop = self.stop(&outcome, elapsed_ms, oom_kills); let end = match &outcome { - Ok(result) => result_end_line(result, elapsed_ms), - Err(error) => error_end_line(error, elapsed_ms), + Ok(result) => result_end_line(result, elapsed_ms, oom_kills, stop), + Err(error) => error_end_line(error, elapsed_ms, oom_kills, stop), }; self.append(LineBody::End(end)); if let Some(writer) = &self.writer @@ -148,6 +211,81 @@ impl InvocationRecorder { Ok(result) } + /// The end line's `stop`, or `None` when it holds nothing (a success). + fn stop( + &self, + outcome: &Result, + elapsed_ms: u64, + oom_kills_at_end: OomKills, + ) -> Option { + let oom_rose = self.exclusive_container + && matches!( + (self.oom_kills_at_start, oom_kills_at_end), + (OomKills::Count(start), OomKills::Count(end)) if end > start + ); + // `forced` is set exactly when a timeout stopped the invocation. + let (facts, forced) = match outcome { + Ok(result) => { + let failed = result.status == RunStatus::Failed; + let adapter_timeout = failed + && result + .error + .as_ref() + .is_some_and(|error| error.code == ADAPTER_TIMEOUT_CODE); + let facts = StopFacts { + status: result.status, + fabric_failed: false, + oom_rose, + timed_out: adapter_timeout, + host_exited: false, + agent_error: failed, + }; + (facts, adapter_timeout.then_some(false)) + } + // A raised error is never the agent's own failure: the adapter + // contract returns a target failure as a `failed` result. An + // out-of-memory rise outranks an error from the exchange itself, + // such as an adapter that raised after its harness was killed; + // core's own errors after a complete response keep rank 1. + Err(error) => { + let code = match error { + FabricError::AdapterLifecycleOperation { code, .. } => Some(code.as_str()), + _ => None, + }; + let forced = code == Some(HOST_TIMEOUT); + let timed_out = forced || code == Some(ADAPTER_TIMEOUT_CODE); + let host_exited = code == Some(HOST_CRASHED); + let exchange_error = self.failed_exchange.is_some(); + let facts = StopFacts { + status: RunStatus::Failed, + fabric_failed: !(timed_out || host_exited || (exchange_error && oom_rose)), + oom_rose, + timed_out, + host_exited, + agent_error: false, + }; + (facts, timed_out.then_some(forced)) + } + }; + let usage = self + .failed_exchange + .map_or(elapsed_ms, |failed| failed.elapsed_ms); + let host_exit = self.failed_exchange.and_then(|failed| failed.host_exit); + let stop = Stop { + cause: stop_cause(facts), + reason: forced.map(|_| StopReason::Timeout), + forced, + limit: forced.map(|_| StopLimit { + limit_type: LimitType::Time, + value: millis(self.timeout), + usage, + }), + exit_code: host_exit.and_then(|exit| exit.exit_code), + exit_signal: host_exit.and_then(|exit| exit.exit_signal), + }; + (stop != Stop::default()).then_some(stop) + } + /// Append one line. After the first failure the record stops. fn append(&mut self, body: LineBody) { let Some(writer) = self.writer.as_mut() else { @@ -169,7 +307,7 @@ impl InvocationRecorder { } } -fn start_line(start: &RecordStart<'_>) -> StartLine { +fn start_line(start: &RecordStart<'_>, container_oom_kills: OomKills) -> StartLine { StartLine { agent_name: cut_text(&start.runtime.agent_name), harness: cut_text(&start.runtime.harness), @@ -190,7 +328,7 @@ fn start_line(start: &RecordStart<'_>) -> StartLine { }) .collect(), fabric_version: crate::version().to_string(), - timeout_ms: u64::try_from(start.timeout.as_millis()).unwrap_or(u64::MAX), + timeout_ms: millis(start.timeout), context: start .request .context @@ -198,9 +336,23 @@ fn start_line(start: &RecordStart<'_>) -> StartLine { .map(|(key, value)| (key.clone(), context_value(key, value))) .collect(), override_keys: override_keys(start.request.overrides.as_ref()), + exclusive_container: exclusive_container(start.plan), + container_oom_kills, } } +fn exclusive_container(plan: &RunPlan) -> bool { + plan.config + .runtime + .record + .as_ref() + .is_some_and(|record| record.exclusive_container) +} + +fn millis(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + /// The sorted top-level key names of `overrides`; empty unless it is an object. fn override_keys(overrides: Option<&Value>) -> Vec { let Some(Value::Object(overrides)) = overrides else { @@ -228,17 +380,29 @@ fn context_value(key: &str, value: &Value) -> ContextValue { } } -fn result_end_line(result: &RunResult, elapsed_ms: u64) -> EndLine { +fn result_end_line( + result: &RunResult, + elapsed_ms: u64, + container_oom_kills: OomKills, + stop: Option, +) -> EndLine { EndLine { status: result.status, elapsed_ms, error: result.error.as_ref().map(result_error), usage: result.usage.as_ref().map(end_usage), artifacts: end_artifacts(&result.artifacts), + container_oom_kills, + stop, } } -fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { +fn error_end_line( + error: &FabricError, + elapsed_ms: u64, + container_oom_kills: OomKills, + stop: Option, +) -> EndLine { let (code, stage) = match error { FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), // After the record starts, only the check of the adapter's result @@ -257,6 +421,8 @@ fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { }), usage: None, artifacts: Vec::new(), + container_oom_kills, + stop, } } @@ -351,6 +517,7 @@ mod tests { request: &request, fabric_home: Path::new("/unused"), timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }; let line = serde_json::to_value(RecordLine { @@ -360,7 +527,7 @@ mod tests { invocation_id: "inv".to_string(), request_id: "req".to_string(), time: "2026-10-14T18:01:10.205Z".to_string(), - body: LineBody::Start(start_line(&start)), + body: LineBody::Start(start_line(&start, OomKills::Count(0))), }) .expect("start line"); @@ -419,9 +586,10 @@ mod tests { request: &request, fabric_home: Path::new("/unused"), timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }; - let line = start_line(&start); + let line = start_line(&start, OomKills::Count(0)); assert_eq!(line.models.len(), 3); assert_eq!(line.context.len(), 2); @@ -460,6 +628,7 @@ mod tests { request: &request, fabric_home: &folder, timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }); let mut result = succeeded_result(&invocation); result.artifacts.artifacts.push(ArtifactRef { @@ -496,6 +665,7 @@ mod tests { request: &request, fabric_home: &folder, timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }); let result = succeeded_result(&invocation); @@ -544,6 +714,7 @@ mod tests { request: &request, fabric_home: &folder, timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), }); let finished = recorder.finish(outcome); test_hooks::FAULT.with(|slot| slot.set(None)); @@ -653,4 +824,98 @@ mod tests { ); } } + + /// Run one invocation whose exchange with the host fails with `code` after + /// the host died of SIGKILL, while a fake cgroup's `oom_kill` count rises + /// from 4 to 5, and return its start and end lines. + fn oom_kill_record(name: &str, exclusive_container: bool, code: &str) -> (Value, Value) { + use crate::config::RecordConfig; + use crate::record::cgroup::tests::{fake_root, memory_events}; + let (mut plan, runtime, invocation) = record_test_fixture(); + plan.config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container, + }); + let root = fake_root(name, "0::/\n", Some(&memory_events(4))); + let folder = root.join("invocation"); + std::fs::create_dir_all(&folder).expect("invocation folder"); + let request = RunRequest::text("input"); + let mut recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + cgroup: CgroupReader::with_root(&root), + }); + std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)) + .expect("oom kill"); + recorder.exchange_failed( + Instant::now(), + Some(HostExit { + exit_code: None, + exit_signal: Some(9), + }), + ); + + let error = recorder + .finish(Err(FabricError::AdapterLifecycleOperation { + operation: "invoke", + runtime_id: "rt".to_string(), + code: code.to_string(), + message: "host exited".to_string(), + diagnostics: String::new(), + })) + .expect_err("error passes through"); + assert!(error.to_string().contains(code)); + let text = std::fs::read_to_string(folder.join(RECORD_FILE_NAME)).expect("record"); + let lines: Vec = text + .lines() + .map(|line| serde_json::from_str(line).expect("json")) + .collect(); + let _ = std::fs::remove_dir_all(root); + let start = lines.first().expect("start line").clone(); + let end = lines.last().expect("end line").clone(); + assert_eq!( + (&start["kind"], &end["kind"]), + (&"start".into(), &"end".into()) + ); + (start, end) + } + + #[test] + fn oom_kill_rise_in_an_exclusive_container_reads_sandbox_failed() { + let (start, end) = oom_kill_record("exclusive", true, HOST_CRASHED); + + assert_eq!(start["container_oom_kills"], 4); + assert_eq!(start["exclusive_container"], true); + assert_eq!(end["container_oom_kills"], 5); + assert_eq!( + end["stop"], + serde_json::json!({"cause": "sandbox_failed", "exit_signal": 9}) + ); + } + + #[test] + fn oom_kill_rise_in_a_shared_container_sets_no_sandbox_cause() { + let (start, end) = oom_kill_record("shared", false, HOST_CRASHED); + + assert_eq!(start["exclusive_container"], false); + assert_eq!(end["container_oom_kills"], 5); + assert_eq!( + end["stop"], + serde_json::json!({"cause": "unclassified", "exit_signal": 9}) + ); + } + + #[test] + fn oom_kill_rise_outranks_an_adapter_error_from_the_exchange() { + let (_, end) = oom_kill_record("adapter-error", true, "lifecycle_adapter_invoke_failed"); + + assert_eq!( + end["stop"], + serde_json::json!({"cause": "sandbox_failed", "exit_signal": 9}) + ); + } } diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index 964a79eda..886df0f2c 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -96,6 +96,11 @@ pub(crate) struct StartLine { /// recorded. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub override_keys: Vec, + /// `runtime.record.exclusive_container`: whether a rise in + /// `container_oom_kills` can set the `sandbox_failed` cause. + pub exclusive_container: bool, + /// The container's cgroup v2 `oom_kill` count when the invocation started. + pub container_oom_kills: OomKills, } /// One configured model. @@ -147,6 +152,114 @@ pub(crate) struct EndLine { /// Result artifacts, with paths relative to the artifact root. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub artifacts: Vec, + /// The container's cgroup v2 `oom_kill` count when the invocation ended. + pub container_oom_kills: OomKills, + /// Why the invocation stopped; absent when it holds nothing, as on success. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stop: Option, +} + +/// The cgroup v2 `memory.events` `oom_kill` count of the container Fabric +/// runs in, or why it could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(untagged)] +pub(crate) enum OomKills { + /// The count. + Count(u64), + /// The count could not be read. + Unavailable { + /// Why the count could not be read. + reason: OomKillsUnavailable, + }, +} + +/// Why the container's `oom_kill` count could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum OomKillsUnavailable { + /// The operating system has no cgroup v2 source. + Platform, + /// The host uses cgroup v1. + CgroupV1, + /// Fabric does not run at the root of its own cgroup, or the cgroup files + /// cannot be read. + NoOwnedCgroup, +} + +/// Why an invocation stopped. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct Stop { + /// Fabric's failure cause; absent on a cancel alone. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cause: Option, + /// What stopped the invocation, when a limit did. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// Whether Fabric force-stopped the adapter host. After a forced stop, + /// `exit_signal` is the signal Fabric sent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub forced: Option, + /// The limit that ended the invocation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, + /// The adapter host's exit code, when it exited. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exit_code: Option, + /// The signal that ended the adapter host, when one did. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exit_signal: Option, +} + +/// Fabric's own closed list of failure causes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum StopCause { + /// Core or the adapter host raised an error other than a timeout or a + /// host exit, such as an invalid adapter result or an adapter crash + /// inside the host. An error from the exchange with the host during an + /// out-of-memory rise in an exclusive container reads `SandboxFailed`. + FabricFailed, + /// The container's `oom_kill` count rose and + /// `runtime.record.exclusive_container` is set. + SandboxFailed, + /// A limit ended the invocation, such as Fabric's timeout. + AgentExhaustedBudget, + /// The adapter host exited, or nothing else matched. + Unclassified, + /// The adapter returned a failed result. + AgentFailed, +} + +/// What stopped an invocation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum StopReason { + /// The invocation reached its time limit. + Timeout, +} + +/// The limit that ended an invocation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct StopLimit { + /// The kind of limit. + #[serde(rename = "type")] + pub limit_type: LimitType, + /// The configured limit, in milliseconds for `time`: the invoke timeout + /// that applied, `runtime.timeout_seconds` or 1 hour when it is unset. + /// Core gives the adapter an earlier deadline, the timeout less + /// 10 seconds and at least half of it. + pub value: u64, + /// What the invocation used when it stopped, in the same unit. For a + /// forced stop, the time until core's timeout fired. + pub usage: u64, +} + +/// Kinds of limit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum LimitType { + /// Wall-clock time. + Time, } /// The end line's error. diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 4b4b2deee..571584a4a 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -13,11 +13,14 @@ //! Everything here is crate-private. The line format is public only through //! the `run-record-line` JSON Schema, version `v1alpha1`. +mod cgroup; mod invocation; mod lines; +mod stop; mod writer; -pub(crate) use invocation::{InvocationRecorder, RecordStart}; +pub(crate) use cgroup::CgroupReader; +pub(crate) use invocation::{HostExit, InvocationRecorder, RecordStart}; pub(crate) use lines::RecordLine; #[cfg(test)] pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/record/stop.rs b/crates/fabric-core/src/record/stop.rs new file mode 100644 index 000000000..c7dd127e3 --- /dev/null +++ b/crates/fabric-core/src/record/stop.rs @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Why an invocation stopped: the end line's `stop.cause`, in a fixed order +//! where the first match wins. A cause never changes `status`. + +use super::lines::StopCause; +use crate::runtime::RunStatus; + +/// What core knows about how an invocation ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct StopFacts { + /// Final status. + pub status: RunStatus, + /// Core or the adapter host raised an error other than a timeout or a + /// host exit, and the error did not come from the exchange with the host + /// during an out-of-memory rise. + pub fabric_failed: bool, + /// The container's `oom_kill` count rose and the run owns its container. + pub oom_rose: bool, + /// Fabric's timeout fired, or the adapter stopped at the deadline core gave it. + pub timed_out: bool, + /// The adapter host exited before it answered. + pub host_exited: bool, + /// The adapter returned a failed result. + pub agent_error: bool, +} + +/// The stop cause for `facts`: none on success and on a cancel alone. +pub(crate) fn stop_cause(facts: StopFacts) -> Option { + if facts.status == RunStatus::Succeeded { + return None; + } + if facts.fabric_failed { + return Some(StopCause::FabricFailed); + } + if facts.oom_rose { + return Some(StopCause::SandboxFailed); + } + if facts.timed_out { + return Some(StopCause::AgentExhaustedBudget); + } + if facts.status == RunStatus::Cancelled { + return None; + } + if facts.host_exited { + return Some(StopCause::Unclassified); + } + if facts.agent_error { + return Some(StopCause::AgentFailed); + } + Some(StopCause::Unclassified) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn failed() -> StopFacts { + StopFacts { + status: RunStatus::Failed, + fabric_failed: false, + oom_rose: false, + timed_out: false, + host_exited: false, + agent_error: false, + } + } + + #[test] + fn stop_cause_succeeded_run_has_no_cause_even_after_an_oom_rise() { + let facts = StopFacts { + status: RunStatus::Succeeded, + oom_rose: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), None); + } + + #[test] + fn stop_cause_core_error_during_an_oom_kill_reads_fabric_failed() { + let facts = StopFacts { + fabric_failed: true, + oom_rose: true, + host_exited: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), Some(StopCause::FabricFailed)); + } + + #[test] + fn stop_cause_oom_rise_with_a_host_exit_reads_sandbox_failed() { + let facts = StopFacts { + oom_rose: true, + host_exited: true, + timed_out: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), Some(StopCause::SandboxFailed)); + } + + #[test] + fn stop_cause_timeout_reads_agent_exhausted_budget() { + let facts = StopFacts { + timed_out: true, + agent_error: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), Some(StopCause::AgentExhaustedBudget)); + } + + #[test] + fn stop_cause_cancel_alone_has_no_cause_but_an_oom_rise_still_counts() { + let cancelled = StopFacts { + status: RunStatus::Cancelled, + ..failed() + }; + + assert_eq!(stop_cause(cancelled), None); + assert_eq!( + stop_cause(StopFacts { + oom_rose: true, + ..cancelled + }), + Some(StopCause::SandboxFailed) + ); + } + + #[test] + fn stop_cause_host_exit_reads_unclassified_before_an_agent_error() { + let facts = StopFacts { + host_exited: true, + agent_error: true, + ..failed() + }; + + assert_eq!(stop_cause(facts), Some(StopCause::Unclassified)); + } + + #[test] + fn stop_cause_agent_error_reads_agent_failed_and_nothing_reads_unclassified() { + assert_eq!( + stop_cause(StopFacts { + agent_error: true, + ..failed() + }), + Some(StopCause::AgentFailed) + ); + assert_eq!(stop_cause(failed()), Some(StopCause::Unclassified)); + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 51e21dd5f..37a035a5b 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -31,10 +31,14 @@ use crate::config::{ validate_tool_definitions, validate_workflow, }; use crate::error::{FabricError, Result}; -use crate::record::{InvocationRecorder, RecordStart}; +use crate::record::{CgroupReader, HostExit, InvocationRecorder, RecordStart}; static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; +/// Lifecycle error code for an invocation that outlived core's timeout. +pub(crate) const HOST_TIMEOUT: &str = "host_timeout"; +/// Lifecycle error code for an adapter host that exited or closed its pipes. +pub(crate) const HOST_CRASHED: &str = "host_crashed"; /// Lifecycle error code for an adapter result that core rejects. pub(crate) const INVALID_AGENT_RUN_RESULT: &str = "invalid_agent_run_result"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; @@ -47,6 +51,8 @@ const LOCAL_HOST_INVOKE_TIMEOUT: Duration = Duration::from_secs(60 * 60); const LOCAL_HOST_ADAPTER_RESERVE: Duration = Duration::from_secs(10); const LOCAL_HOST_STOP_TIMEOUT: Duration = Duration::from_secs(10); const LOCAL_HOST_EXIT_GRACE: Duration = Duration::from_secs(2); +// How long a run record waits for a crashed host's exit status. +const LOCAL_HOST_CRASH_WAIT: Duration = Duration::from_secs(1); const LOCAL_HOST_DIAGNOSTIC_LIMIT: usize = 16 * 1024; #[cfg(test)] pub(crate) const OPENAI_STREAM_HOST: &str = "127.0.0.1"; @@ -1546,7 +1552,7 @@ impl RuntimeAdapter for LocalHostAdapter { let host_crashed = matches!( &result, Err(FabricError::AdapterLifecycleOperation { code, .. }) - if code == "host_crashed" + if code == HOST_CRASHED ); if !host_crashed { result?; @@ -1813,6 +1819,7 @@ fn local_host_invocation_result( request: &request, fabric_home: &fabric_home, timeout: invoke_timeout, + cgroup: CgroupReader::system(), })); } adapter_invocation.runtime_context.deadline_millis = @@ -1869,12 +1876,21 @@ fn local_host_invocation_result( )) } Err(error) => { + let failed_at = Instant::now(); if matches!( &error, - FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" + FabricError::AdapterLifecycleOperation { code, .. } if code == HOST_TIMEOUT ) { invalidate_timed_out_local_host(&runtime.runtime_id, &host, &mut host_guard); } + // Only a record reads the exit status, so a run without one never waits. + if let Some(recorder) = recorder.as_mut() { + let crashed = matches!( + &error, + FabricError::AdapterLifecycleOperation { code, .. } if code == HOST_CRASHED + ); + recorder.exchange_failed(failed_at, local_host_exit(&mut host_guard, crashed)); + } Err(error) } } @@ -2039,6 +2055,25 @@ fn local_host_invocation_result( }) } +/// The host's wait status after a failed exchange, when it has exited. A +/// crashed host gets up to [`LOCAL_HOST_CRASH_WAIT`] for its exit to become +/// observable. +fn local_host_exit(host: &mut LocalAdapterHost, crashed: bool) -> Option { + let deadline = Instant::now() + + if crashed { + LOCAL_HOST_CRASH_WAIT + } else { + Duration::ZERO + }; + loop { + match host.child.try_wait() { + Ok(Some(status)) => return Some(HostExit::from_status(status)), + Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(10)), + Ok(None) | Err(_) => return None, + } + } +} + fn invalidate_timed_out_local_host( runtime_id: &str, expected_host: &Arc>, @@ -2148,7 +2183,7 @@ fn stop_service_host(record: &mut LocalServiceRecord) -> Result<()> { let removal = remove_local_host_files(&record.host); let host_crashed = matches!( &result, - Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "host_crashed" + Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == HOST_CRASHED ); if !host_crashed { result?; @@ -2438,7 +2473,7 @@ fn exchange_lifecycle_message( return Err(lifecycle_error( operation, runtime_id, - "host_crashed", + HOST_CRASHED, format!( "persistent local adapter host exited before {} ({status})", operation.as_str() @@ -2456,7 +2491,7 @@ fn exchange_lifecycle_message( let code = match (source.kind(), host.child.try_wait()) { // A closed lifecycle pipe is terminal even when the child exit // status is not observable yet. - (ErrorKind::BrokenPipe, _) | (_, Ok(Some(_))) => "host_crashed", + (ErrorKind::BrokenPipe, _) | (_, Ok(Some(_))) => HOST_CRASHED, _ => "host_io", }; return Err(lifecycle_error( @@ -2477,7 +2512,7 @@ fn exchange_lifecycle_message( return Err(lifecycle_error( operation, runtime_id, - "host_timeout", + HOST_TIMEOUT, format!( "persistent local adapter host did not complete {} within {} ms", operation.as_str(), @@ -2490,7 +2525,7 @@ fn exchange_lifecycle_message( return Err(lifecycle_error( operation, runtime_id, - "host_crashed", + HOST_CRASHED, format!( "persistent local adapter host exited while processing {}", operation.as_str() @@ -3951,6 +3986,17 @@ for line in sys.stdin: if MODE == "invoke_failure": response("invoke", error=failure("invoke", "fake_invoke", "invoke rejected")) continue + if MODE == "invoke_deadline_error": + response("invoke", error=failure("invoke", "timeout", "deadline reached")) + continue + if MODE == "exit_3": + sys.exit(3) + if MODE == "self_kill": + os.kill(os.getpid(), 9) + if MODE == "allocate_until_killed": + blocks = [] + while True: + blocks.append(b"x" * (16 * 1024 * 1024)) invocation = message["payload"] expected_fields = ( {"runtime_context", "request", "stream"} @@ -3990,7 +4036,15 @@ for line in sys.stdin: "context_env": invocation["runtime_context"]["environment"].get("env", {}), "deadline_millis": invocation["runtime_context"].get("deadline_millis"), } - if MODE in {"adapter_reported_failure", "blank_error_code", "free_text_error_code"}: + if MODE == "cancelled": + result = {"status": "cancelled", "output": output} + elif MODE == "adapter_deadline": + result = { + "status": "failed", + "output": output, + "error": {"code": "timeout", "message": "deadline reached", "retryable": False}, + } + elif MODE in {"adapter_reported_failure", "blank_error_code", "free_text_error_code"}: codes = { "blank_error_code": " \t", "free_text_error_code": "Rate limit for key sk-live-123 exceeded", @@ -5508,7 +5562,10 @@ for line in sys.stdin: fn record_plan(mode: &str) -> (PathBuf, RunPlan) { let (root, mut plan) = local_host_plan(mode); - plan.config.runtime.record = Some(crate::config::RecordConfig { enabled: true }); + plan.config.runtime.record = Some(crate::config::RecordConfig { + enabled: true, + exclusive_container: false, + }); (root, plan) } @@ -5725,6 +5782,21 @@ for line in sys.stdin: end["error"], serde_json::json!({"code": "host_timeout", "stage": "invoke", "retryable": false}) ); + let stop = &end["stop"]; + assert_eq!(stop["cause"], "agent_exhausted_budget"); + assert_eq!(stop["reason"], "timeout"); + assert_eq!(stop["forced"], true); + assert_eq!(stop["limit"]["value"], 100); + let usage = stop["limit"]["usage"].as_u64().expect("usage"); + // Usage stops when the timeout fires; the end line's elapsed time also + // covers core's stop of the hung host. + assert!(usage >= 100, "{stop}"); + assert!( + usage < end["elapsed_ms"].as_u64().expect("elapsed"), + "{end}" + ); + #[cfg(unix)] + assert_eq!(stop["exit_signal"], 9, "{stop}"); let _ = stop_runtime(&plan, &runtime); let _ = fs::remove_dir_all(root); } @@ -5774,4 +5846,153 @@ for line in sys.stdin: } let _ = fs::remove_dir_all(root); } + + fn end_line_of(root: &Path) -> Value { + let (text, lines) = only_record(root); + let end = lines.last().expect("end line").clone(); + assert_eq!(end["kind"], "end", "{text}"); + end + } + + fn failed_invocation_end_line(mode: &str) -> (FabricError, Value) { + let (root, plan) = record_plan(mode); + let runtime = start_runtime(&plan).expect("start local host"); + let error = invoke_runtime(&plan, &runtime, RunRequest::text("fail")) + .expect_err("lifecycle error still raises"); + let end = end_line_of(&root); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + (error, end) + } + + fn result_end_line(mode: &str) -> (RunResult, Value) { + let (root, plan) = record_plan(mode); + let result = run_plan(&plan, RunRequest::text("run")).expect("result"); + let end = end_line_of(&root); + let _ = fs::remove_dir_all(root); + (result, end) + } + + #[test] + fn stop_succeeded_invocation_has_no_stop() { + let (result, end) = result_end_line("success"); + + assert_eq!(result.status, RunStatus::Succeeded); + assert!(end.get("stop").is_none(), "{end}"); + assert!(end.get("container_oom_kills").is_some(), "{end}"); + } + + #[test] + fn stop_host_exit_3_reads_unclassified_with_the_exit_code() { + let (error, end) = failed_invocation_end_line("exit_3"); + + assert!(error.to_string().contains("host_crashed"), "{error}"); + assert_eq!(end["error"]["code"], "host_crashed"); + assert_eq!( + end["stop"], + serde_json::json!({"cause": "unclassified", "exit_code": 3}) + ); + } + + #[cfg(unix)] + #[test] + fn stop_host_killed_by_a_signal_reads_unclassified_with_the_signal() { + let (_, end) = failed_invocation_end_line("self_kill"); + + assert_eq!( + end["stop"], + serde_json::json!({"cause": "unclassified", "exit_signal": 9}) + ); + } + + #[test] + fn stop_invalid_adapter_result_reads_fabric_failed() { + let (error, end) = failed_invocation_end_line("blank_error_code"); + + assert!( + error.to_string().contains("invalid_agent_run_result"), + "{error}" + ); + assert_eq!(end["stop"], serde_json::json!({"cause": "fabric_failed"})); + } + + #[test] + fn stop_host_reported_invoke_failure_reads_fabric_failed() { + let (_, end) = failed_invocation_end_line("invoke_failure"); + + assert_eq!(end["error"]["code"], "fake_invoke"); + assert_eq!(end["stop"], serde_json::json!({"cause": "fabric_failed"})); + } + + #[test] + fn stop_adapter_deadline_reported_as_a_lifecycle_error_reads_an_unforced_time_limit() { + let (_, end) = failed_invocation_end_line("invoke_deadline_error"); + + assert_eq!(end["error"]["code"], "timeout"); + let stop = &end["stop"]; + assert_eq!(stop["cause"], "agent_exhausted_budget"); + assert_eq!(stop["forced"], false); + assert_eq!(stop["limit"]["value"], 3_600_000); + } + + /// The out-of-memory path end to end. It needs Linux with cgroup v2 and + /// a memory-limited container in which Fabric owns the root cgroup, so it + /// is ignored by default; the PR body gives the `docker run` command. + #[cfg(target_os = "linux")] + #[test] + #[ignore = "needs a memory-limited cgroup v2 container"] + fn stop_oom_kill_in_an_exclusive_container_reads_sandbox_failed() { + let (root, mut plan) = record_plan("allocate_until_killed"); + plan.config.runtime.record = Some(crate::config::RecordConfig { + enabled: true, + exclusive_container: true, + }); + let runtime = start_runtime(&plan).expect("start local host"); + + invoke_runtime(&plan, &runtime, RunRequest::text("allocate")) + .expect_err("the kernel kills the host"); + let (_, lines) = only_record(&root); + + let count = |kind| line_of_kind(&lines, kind)["container_oom_kills"].as_u64(); + let (start, end) = ( + count("start").expect("start count"), + count("end").expect("end count"), + ); + assert!(end > start, "{lines:?}"); + let stop = &line_of_kind(&lines, "end")["stop"]; + assert_eq!(stop["cause"], "sandbox_failed", "{stop}"); + assert_eq!(stop["exit_signal"], 9, "{stop}"); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn stop_failed_result_reads_agent_failed() { + let (result, end) = result_end_line("adapter_reported_failure"); + + assert_eq!(result.status, RunStatus::Failed); + assert_eq!(end["stop"], serde_json::json!({"cause": "agent_failed"})); + } + + #[test] + fn stop_cancelled_result_has_no_stop() { + let (result, end) = result_end_line("cancelled"); + + assert_eq!(result.status, RunStatus::Cancelled); + assert_eq!(end["status"], "cancelled"); + assert!(end.get("stop").is_none(), "{end}"); + } + + #[test] + fn stop_adapter_deadline_reads_an_unforced_time_limit() { + let (_, end) = result_end_line("adapter_deadline"); + + let stop = &end["stop"]; + assert_eq!(stop["cause"], "agent_exhausted_budget"); + assert_eq!(stop["reason"], "timeout"); + assert_eq!(stop["forced"], false); + assert_eq!(stop["limit"]["type"], "time"); + assert_eq!(stop["limit"]["value"], 3_600_000); + assert!(stop["limit"]["usage"].is_u64(), "{stop}"); + } } diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index 703cff7b4..c12d9e71d 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -518,6 +518,7 @@ The model defines the following fields: | Field | Type | Required | Default | Constraints | Description | | --- | --- | --- | --- | --- | --- | | `enabled` | `StrictBool` | Yes | — | `Strict(strict=True)` | — | +| `exclusive_container` | `StrictBool \| None` | No | `None` | — | Whether nothing else runs in the container during an invocation, including other runtimes in the same process, so a rise in the container's out-of-memory kill count reads sandbox_failed in the record. Unset means false. | --- diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx index d33e42889..b72fc30ee 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n}"}} />
+
bool,\n    pub exclusive_container: bool,\n}"}} />
Per-invocation run record settings. @@ -21,6 +21,10 @@ When enabled, core appends a start line and an end line for each local invocatio Whether to write a run record for each invocation. Requires an artifact root, the `local` environment provider, and a `process` or `python` adapter. +### `exclusive_container: bool` + +Whether nothing else runs in the container during an invocation, including other runtimes in the same process, so a rise in the container's out-of-memory kill count reads `sandbox_failed` in the record. Unset means false. + ## Trait Implementations ### `impl Clone for RecordConfig` diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 3fa5363a9..59e092b21 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -995,6 +995,62 @@ The end line describes the invocation. A one-shot `Fabric.run` stops the runtime after the invocation; when that stop fails, the result reads `failed` with stage `stop`, and the record still holds the invocation's own end line. +The end line's `stop` says why the invocation stopped. It is absent on +success. `stop.cause` comes from NeMo Fabric's own closed list. Three names are +Harbor's failure causes: `sandbox_failed`, `agent_exhausted_budget`, and +`unclassified`. A Harbor runner maps `fabric_failed` to `harbor_failed` and +`agent_failed` to `unclassified`. The first rule that matches sets the cause: + +1. `fabric_failed`: NeMo Fabric or the adapter host raised an error that is + not a timeout or a host exit. Examples are an invalid adapter result and an + adapter that crashed inside the adapter host. The adapter contract returns + the agent's own failure as a `failed` result, never as a raised error. An + error from the exchange with the adapter host during an out-of-memory rise + in an exclusive container reads `sandbox_failed` instead, because a kill + can make the adapter raise; NeMo Fabric's own errors after a complete + response stay `fabric_failed`. +2. `sandbox_failed`: the container's out-of-memory kill count rose during the + invocation, and `runtime.record.exclusive_container` is `True`. +3. `agent_exhausted_budget`: the invocation reached its time limit. +4. A cancelled invocation that matched none of the above has no cause. +5. `unclassified`: the adapter host exited before it answered. +6. `agent_failed`: the adapter returned a `failed` result. +7. `unclassified`: anything else. + +An adapter that sets its own error code for a harness crash or a setup error +in a `failed` result, such as `claude_process_failed`, reads `agent_failed`. +NeMo Fabric does not interpret adapter-specific codes. + +A timeout sets `stop.reason` to `timeout` and `stop.limit` to +`{"type": "time", "value": , "usage": }`. `value` is the invoke +timeout that applied: `runtime.timeout_seconds`, or 1 hour when it is unset. +Two timeouts reach this rule: + +- NeMo Fabric's own timeout fired (error code `host_timeout`). `forced` is + `true`, and `usage` is the time until the timeout fired. `elapsed_ms` also + covers NeMo Fabric stopping the hung adapter host, so it is larger. + `exit_signal` is then the signal NeMo Fabric sent, not a kernel kill. +- The adapter stopped at the deadline NeMo Fabric gave it and reported error + code `timeout`, in a `failed` result or as an error. `forced` is `false`. + That deadline is earlier than `value`: the timeout less 10 seconds, and at + least half of it. + +When the adapter host exited, `stop.exit_code` or `stop.exit_signal` holds its +wait status. + +On Linux with cgroup v2, when NeMo Fabric runs at the root of its own +container's cgroup, both lines record the container's `memory.events` +`oom_kill` count as `container_oom_kills`. Otherwise the field reads +`{"reason": "platform" | "cgroup_v1" | "no_owned_cgroup"}`. The start line +also records `exclusive_container`. Set `exclusive_container` in +`runtime.record` to `True` only when nothing else runs in the container during +an invocation, including other runtimes in the same process. Otherwise a kill +can belong to another run, so the count is recorded but sets no cause: + +```python +record = RecordConfig(enabled=True, exclusive_container=True) +``` + `run-record.jsonl` never holds prompts, model output, error messages, tool payloads, adapter metadata, or context values outside the keys above. The same folder also holds `adapter-invocation.json` and `stdout.txt`, which NeMo Fabric diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index 7d6c5e994..6ba4ad3ee 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -565,6 +565,10 @@ "enabled": { "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", "type": "boolean" + }, + "exclusive_container": { + "description": "Whether nothing else runs in the container during an invocation,\nincluding other runtimes in the same process, so a rise in the\ncontainer's out-of-memory kill count reads `sandbox_failed` in the\nrecord. Unset means false.", + "type": "boolean" } }, "required": [ diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index 5d3211b4e..03499b493 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -1933,6 +1933,10 @@ "enabled": { "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", "type": "boolean" + }, + "exclusive_container": { + "description": "Whether nothing else runs in the container during an invocation,\nincluding other runtimes in the same process, so a rise in the\ncontainer's out-of-memory kill count reads `sandbox_failed` in the\nrecord. Unset means false.", + "type": "boolean" } }, "required": [ diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index 7b0f8642e..f9c976bfb 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -84,6 +84,10 @@ }, "type": "array" }, + "container_oom_kills": { + "$ref": "#/$defs/OomKills", + "description": "The container's cgroup v2 `oom_kill` count when the invocation ended." + }, "elapsed_ms": { "description": "Monotonic time from the start line to the end line, in milliseconds.", "format": "uint64", @@ -105,6 +109,17 @@ "$ref": "#/$defs/RunStatus", "description": "Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`." }, + "stop": { + "anyOf": [ + { + "$ref": "#/$defs/Stop" + }, + { + "type": "null" + } + ], + "description": "Why the invocation stopped; absent when it holds nothing, as on success." + }, "usage": { "anyOf": [ { @@ -119,7 +134,8 @@ }, "required": [ "status", - "elapsed_ms" + "elapsed_ms", + "container_oom_kills" ], "type": "object" }, @@ -226,6 +242,16 @@ } ] }, + "LimitType": { + "description": "Kinds of limit.", + "oneOf": [ + { + "const": "time", + "description": "Wall-clock time.", + "type": "string" + } + ] + }, "ModelName": { "description": "One configured model.", "properties": { @@ -258,6 +284,50 @@ ], "type": "object" }, + "OomKills": { + "anyOf": [ + { + "description": "The count.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + { + "description": "The count could not be read.", + "properties": { + "reason": { + "$ref": "#/$defs/OomKillsUnavailable", + "description": "Why the count could not be read." + } + }, + "required": [ + "reason" + ], + "type": "object" + } + ], + "description": "The cgroup v2 `memory.events` `oom_kill` count of the container Fabric\nruns in, or why it could not be read." + }, + "OomKillsUnavailable": { + "description": "Why the container's `oom_kill` count could not be read.", + "oneOf": [ + { + "const": "platform", + "description": "The operating system has no cgroup v2 source.", + "type": "string" + }, + { + "const": "cgroup_v1", + "description": "The host uses cgroup v1.", + "type": "string" + }, + { + "const": "no_owned_cgroup", + "description": "Fabric does not run at the root of its own cgroup, or the cgroup files\ncannot be read.", + "type": "string" + } + ] + }, "RecordSchema": { "description": "The record's schema name.", "oneOf": [ @@ -312,6 +382,10 @@ "description": "Config name (`metadata.name`).", "type": "string" }, + "container_oom_kills": { + "$ref": "#/$defs/OomKills", + "description": "The container's cgroup v2 `oom_kill` count when the invocation started." + }, "context": { "additionalProperties": { "$ref": "#/$defs/ContextValue" @@ -319,6 +393,10 @@ "description": "`RunRequest.context`. Allowlisted identity and Harbor link keys keep a\nstring or integer value; every other key is listed by name only.", "type": "object" }, + "exclusive_container": { + "description": "`runtime.record.exclusive_container`: whether a rise in\n`container_oom_kills` can set the `sandbox_failed` cause.", + "type": "boolean" + }, "fabric_version": { "description": "NeMo Fabric version that wrote the record.", "type": "string" @@ -352,9 +430,140 @@ "agent_name", "harness", "fabric_version", - "timeout_ms" + "timeout_ms", + "exclusive_container", + "container_oom_kills" ], "type": "object" + }, + "Stop": { + "description": "Why an invocation stopped.", + "properties": { + "cause": { + "anyOf": [ + { + "$ref": "#/$defs/StopCause" + }, + { + "type": "null" + } + ], + "description": "Fabric's failure cause; absent on a cancel alone." + }, + "exit_code": { + "description": "The adapter host's exit code, when it exited.", + "format": "int32", + "type": [ + "integer", + "null" + ] + }, + "exit_signal": { + "description": "The signal that ended the adapter host, when one did.", + "format": "int32", + "type": [ + "integer", + "null" + ] + }, + "forced": { + "description": "Whether Fabric force-stopped the adapter host. After a forced stop,\n`exit_signal` is the signal Fabric sent.", + "type": [ + "boolean", + "null" + ] + }, + "limit": { + "anyOf": [ + { + "$ref": "#/$defs/StopLimit" + }, + { + "type": "null" + } + ], + "description": "The limit that ended the invocation." + }, + "reason": { + "anyOf": [ + { + "$ref": "#/$defs/StopReason" + }, + { + "type": "null" + } + ], + "description": "What stopped the invocation, when a limit did." + } + }, + "type": "object" + }, + "StopCause": { + "description": "Fabric's own closed list of failure causes.", + "oneOf": [ + { + "const": "fabric_failed", + "description": "Core or the adapter host raised an error other than a timeout or a\nhost exit, such as an invalid adapter result or an adapter crash\ninside the host. An error from the exchange with the host during an\nout-of-memory rise in an exclusive container reads `SandboxFailed`.", + "type": "string" + }, + { + "const": "sandbox_failed", + "description": "The container's `oom_kill` count rose and\n`runtime.record.exclusive_container` is set.", + "type": "string" + }, + { + "const": "agent_exhausted_budget", + "description": "A limit ended the invocation, such as Fabric's timeout.", + "type": "string" + }, + { + "const": "unclassified", + "description": "The adapter host exited, or nothing else matched.", + "type": "string" + }, + { + "const": "agent_failed", + "description": "The adapter returned a failed result.", + "type": "string" + } + ] + }, + "StopLimit": { + "description": "The limit that ended an invocation.", + "properties": { + "type": { + "$ref": "#/$defs/LimitType", + "description": "The kind of limit." + }, + "usage": { + "description": "What the invocation used when it stopped, in the same unit. For a\nforced stop, the time until core's timeout fired.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "value": { + "description": "The configured limit, in milliseconds for `time`: the invoke timeout\nthat applied, `runtime.timeout_seconds` or 1 hour when it is unset.\nCore gives the adapter an earlier deadline, the timeout less\n10 seconds and at least half of it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "type", + "value", + "usage" + ], + "type": "object" + }, + "StopReason": { + "description": "What stopped an invocation.", + "oneOf": [ + { + "const": "timeout", + "description": "The invocation reached its time limit.", + "type": "string" + } + ] } }, "$schema": "https://json-schema.org/draft/2020-12/schema", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index 66ca0ba7d..a378da50e 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -200,6 +200,15 @@ class RecordConfig(FabricBaseModel): model_config = ConfigDict(extra="forbid") enabled: StrictBool + exclusive_container: StrictBool | None = Field( + default=None, + description=( + "Whether nothing else runs in the container during an invocation, " + "including other runtimes in the same process, so a rise in the " + "container's out-of-memory kill count reads sandbox_failed in the " + "record. Unset means false." + ), + ) class RuntimeConfig(FabricBaseModel): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 0456b68ff..34b218914 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -372,7 +372,8 @@ class _RuntimeConfig(_ConfigMapping): artifacts: Optional artifact-root path. timeout_seconds: Optional invocation deadline in seconds. max_turns: Optional harness turn limit. - record: Optional run record settings, ``{"enabled": bool}``. + record: Optional run record settings, ``{"enabled": bool}`` with an + optional boolean ``exclusive_container``. extra_fields: Preserved extension fields not recognized by this SDK. """ @@ -430,11 +431,16 @@ def __init__( values["max_turns"] = max_turns if record is not None: record = _mapping(record, "runtime record") - if set(record) != {"enabled"} or not isinstance(record["enabled"], bool): + if ( + "enabled" not in record + or not set(record) <= {"enabled", "exclusive_container"} + or not all(isinstance(value, bool) for value in record.values()) + ): raise FabricConfigError( - "runtime record must be an object with only a boolean enabled field" + "runtime record must be an object with a boolean enabled field " + "and an optional boolean exclusive_container field" ) - values["record"] = {"enabled": record["enabled"]} + values["record"] = dict(record) super().__init__(values, extra_fields=extra_fields) @classmethod diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 1c06367a3..45e8d823b 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -55,7 +55,11 @@ default and needs an artifact root, the `local` environment provider, and a `process` or `python` adapter. The result lists the file as the artifact with kind `run_record`. The file has a `start` line, written before the request, and an `end` line with status, -elapsed time, error code, numeric usage, and artifact names. It holds no +elapsed time, error code, numeric usage, artifact names, and `stop`, which +gives NeMo Fabric's failure cause (`fabric_failed`, `sandbox_failed`, +`agent_exhausted_budget`, `unclassified`, or `agent_failed`), the time limit +for a timeout, and the adapter host's exit status. Only a `failed` result +reads `agent_failed`; an error NeMo Fabric raises never does. It holds no prompts, output, or error messages. A record without an end line has an unknown outcome; do not count it as failed. Recording is synchronous and best-effort: a failed record write adds a `record_write_failed` event and does diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index e527e819a..a20f9f2da 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -721,6 +721,24 @@ def test_runtime_record_round_trips_and_rejects_other_shapes(): _FabricConfigSnapshot.from_mapping(raw) +def test_runtime_record_exclusive_container_round_trips_and_rejects_non_boolean(): + record = RecordConfig(enabled=True, exclusive_container=True) + + assert record.to_mapping() == {"enabled": True, "exclusive_container": True} + assert RecordConfig(enabled=True).to_mapping() == {"enabled": True} + with pytest.raises(ValidationError): + RecordConfig(enabled=True, exclusive_container="yes") + raw = _plan()["config"] + raw["runtime"] = {"record": {"enabled": True, "exclusive_container": True}} + assert _FabricConfigSnapshot.from_mapping(raw).runtime.record == { + "enabled": True, + "exclusive_container": True, + } + raw["runtime"]["record"]["exclusive_container"] = "yes" + with pytest.raises(FabricConfigError, match="optional boolean exclusive_container"): + _FabricConfigSnapshot.from_mapping(raw) + + def test_max_turns_matches_rust_u32_range(): maximum = (1 << 32) - 1 From 41670fff7d74a1a959d14e2b80fdb90a81569afe Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 05:56:36 -0700 Subject: [PATCH 028/159] test: drop impossible host exits from two stop rule tests InvocationRecorder::stop never sets fabric_failed together with host_exited, and a timeout and a host exit come from one error code, so they are never both true either. Two rule tests used those states, which taught the rule from cases that cannot happen. The second test now checks the real boundary: an out-of-memory rise outranks a timeout. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/stop.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/fabric-core/src/record/stop.rs b/crates/fabric-core/src/record/stop.rs index c7dd127e3..53da0ef13 100644 --- a/crates/fabric-core/src/record/stop.rs +++ b/crates/fabric-core/src/record/stop.rs @@ -83,7 +83,6 @@ mod tests { let facts = StopFacts { fabric_failed: true, oom_rose: true, - host_exited: true, ..failed() }; @@ -91,10 +90,9 @@ mod tests { } #[test] - fn stop_cause_oom_rise_with_a_host_exit_reads_sandbox_failed() { + fn stop_cause_oom_rise_outranks_a_timeout() { let facts = StopFacts { oom_rose: true, - host_exited: true, timed_out: true, ..failed() }; From ff6141cfd80733f933f155d4f5c7b7715ca12c59 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 05:56:37 -0700 Subject: [PATCH 029/159] docs: say when a forced stop records the signal Fabric sent terminate_local_host waits 2 seconds for the host to exit on its own before it kills it, so a host that exits in that window records its own status. On Windows the Job Object stop gives exit code 1 and no signal. State both instead of saying exit_signal is always the signal Fabric sent. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/lines.rs | 5 +++-- docs/sdk/python.mdx | 4 +++- schemas/sdk/run-record-line.schema.json | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index 886df0f2c..3e02f1789 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -195,8 +195,9 @@ pub(crate) struct Stop { /// What stopped the invocation, when a limit did. #[serde(default, skip_serializing_if = "Option::is_none")] pub reason: Option, - /// Whether Fabric force-stopped the adapter host. After a forced stop, - /// `exit_signal` is the signal Fabric sent. + /// Whether Fabric force-stopped the adapter host. After a forced stop on + /// Linux and macOS, `exit_signal` is usually the signal Fabric sent (9), + /// unless the host exited by itself during the 2-second grace period. #[serde(default, skip_serializing_if = "Option::is_none")] pub forced: Option, /// The limit that ended the invocation. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 59e092b21..60eca0840 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1029,7 +1029,9 @@ Two timeouts reach this rule: - NeMo Fabric's own timeout fired (error code `host_timeout`). `forced` is `true`, and `usage` is the time until the timeout fired. `elapsed_ms` also covers NeMo Fabric stopping the hung adapter host, so it is larger. - `exit_signal` is then the signal NeMo Fabric sent, not a kernel kill. + On Linux and macOS, `exit_signal` is then usually the signal NeMo Fabric + sent (9), unless the host exited by itself during the 2-second grace period. + On Windows, a forced stop gives `exit_code` 1 and no signal. - The adapter stopped at the deadline NeMo Fabric gave it and reported error code `timeout`, in a `failed` result or as an error. `forced` is `false`. That deadline is earlier than `value`: the timeout less 10 seconds, and at diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index f9c976bfb..99839a0fb 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -467,7 +467,7 @@ ] }, "forced": { - "description": "Whether Fabric force-stopped the adapter host. After a forced stop,\n`exit_signal` is the signal Fabric sent.", + "description": "Whether Fabric force-stopped the adapter host. After a forced stop on\nLinux and macOS, `exit_signal` is usually the signal Fabric sent (9),\nunless the host exited by itself during the 2-second grace period.", "type": [ "boolean", "null" From 59cee92e161d69ecde1dc82adbe6f81ffcff85bb Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:52:38 -0700 Subject: [PATCH 030/159] fix: read the end oom_kill count when the exchange ends finish read the container's oom_kill count after core had released the host lock and processed the response. In that window another caller on the same runtime can start the next exchange, and its out-of-memory kill would enter this record's rise and could set sandbox_failed on the wrong invocation. Read the count under the host lock as soon as exchange_lifecycle_message returns, before a timed-out host is stopped, and carry it to finish in an ExchangeEnd value together with the exchange's elapsed time and, after a failure, the host's wait status. ExchangeEnd replaces exchange_failed and is set on success too. After a host_crashed error the count is read again once the exit is observed and the larger value is kept, because the kernel counts a kill after it sends SIGKILL. File writes and the sync stay outside the lock. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 177 ++++++++++++++++---- crates/fabric-core/src/runtime.rs | 25 ++- docs/sdk/python.mdx | 6 +- 3 files changed, 169 insertions(+), 39 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 2c18fa180..d101e9466 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -111,18 +111,37 @@ pub(crate) struct InvocationRecorder { cgroup: CgroupReader, oom_kills_at_start: OomKills, exclusive_container: bool, - failed_exchange: Option, + exchange: Option, } -/// What core saw when the exchange with the adapter host failed. -#[derive(Debug, Clone, Copy)] -struct FailedExchange { - /// Time from the start line to the failure, before core's own cleanup. +/// What core saw when the exchange with the adapter host ended. Core reads +/// it under the host lock, before it stops a timed-out host and before a +/// later invocation can run, so every value describes this invocation. +#[derive(Debug, Clone)] +pub(crate) struct ExchangeEnd { + /// Time from the start line to the end of the exchange, before core's + /// own cleanup. elapsed_ms: u64, - /// The host's wait status, when it had exited. + /// The container's out-of-memory kill count at that moment, or the + /// larger count read after a crashed host's exit. + oom_kills: OomKills, + /// Whether the exchange raised an error. + failed: bool, + /// The host's wait status after a failed exchange, when it had exited. host_exit: Option, } +impl ExchangeEnd { + /// Mark the exchange failed, with the host's wait status when it exited. + pub(crate) fn with_host_exit(self, host_exit: Option) -> Self { + Self { + failed: true, + host_exit, + ..self + } + } +} + impl InvocationRecorder { /// Create the record and write its start line. pub(crate) fn start(start: RecordStart<'_>) -> Self { @@ -141,7 +160,7 @@ impl InvocationRecorder { cgroup: start.cgroup.clone(), oom_kills_at_start, exclusive_container: exclusive_container(start.plan), - failed_exchange: None, + exchange: None, }; match RecordWriter::create(&recorder.path) { Ok(writer) => { @@ -154,14 +173,33 @@ impl InvocationRecorder { recorder } - /// Keep when the exchange with the adapter host failed and the host's - /// wait status, for the end line's `stop`. Core calls this before its - /// cleanup, such as stopping a timed-out host, so `failed_at` excludes it. - pub(crate) fn exchange_failed(&mut self, failed_at: Instant, host_exit: Option) { - self.failed_exchange = Some(FailedExchange { - elapsed_ms: millis(failed_at.saturating_duration_since(self.started)), - host_exit, - }); + /// The end values at `ended_at`, when the exchange with the host + /// returned: the container's out-of-memory kill count, read now. Core + /// calls this under the host lock. + pub(crate) fn read_exchange_end(&self, ended_at: Instant) -> ExchangeEnd { + ExchangeEnd { + elapsed_ms: millis(ended_at.saturating_duration_since(self.started)), + oom_kills: self.cgroup.oom_kills(), + failed: false, + host_exit: None, + } + } + + /// `end` with the larger of its out-of-memory kill count and a count + /// read now. Core calls this after a crashed host's exit is observed: + /// the kernel counts a kill after it sends SIGKILL, so a host killed for + /// another process's allocation can exit before the count rises. + pub(crate) fn recount_oom_kills(&self, end: ExchangeEnd) -> ExchangeEnd { + let oom_kills = match (end.oom_kills, self.cgroup.oom_kills()) { + (OomKills::Count(at_end), OomKills::Count(now)) => OomKills::Count(at_end.max(now)), + (at_end, _) => at_end, + }; + ExchangeEnd { oom_kills, ..end } + } + + /// Keep what the exchange saw for the end line. + pub(crate) fn exchange_ended(&mut self, end: ExchangeEnd) { + self.exchange = Some(end); } /// Write and sync the end line for `outcome`, then return it. A result @@ -169,7 +207,11 @@ impl InvocationRecorder { /// `record_write_failed` event when any write or the sync failed. pub(crate) fn finish(mut self, outcome: Result) -> Result { let elapsed_ms = millis(self.started.elapsed()); - let oom_kills = self.cgroup.oom_kills(); + // Core always records the exchange's end; reading here is a fallback. + let oom_kills = self + .exchange + .as_ref() + .map_or_else(|| self.cgroup.oom_kills(), |exchange| exchange.oom_kills); let stop = self.stop(&outcome, elapsed_ms, oom_kills); let end = match &outcome { Ok(result) => result_end_line(result, elapsed_ms, oom_kills, stop), @@ -218,6 +260,7 @@ impl InvocationRecorder { elapsed_ms: u64, oom_kills_at_end: OomKills, ) -> Option { + let exchange = self.exchange.as_ref(); let oom_rose = self.exclusive_container && matches!( (self.oom_kills_at_start, oom_kills_at_end), @@ -255,7 +298,7 @@ impl InvocationRecorder { let forced = code == Some(HOST_TIMEOUT); let timed_out = forced || code == Some(ADAPTER_TIMEOUT_CODE); let host_exited = code == Some(HOST_CRASHED); - let exchange_error = self.failed_exchange.is_some(); + let exchange_error = exchange.is_some_and(|exchange| exchange.failed); let facts = StopFacts { status: RunStatus::Failed, fabric_failed: !(timed_out || host_exited || (exchange_error && oom_rose)), @@ -267,10 +310,8 @@ impl InvocationRecorder { (facts, timed_out.then_some(forced)) } }; - let usage = self - .failed_exchange - .map_or(elapsed_ms, |failed| failed.elapsed_ms); - let host_exit = self.failed_exchange.and_then(|failed| failed.host_exit); + let usage = exchange.map_or(elapsed_ms, |exchange| exchange.elapsed_ms); + let host_exit = exchange.and_then(|exchange| exchange.host_exit); let stop = Stop { cause: stop_cause(facts), reason: forced.map(|_| StopReason::Timeout), @@ -827,8 +868,15 @@ mod tests { /// Run one invocation whose exchange with the host fails with `code` after /// the host died of SIGKILL, while a fake cgroup's `oom_kill` count rises - /// from 4 to 5, and return its start and end lines. - fn oom_kill_record(name: &str, exclusive_container: bool, code: &str) -> (Value, Value) { + /// from 4 to 5, and return its start and end lines. With `counted_late`, + /// the count rises only after the exchange's end was read, and the end is + /// recounted as core does for a crashed host. + fn oom_kill_record( + name: &str, + exclusive_container: bool, + code: &str, + counted_late: bool, + ) -> (Value, Value) { use crate::config::RecordConfig; use crate::record::cgroup::tests::{fake_root, memory_events}; let (mut plan, runtime, invocation) = record_test_fixture(); @@ -849,15 +897,22 @@ mod tests { timeout: Duration::from_secs(5), cgroup: CgroupReader::with_root(&root), }); - std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)) - .expect("oom kill"); - recorder.exchange_failed( - Instant::now(), - Some(HostExit { + let count_the_kill = + || std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)); + if !counted_late { + count_the_kill().expect("oom kill"); + } + let mut end = recorder + .read_exchange_end(Instant::now()) + .with_host_exit(Some(HostExit { exit_code: None, exit_signal: Some(9), - }), - ); + })); + if counted_late { + count_the_kill().expect("oom kill"); + end = recorder.recount_oom_kills(end); + } + recorder.exchange_ended(end); let error = recorder .finish(Err(FabricError::AdapterLifecycleOperation { @@ -886,7 +941,7 @@ mod tests { #[test] fn oom_kill_rise_in_an_exclusive_container_reads_sandbox_failed() { - let (start, end) = oom_kill_record("exclusive", true, HOST_CRASHED); + let (start, end) = oom_kill_record("exclusive", true, HOST_CRASHED, false); assert_eq!(start["container_oom_kills"], 4); assert_eq!(start["exclusive_container"], true); @@ -899,7 +954,7 @@ mod tests { #[test] fn oom_kill_rise_in_a_shared_container_sets_no_sandbox_cause() { - let (start, end) = oom_kill_record("shared", false, HOST_CRASHED); + let (start, end) = oom_kill_record("shared", false, HOST_CRASHED, false); assert_eq!(start["exclusive_container"], false); assert_eq!(end["container_oom_kills"], 5); @@ -911,11 +966,67 @@ mod tests { #[test] fn oom_kill_rise_outranks_an_adapter_error_from_the_exchange() { - let (_, end) = oom_kill_record("adapter-error", true, "lifecycle_adapter_invoke_failed"); + let (_, end) = oom_kill_record( + "adapter-error", + true, + "lifecycle_adapter_invoke_failed", + false, + ); assert_eq!( end["stop"], serde_json::json!({"cause": "sandbox_failed", "exit_signal": 9}) ); } + + #[test] + fn oom_kill_counted_after_the_crashed_host_exited_still_reads_sandbox_failed() { + let (_, end) = oom_kill_record("counted-late", true, HOST_CRASHED, true); + + assert_eq!(end["container_oom_kills"], 5); + assert_eq!( + end["stop"], + serde_json::json!({"cause": "sandbox_failed", "exit_signal": 9}) + ); + } + + #[test] + fn oom_kill_counted_after_the_exchange_ended_stays_out_of_the_record() { + use crate::config::RecordConfig; + use crate::record::cgroup::tests::{fake_root, memory_events}; + let (mut plan, runtime, invocation) = record_test_fixture(); + plan.config.runtime.record = Some(RecordConfig { + enabled: true, + exclusive_container: true, + }); + let root = fake_root("after-exchange", "0::/\n", Some(&memory_events(4))); + let folder = root.join("invocation"); + std::fs::create_dir_all(&folder).expect("invocation folder"); + let request = RunRequest::text("input"); + let mut recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + cgroup: CgroupReader::with_root(&root), + }); + recorder.exchange_ended(recorder.read_exchange_end(Instant::now())); + // Core released the host lock, so a later invocation on the same + // runtime can be killed before this result is finished. + std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)) + .expect("later oom kill"); + let mut result = succeeded_result(&invocation); + result.status = RunStatus::Failed; + + recorder.finish(Ok(result)).expect("result"); + + let text = std::fs::read_to_string(folder.join(RECORD_FILE_NAME)).expect("record"); + let _ = std::fs::remove_dir_all(root); + let end: Value = + serde_json::from_str(text.lines().last().expect("end line")).expect("json"); + assert_eq!(end["container_oom_kills"], 4, "{end}"); + assert_eq!(end["stop"], serde_json::json!({"cause": "agent_failed"})); + } } diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 37a035a5b..d9c2b1a92 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1856,13 +1856,23 @@ fn local_host_invocation_result( } }; let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; - match exchange_lifecycle_message( + let exchange = exchange_lifecycle_message( &mut host_guard, &runtime.runtime_id, &lifecycle_request, invoke_timeout, - ) { + ); + let ended_at = Instant::now(); + // Read while the host lock is held and before a timed-out host is + // stopped, so the end values count this invocation and no later one. + let exchange_end = recorder + .as_ref() + .map(|recorder| recorder.read_exchange_end(ended_at)); + match exchange { Ok(output) => { + if let (Some(recorder), Some(end)) = (recorder.as_mut(), exchange_end) { + recorder.exchange_ended(end); + } let stderr = take_local_host_stderr(&mut host_guard); Ok(( output, @@ -1876,7 +1886,6 @@ fn local_host_invocation_result( )) } Err(error) => { - let failed_at = Instant::now(); if matches!( &error, FabricError::AdapterLifecycleOperation { code, .. } if code == HOST_TIMEOUT @@ -1884,12 +1893,18 @@ fn local_host_invocation_result( invalidate_timed_out_local_host(&runtime.runtime_id, &host, &mut host_guard); } // Only a record reads the exit status, so a run without one never waits. - if let Some(recorder) = recorder.as_mut() { + if let (Some(recorder), Some(end)) = (recorder.as_mut(), exchange_end) { let crashed = matches!( &error, FabricError::AdapterLifecycleOperation { code, .. } if code == HOST_CRASHED ); - recorder.exchange_failed(failed_at, local_host_exit(&mut host_guard, crashed)); + let end = end.with_host_exit(local_host_exit(&mut host_guard, crashed)); + let end = if crashed { + recorder.recount_oom_kills(end) + } else { + end + }; + recorder.exchange_ended(end); } Err(error) } diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 60eca0840..57ad9a1c6 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1043,7 +1043,11 @@ wait status. On Linux with cgroup v2, when NeMo Fabric runs at the root of its own container's cgroup, both lines record the container's `memory.events` `oom_kill` count as `container_oom_kills`. Otherwise the field reads -`{"reason": "platform" | "cgroup_v1" | "no_owned_cgroup"}`. The start line +`{"reason": "platform" | "cgroup_v1" | "no_owned_cgroup"}`. The end line's +count is read when the adapter host answers, crashes, or reaches the timeout, +before NeMo Fabric stops a hung host and before the next invocation on the +runtime can start. After a crash, NeMo Fabric reads the count again once the +host's exit is seen and keeps the larger value. The start line also records `exclusive_container`. Set `exclusive_container` in `runtime.record` to `True` only when nothing else runs in the container during an invocation, including other runtimes in the same process. Otherwise a kill From 51077a454b757a49815e40de4e59a639a4f0d7ad Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:52:51 -0700 Subject: [PATCH 031/159] test: pin no stop cause for a cancelled result with a timeout code The adapter deadline code counts as a timeout only in a failed result or a raised error, so a cancelled result that carries error code timeout has no stop cause: cancellation takes precedence. Add a contract test from a real local host for that case and state the exception in the stop rules. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 15 +++++++++++++++ docs/sdk/python.mdx | 5 ++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d9c2b1a92..95adae888 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -4053,6 +4053,12 @@ for line in sys.stdin: } if MODE == "cancelled": result = {"status": "cancelled", "output": output} + elif MODE == "cancelled_at_deadline": + result = { + "status": "cancelled", + "output": output, + "error": {"code": "timeout", "message": "deadline reached", "retryable": False}, + } elif MODE == "adapter_deadline": result = { "status": "failed", @@ -5998,6 +6004,15 @@ for line in sys.stdin: assert!(end.get("stop").is_none(), "{end}"); } + #[test] + fn stop_cancelled_result_with_a_timeout_error_code_has_no_stop() { + let (result, end) = result_end_line("cancelled_at_deadline"); + + assert_eq!(result.status, RunStatus::Cancelled); + assert_eq!(end["error"]["code"], "timeout"); + assert!(end.get("stop").is_none(), "{end}"); + } + #[test] fn stop_adapter_deadline_reads_an_unforced_time_limit() { let (_, end) = result_end_line("adapter_deadline"); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 57ad9a1c6..9ab2b38a6 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1012,7 +1012,10 @@ Harbor's failure causes: `sandbox_failed`, `agent_exhausted_budget`, and 2. `sandbox_failed`: the container's out-of-memory kill count rose during the invocation, and `runtime.record.exclusive_container` is `True`. 3. `agent_exhausted_budget`: the invocation reached its time limit. -4. A cancelled invocation that matched none of the above has no cause. +4. A cancelled invocation that matched none of the above has no cause. A + `cancelled` result with error code `timeout` also has no cause: the + adapter's deadline code counts as a timeout only in a `failed` result or a + raised error. 5. `unclassified`: the adapter host exited before it answered. 6. `agent_failed`: the adapter returned a `failed` result. 7. `unclassified`: anything else. From b13430a8c6431ccb4cc3df23ef1ad1dbe9b94236 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 06:52:52 -0700 Subject: [PATCH 032/159] docs: qualify forced stop exit status and the Harbor cause mapping The Windows exit code 1 and the Unix signal 9 appear only when Fabric terminates the host; a host that exits by itself during the 2-second grace period records its own exit status while forced stays true. The fabric_failed and agent_failed mapping is guidance for a Harbor consumer of the record; nothing in Fabric reads the record for Harbor. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 9ab2b38a6..5882c63ad 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -998,8 +998,9 @@ with stage `stop`, and the record still holds the invocation's own end line. The end line's `stop` says why the invocation stopped. It is absent on success. `stop.cause` comes from NeMo Fabric's own closed list. Three names are Harbor's failure causes: `sandbox_failed`, `agent_exhausted_budget`, and -`unclassified`. A Harbor runner maps `fabric_failed` to `harbor_failed` and -`agent_failed` to `unclassified`. The first rule that matches sets the cause: +`unclassified`. A Harbor consumer of the record can map `fabric_failed` to +`harbor_failed` and `agent_failed` to `unclassified`. The first rule that +matches sets the cause: 1. `fabric_failed`: NeMo Fabric or the adapter host raised an error that is not a timeout or a host exit. Examples are an invalid adapter result and an @@ -1032,9 +1033,10 @@ Two timeouts reach this rule: - NeMo Fabric's own timeout fired (error code `host_timeout`). `forced` is `true`, and `usage` is the time until the timeout fired. `elapsed_ms` also covers NeMo Fabric stopping the hung adapter host, so it is larger. - On Linux and macOS, `exit_signal` is then usually the signal NeMo Fabric - sent (9), unless the host exited by itself during the 2-second grace period. - On Windows, a forced stop gives `exit_code` 1 and no signal. + When NeMo Fabric terminates the host, `exit_signal` is the signal it sent + (9) on Linux and macOS, and on Windows `exit_code` is 1 with no signal. A + host that exits by itself during the 2-second grace period records its own + exit status instead, while `forced` stays `true`. - The adapter stopped at the deadline NeMo Fabric gave it and reported error code `timeout`, in a `failed` result or as an error. `forced` is `false`. That deadline is earlier than `value`: the timeout less 10 seconds, and at From 2fcc672192012db9e6d6c8e257eb22ee3131cdec Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:57:52 -0700 Subject: [PATCH 033/159] test: lower the exited-child CPU threshold to two clock ticks The exited-child snapshot test required 0.1 s of CPU from a shell loop of 100,000 iterations. On a fast Linux runner with dash and 10 ms clock ticks the loop can finish in less, so the test could fail with a correct reader. The host only sleeps, so 0.02 s (two ticks) still proves that the exited child's CPU time is counted, with the same workload. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 8548b799e..61755580b 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -834,9 +834,12 @@ mod tests { assert_eq!(running.process_count, 2, "child never seen: {running:?}"); assert_eq!(exited.process_count, 1, "child still running: {exited:?}"); + // The host only sleeps, so two 10 ms clock ticks can come only from + // the exited child. A fast shell can finish the loop in well under + // 100 ms. assert!( - exited.cpu_time_seconds >= 0.1, - "exited child's CPU missing: {exited:?}" + exited.cpu_time_seconds >= 0.02, + "exited child's CPU (at least two clock ticks) missing: {exited:?}" ); } From 5ef35935dcd73ae16bc498bc1f9eca0a7f4b2184 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:26:46 -0700 Subject: [PATCH 034/159] feat: add sample lines and resource snapshots to run records A run record's start and end lines say nothing about the work in between, so a reader cannot tell a long-running invocation from a quiet one, or see how much CPU and memory the invocation used. One process-wide thread, started with the first recorded invocation, appends a sample line every 10 seconds to each active record with the elapsed time and the runtime's resource snapshot. The thread holds only a weak reference to its ticker. A snapshot read that panics is written as unavailable. A forked child sees the parent's started thread but has none, so its start lines omit sample_interval_ms and its records get no sample lines. The start and end lines carry a snapshot too. Core reads the end snapshot under the host lock together with the end oom_kill count in ExchangeEnd, right after the adapter host answers or the timeout fires, and reuses it for RunResult.resource_snapshot. A snapshot that cannot be read is written as its error code. The record file closes at the end line. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/file.rs | 96 +++++ crates/fabric-core/src/record/invocation.rs | 404 +++++++++++++----- crates/fabric-core/src/record/lines.rs | 76 +++- crates/fabric-core/src/record/mod.rs | 14 +- crates/fabric-core/src/record/sampler.rs | 198 +++++++++ crates/fabric-core/src/record/writer.rs | 1 + crates/fabric-core/src/runtime.rs | 63 ++- docs/sdk/python.mdx | 42 +- schemas/sdk/run-record-line.schema.json | 138 +++++- .../references/results-and-errors.md | 27 +- 10 files changed, 933 insertions(+), 126 deletions(-) create mode 100644 crates/fabric-core/src/record/file.rs create mode 100644 crates/fabric-core/src/record/sampler.rs diff --git a/crates/fabric-core/src/record/file.rs b/crates/fabric-core/src/record/file.rs new file mode 100644 index 000000000..373d8908c --- /dev/null +++ b/crates/fabric-core/src/record/file.rs @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The open record file that the invocation and the sample thread share. + +use std::io; +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::SystemTime; + +use super::lines::{LineBody, RecordLine, RecordSchema, RecordVersion}; +use super::writer::RecordWriter; +use crate::time::rfc3339_millis; + +/// One invocation's record file. After the first failed write it stops. +#[derive(Debug)] +pub(crate) struct RecordFile { + pub runtime_id: String, + pub invocation_id: String, + pub request_id: String, + writer: Option, + /// The first create, write or sync failure. + pub failure: Option, + /// Set when the end line is written; no sample line follows it. + ended: bool, +} + +impl RecordFile { + /// Create the file at `path`. A failed create is kept in `failure`, and + /// every later append is skipped. + pub(crate) fn create( + path: &Path, + runtime_id: String, + invocation_id: String, + request_id: String, + ) -> Self { + let (writer, failure) = match RecordWriter::create(path) { + Ok(writer) => (Some(writer), None), + Err(error) => (None, Some(error)), + }; + Self { + runtime_id, + invocation_id, + request_id, + writer, + failure, + ended: false, + } + } + + /// Append one line. + pub(crate) fn append(&mut self, body: LineBody) { + let Some(writer) = self.writer.as_mut() else { + return; + }; + let line = RecordLine { + schema: RecordSchema::RunRecord, + version: RecordVersion::V1Alpha1, + runtime_id: self.runtime_id.clone(), + invocation_id: self.invocation_id.clone(), + request_id: self.request_id.clone(), + time: rfc3339_millis(SystemTime::now()), + body, + }; + if let Err(error) = writer.append(&line) { + self.failure = Some(error); + self.writer = None; + } + } + + /// Append a sample line, unless the end line is already written. + pub(crate) fn append_sample(&mut self, body: LineBody) { + if !self.ended { + self.append(body); + } + } + + /// Append the end line, sync the file and close it. No sample line + /// follows, and a sample tick that still holds this record keeps no open + /// file handle. + pub(crate) fn append_end(&mut self, body: LineBody) { + self.ended = true; + self.append(body); + if let Some(writer) = self.writer.take() + && let Err(error) = writer.sync() + { + self.failure = Some(error); + } + } +} + +/// Lock a shared record file. A panic elsewhere while it was held leaves the +/// file usable: every write is one append. +pub(crate) fn lock(file: &Mutex) -> MutexGuard<'_, RecordFile> { + file.lock().unwrap_or_else(|error| error.into_inner()) +} diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index d101e9466..042b332f1 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! One invocation's record: the start line before the request, the end line -//! after the invocation. +//! One invocation's record: the start line before the request, sample lines +//! from the process-wide ticker while it runs, and the end line after it. //! //! Record I/O errors do not replace the invocation result. A failed create, //! write or sync stops the record for that invocation and adds a @@ -10,28 +10,30 @@ //! error to the caller. use std::collections::BTreeMap; -use std::io; use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant, SystemTime}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; use serde_json::Value; use super::cgroup::CgroupReader; +use super::file::{RecordFile, lock}; use super::lines::{ ContextValue, EndArtifact, EndError, EndLine, EndUsage, LimitType, LineBody, ModelName, - NotRecorded, OomKills, RecordLine, RecordSchema, RecordVersion, StartLine, Stop, StopLimit, - StopReason, cut_text, recorded_code, + NotRecorded, OomKills, Resources, StartLine, Stop, StopLimit, StopReason, cut_text, + recorded_code, }; +use super::sampler::{ReadResources, Registration, SampleTicker}; use super::stop::{StopFacts, stop_cause}; -use super::writer::{RECORD_FILE_NAME, RecordWriter}; +use super::writer::RECORD_FILE_NAME; use crate::config::RunPlan; use crate::error::{FabricError, Result}; +use crate::resource_snapshot::RuntimeResourceSnapshot; use crate::runtime::{ ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, HOST_CRASHED, HOST_TIMEOUT, INVALID_AGENT_RUN_RESULT, InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, RuntimeHandle, event_with_metadata, unique_artifact_name, }; -use crate::time::rfc3339_millis; /// Name of the result artifact that points at the record. const RECORD_ARTIFACT_NAME: &str = "record"; @@ -71,6 +73,10 @@ pub(crate) struct RecordStart<'a> { pub timeout: Duration, /// Reads the container's out-of-memory kill count. pub cgroup: CgroupReader, + /// Reads the runtime's resource snapshot. + pub read_resources: ReadResources, + /// Appends the sample lines. + pub ticker: Arc, } /// The adapter host's wait status, when it exited. @@ -97,21 +103,17 @@ impl HostExit { } /// The record of one running invocation. -#[derive(Debug)] pub(crate) struct InvocationRecorder { - runtime_id: String, - invocation_id: String, - request_id: String, + file: Arc>, path: PathBuf, started: Instant, - writer: Option, wrote_start: bool, - failure: Option, timeout: Duration, cgroup: CgroupReader, oom_kills_at_start: OomKills, exclusive_container: bool, exchange: Option, + sampling: Option, } /// What core saw when the exchange with the adapter host ended. Core reads @@ -122,6 +124,8 @@ pub(crate) struct ExchangeEnd { /// Time from the start line to the end of the exchange, before core's /// own cleanup. elapsed_ms: u64, + /// The runtime's resource snapshot at that moment. + resources: Resources, /// The container's out-of-memory kill count at that moment, or the /// larger count read after a crashed host's exit. oom_kills: OomKills, @@ -143,42 +147,58 @@ impl ExchangeEnd { } impl InvocationRecorder { - /// Create the record and write its start line. + /// Create the record, write its start line, and register it with the + /// sample ticker. pub(crate) fn start(start: RecordStart<'_>) -> Self { let path = start.fabric_home.join(RECORD_FILE_NAME); + let started = Instant::now(); let oom_kills_at_start = start.cgroup.oom_kills(); - let mut recorder = Self { - runtime_id: start.invocation.runtime_id.clone(), - invocation_id: start.invocation.invocation_id.clone(), - request_id: start.invocation.request_id.clone(), - started: Instant::now(), - writer: None, - wrote_start: false, - failure: None, + let mut file = RecordFile::create( + &path, + start.invocation.runtime_id.clone(), + start.invocation.invocation_id.clone(), + start.invocation.request_id.clone(), + ); + let sample_interval = start.ticker.start(); + file.append(LineBody::Start(start_line( + &start, + sample_interval, + oom_kills_at_start, + ))); + let wrote_start = file.failure.is_none(); + let file = Arc::new(Mutex::new(file)); + let sampling = (wrote_start && sample_interval.is_some()).then(|| { + start.ticker.register( + Arc::clone(&file), + started, + Arc::clone(&start.read_resources), + ) + }); + Self { + file, path, + started, + wrote_start, timeout: start.timeout, cgroup: start.cgroup.clone(), oom_kills_at_start, exclusive_container: exclusive_container(start.plan), exchange: None, - }; - match RecordWriter::create(&recorder.path) { - Ok(writer) => { - recorder.writer = Some(writer); - recorder.append(LineBody::Start(start_line(&start, oom_kills_at_start))); - recorder.wrote_start = recorder.failure.is_none(); - } - Err(error) => recorder.failure = Some(error), + sampling, } - recorder } /// The end values at `ended_at`, when the exchange with the host - /// returned: the container's out-of-memory kill count, read now. Core - /// calls this under the host lock. - pub(crate) fn read_exchange_end(&self, ended_at: Instant) -> ExchangeEnd { + /// returned: `snapshot` and the container's out-of-memory kill count, + /// read now. Core calls this under the host lock. + pub(crate) fn read_exchange_end( + &self, + ended_at: Instant, + snapshot: &Result, + ) -> ExchangeEnd { ExchangeEnd { elapsed_ms: millis(ended_at.saturating_duration_since(self.started)), + resources: Resources::from(snapshot), oom_kills: self.cgroup.oom_kills(), failed: false, host_exit: None, @@ -206,23 +226,29 @@ impl InvocationRecorder { /// gains the `record` artifact when the start line was written, and a /// `record_write_failed` event when any write or the sync failed. pub(crate) fn finish(mut self, outcome: Result) -> Result { + self.sampling = None; let elapsed_ms = millis(self.started.elapsed()); // Core always records the exchange's end; reading here is a fallback. - let oom_kills = self - .exchange - .as_ref() - .map_or_else(|| self.cgroup.oom_kills(), |exchange| exchange.oom_kills); + let (resources, oom_kills) = match self.exchange.as_ref() { + Some(exchange) => (exchange.resources.clone(), exchange.oom_kills), + None => (Resources::UNAVAILABLE, self.cgroup.oom_kills()), + }; let stop = self.stop(&outcome, elapsed_ms, oom_kills); + let end = EndValues { + elapsed_ms, + resources, + oom_kills, + stop, + }; let end = match &outcome { - Ok(result) => result_end_line(result, elapsed_ms, oom_kills, stop), - Err(error) => error_end_line(error, elapsed_ms, oom_kills, stop), + Ok(result) => result_end_line(result, end), + Err(error) => error_end_line(error, end), }; - self.append(LineBody::End(end)); - if let Some(writer) = &self.writer - && let Err(error) = writer.sync() - { - self.failure = Some(error); - } + let mut file = lock(&self.file); + file.append_end(LineBody::End(end)); + let failure = file.failure.take(); + let (runtime_id, invocation_id) = (file.runtime_id.clone(), file.invocation_id.clone()); + drop(file); let mut result = outcome?; if self.wrote_start { let name = unique_artifact_name(&result.artifacts, RECORD_ARTIFACT_NAME); @@ -234,19 +260,13 @@ impl InvocationRecorder { metadata: BTreeMap::new(), }); } - if let Some(error) = &self.failure { + if let Some(error) = failure { result.events.push(event_with_metadata( RECORD_WRITE_FAILED, format!("run record write failed; the record stops here: {error}"), BTreeMap::from([ - ( - "runtime_id".to_string(), - Value::from(self.runtime_id.as_str()), - ), - ( - "invocation_id".to_string(), - Value::from(self.invocation_id.as_str()), - ), + ("runtime_id".to_string(), Value::from(runtime_id)), + ("invocation_id".to_string(), Value::from(invocation_id)), ]), )); } @@ -326,29 +346,21 @@ impl InvocationRecorder { }; (stop != Stop::default()).then_some(stop) } +} - /// Append one line. After the first failure the record stops. - fn append(&mut self, body: LineBody) { - let Some(writer) = self.writer.as_mut() else { - return; - }; - let line = RecordLine { - schema: RecordSchema::RunRecord, - version: RecordVersion::V1Alpha1, - runtime_id: self.runtime_id.clone(), - invocation_id: self.invocation_id.clone(), - request_id: self.request_id.clone(), - time: rfc3339_millis(SystemTime::now()), - body, - }; - if let Err(error) = writer.append(&line) { - self.failure = Some(error); - self.writer = None; - } - } +/// The end line's values that do not come from the outcome. +struct EndValues { + elapsed_ms: u64, + resources: Resources, + oom_kills: OomKills, + stop: Option, } -fn start_line(start: &RecordStart<'_>, container_oom_kills: OomKills) -> StartLine { +fn start_line( + start: &RecordStart<'_>, + sample_interval: Option, + container_oom_kills: OomKills, +) -> StartLine { StartLine { agent_name: cut_text(&start.runtime.agent_name), harness: cut_text(&start.runtime.harness), @@ -370,6 +382,8 @@ fn start_line(start: &RecordStart<'_>, container_oom_kills: OomKills) -> StartLi .collect(), fabric_version: crate::version().to_string(), timeout_ms: millis(start.timeout), + sample_interval_ms: sample_interval.map(millis), + resources: Resources::from(&(start.read_resources)()), context: start .request .context @@ -421,29 +435,20 @@ fn context_value(key: &str, value: &Value) -> ContextValue { } } -fn result_end_line( - result: &RunResult, - elapsed_ms: u64, - container_oom_kills: OomKills, - stop: Option, -) -> EndLine { +fn result_end_line(result: &RunResult, end: EndValues) -> EndLine { EndLine { status: result.status, - elapsed_ms, + elapsed_ms: end.elapsed_ms, error: result.error.as_ref().map(result_error), usage: result.usage.as_ref().map(end_usage), artifacts: end_artifacts(&result.artifacts), - container_oom_kills, - stop, + resources: end.resources, + container_oom_kills: end.oom_kills, + stop: end.stop, } } -fn error_end_line( - error: &FabricError, - elapsed_ms: u64, - container_oom_kills: OomKills, - stop: Option, -) -> EndLine { +fn error_end_line(error: &FabricError, end: EndValues) -> EndLine { let (code, stage) = match error { FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), // After the record starts, only the check of the adapter's result @@ -454,7 +459,7 @@ fn error_end_line( }; EndLine { status: RunStatus::Failed, - elapsed_ms, + elapsed_ms: end.elapsed_ms, error: Some(EndError { code: recorded_code(code), stage, @@ -462,8 +467,9 @@ fn error_end_line( }), usage: None, artifacts: Vec::new(), - container_oom_kills, - stop, + resources: end.resources, + container_oom_kills: end.oom_kills, + stop: end.stop, } } @@ -507,8 +513,25 @@ fn end_artifacts(manifest: &ArtifactManifest) -> Vec { mod tests { use super::*; use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; + use crate::record::lines::{RecordLine, RecordSchema, RecordVersion}; use crate::record::test_hooks::{self, Fault}; + fn snapshot() -> RuntimeResourceSnapshot { + RuntimeResourceSnapshot { + runtime_id: "rt".to_string(), + started_at: "2026-10-14T18:01:10.205Z".to_string(), + observed_at: "2026-10-14T18:01:11.205Z".to_string(), + resident_memory_bytes: 4096, + process_count: 2, + cpu_time_seconds: 0.5, + } + } + + /// A ticker whose thread never ticks during a test. + fn idle_ticker() -> Arc { + SampleTicker::new(Duration::from_secs(3600)) + } + fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { let config: FabricConfig = serde_json::from_value(serde_json::json!({ "schema_version": "fabric.agent/v1alpha1", @@ -559,6 +582,8 @@ mod tests { fabric_home: Path::new("/unused"), timeout: Duration::from_secs(5), cgroup: CgroupReader::system(), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }; let line = serde_json::to_value(RecordLine { @@ -568,7 +593,11 @@ mod tests { invocation_id: "inv".to_string(), request_id: "req".to_string(), time: "2026-10-14T18:01:10.205Z".to_string(), - body: LineBody::Start(start_line(&start, OomKills::Count(0))), + body: LineBody::Start(start_line( + &start, + Some(Duration::from_secs(10)), + OomKills::Count(0), + )), }) .expect("start line"); @@ -594,6 +623,8 @@ mod tests { 256 ); assert_eq!(line["timeout_ms"], 5_000); + assert_eq!(line["sample_interval_ms"], 10_000); + assert_eq!(line["resources"]["process_count"], 2); assert_eq!( line["models"], serde_json::json!({"default": {"provider": "nvidia", "model": "nemotron"}}) @@ -628,9 +659,11 @@ mod tests { fabric_home: Path::new("/unused"), timeout: Duration::from_secs(5), cgroup: CgroupReader::system(), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }; - let line = start_line(&start, OomKills::Count(0)); + let line = start_line(&start, None, OomKills::Count(0)); assert_eq!(line.models.len(), 3); assert_eq!(line.context.len(), 2); @@ -670,6 +703,8 @@ mod tests { fabric_home: &folder, timeout: Duration::from_secs(5), cgroup: CgroupReader::system(), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }); let mut result = succeeded_result(&invocation); result.artifacts.artifacts.push(ArtifactRef { @@ -707,6 +742,8 @@ mod tests { fabric_home: &folder, timeout: Duration::from_secs(5), cgroup: CgroupReader::system(), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }); let result = succeeded_result(&invocation); @@ -756,6 +793,8 @@ mod tests { fabric_home: &folder, timeout: Duration::from_secs(5), cgroup: CgroupReader::system(), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }); let finished = recorder.finish(outcome); test_hooks::FAULT.with(|slot| slot.set(None)); @@ -896,6 +935,8 @@ mod tests { fabric_home: &folder, timeout: Duration::from_secs(5), cgroup: CgroupReader::with_root(&root), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }); let count_the_kill = || std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)); @@ -903,7 +944,7 @@ mod tests { count_the_kill().expect("oom kill"); } let mut end = recorder - .read_exchange_end(Instant::now()) + .read_exchange_end(Instant::now(), &Ok(snapshot())) .with_host_exit(Some(HostExit { exit_code: None, exit_signal: Some(9), @@ -1011,8 +1052,10 @@ mod tests { fabric_home: &folder, timeout: Duration::from_secs(5), cgroup: CgroupReader::with_root(&root), + read_resources: Arc::new(|| Ok(snapshot())), + ticker: idle_ticker(), }); - recorder.exchange_ended(recorder.read_exchange_end(Instant::now())); + recorder.exchange_ended(recorder.read_exchange_end(Instant::now(), &Ok(snapshot()))); // Core released the host lock, so a later invocation on the same // runtime can be killed before this result is finished. std::fs::write(root.join("sys/fs/cgroup/memory.events"), memory_events(5)) @@ -1029,4 +1072,169 @@ mod tests { assert_eq!(end["container_oom_kills"], 4, "{end}"); assert_eq!(end["stop"], serde_json::json!({"cause": "agent_failed"})); } + + fn record_line_validator() -> jsonschema::Validator { + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunRecordLine) + .expect("record line schema"); + jsonschema::validator_for(&schema).expect("valid schema") + } + + /// Start a record in a fresh folder named `name`, sampled by `ticker`. + fn start_recorder( + name: &str, + ticker: &Arc, + read: ReadResources, + ) -> (PathBuf, InvocationRecorder) { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder(name); + let request = RunRequest::text("input"); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + cgroup: CgroupReader::system(), + read_resources: read, + ticker: Arc::clone(ticker), + }); + (folder, recorder) + } + + fn read_lines(path: &Path) -> Vec { + std::fs::read_to_string(path) + .expect("record") + .lines() + .map(|line| serde_json::from_str(line).expect("json line")) + .collect() + } + + fn samples(path: &Path) -> Vec { + read_lines(path) + .into_iter() + .filter(|line| line["kind"] == "sample") + .collect() + } + + fn wait_for_samples(path: &Path, count: usize) { + // Wait for lines, not a fixed time, so a loaded runner cannot flake it. + let deadline = Instant::now() + Duration::from_secs(10); + while samples(path).len() < count && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + samples(path).len() >= count, + "fewer than {count} sample lines in {}", + path.display() + ); + } + + #[test] + fn two_records_share_one_ticker_and_samples_stop_at_the_end_line() { + let ticker = SampleTicker::new(Duration::from_millis(20)); + let (first_folder, mut first) = + start_recorder("share-1", &ticker, Arc::new(|| Ok(snapshot()))); + let (second_folder, second) = + start_recorder("share-2", &ticker, Arc::new(|| Ok(snapshot()))); + let first_path = first_folder.join(RECORD_FILE_NAME); + let second_path = second_folder.join(RECORD_FILE_NAME); + + wait_for_samples(&first_path, 2); + wait_for_samples(&second_path, 2); + let end = first + .read_exchange_end(Instant::now(), &Ok(snapshot())) + .with_host_exit(None); + first.exchange_ended(end); + let error = FabricError::AdapterLifecycleOperation { + operation: "invoke", + runtime_id: "rt".to_string(), + code: HOST_CRASHED.to_string(), + message: "host exited".to_string(), + diagnostics: String::new(), + }; + first.finish(Err(error)).expect_err("error passes through"); + std::thread::sleep(Duration::from_millis(60)); + + let lines = read_lines(&first_path); + let validator = record_line_validator(); + for line in lines.iter().chain(&read_lines(&second_path)) { + assert!(validator.is_valid(line), "{line}"); + } + assert_eq!(lines.last().expect("end")["kind"], "end"); + assert_eq!(lines.last().expect("end")["resources"]["process_count"], 2); + let samples = samples(&first_path); + assert!(samples.len() >= 2, "{lines:?}"); + assert!(samples[0]["elapsed_ms"].is_u64()); + assert_eq!(samples[0]["resources"]["process_count"], 2); + assert_eq!(lines[0]["sample_interval_ms"], 20); + drop(second); + let _ = std::fs::remove_dir_all(first_folder); + let _ = std::fs::remove_dir_all(second_folder); + } + + #[test] + fn sample_line_with_a_snapshot_error_keeps_only_the_error_code() { + let ticker = SampleTicker::new(Duration::from_millis(20)); + let (folder, recorder) = start_recorder( + "snapshot-error", + &ticker, + Arc::new(|| { + Err(FabricError::ResourceObservationUnavailable { + runtime_id: "rt".to_string(), + reason: "adapter host process has exited".to_string(), + }) + }), + ); + let path = folder.join(RECORD_FILE_NAME); + + wait_for_samples(&path, 1); + drop(recorder); + + let lines = read_lines(&path); + let unavailable = serde_json::json!({"error": "resource_observation_unavailable"}); + assert_eq!(samples(&path)[0]["resources"], unavailable); + assert_eq!(lines[0]["resources"], unavailable); + assert!( + !std::fs::read_to_string(&path) + .expect("record") + .contains("exited") + ); + let _ = std::fs::remove_dir_all(folder); + } + + #[test] + fn a_panicking_snapshot_read_still_writes_sample_lines_without_the_panic_text() { + let ticker = SampleTicker::new(Duration::from_millis(20)); + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let reads = Arc::clone(&calls); + let (folder, recorder) = start_recorder( + "panicking-read", + &ticker, + Arc::new(move || { + // The start line's read succeeds; every sample read panics. + if reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed) > 0 { + panic!("snapshot reader bug"); + } + Ok(snapshot()) + }), + ); + let path = folder.join(RECORD_FILE_NAME); + + wait_for_samples(&path, 2); + drop(recorder); + + let samples = samples(&path); + assert!(samples.len() >= 2, "{samples:?}"); + assert_eq!( + samples[1]["resources"], + serde_json::json!({"error": "resource_observation_unavailable"}) + ); + assert!( + !std::fs::read_to_string(&path) + .expect("record") + .contains("reader bug") + ); + let _ = std::fs::remove_dir_all(folder); + } } diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index 3e02f1789..405d46f15 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -13,8 +13,12 @@ use std::collections::BTreeMap; use schemars::JsonSchema; use serde::Serialize; +use crate::error::{FabricError, Result}; +use crate::resource_snapshot::RuntimeResourceSnapshot; use crate::runtime::{ErrorStage, RunStatus}; +/// Interval between sample lines while an invocation runs. +pub(crate) const SAMPLE_INTERVAL_MS: u64 = 10_000; /// Longest value a line keeps from config or the request, in characters. /// Model role and context key names are kept whole, so map keys stay distinct. const MAX_VALUE_CHARS: usize = 256; @@ -67,6 +71,9 @@ pub(crate) enum RecordVersion { pub(crate) enum LineBody { /// Written once, before core sends the request to the adapter host. Start(StartLine), + /// Written about every 10 s while the invocation runs, when the start + /// line has `sample_interval_ms`. + Sample(SampleLine), /// Written once, after the invocation returned a result or failed. End(EndLine), } @@ -88,6 +95,12 @@ pub(crate) struct StartLine { pub fabric_version: String, /// Host invocation timeout that applied, in milliseconds. pub timeout_ms: u64, + /// Interval between sample lines, in milliseconds. Absent when this + /// process has no sample thread, so the record gets no sample lines. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sample_interval_ms: Option, + /// The runtime's process values when the invocation started. + pub resources: Resources, /// `RunRequest.context`. Allowlisted identity and Harbor link keys keep a /// string or integer value; every other key is listed by name only. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] @@ -152,13 +165,74 @@ pub(crate) struct EndLine { /// Result artifacts, with paths relative to the artifact root. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub artifacts: Vec, - /// The container's cgroup v2 `oom_kill` count when the invocation ended. + /// The runtime's process values when the invocation ended, read under + /// the runtime's host lock right after the adapter host answered or the + /// timeout fired. + pub resources: Resources, + /// The container's cgroup v2 `oom_kill` count when the invocation ended, + /// read with `resources`. pub container_oom_kills: OomKills, /// Why the invocation stopped; absent when it holds nothing, as on success. #[serde(default, skip_serializing_if = "Option::is_none")] pub stop: Option, } +/// Fields of a sample line. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub(crate) struct SampleLine { + /// Monotonic time since the start line, in milliseconds. + pub elapsed_ms: u64, + /// The runtime's process values now. + pub resources: Resources, +} + +/// A runtime resource snapshot, or the error code that explains why none +/// could be read. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +#[serde(untagged)] +pub(crate) enum Resources { + /// The snapshot. + Snapshot(RuntimeResourceSnapshot), + /// No snapshot could be read. + Error { + /// Why no snapshot could be read. + error: ResourcesError, + }, +} + +impl Resources { + /// No snapshot, for a read that did not happen or did not finish. + pub(crate) const UNAVAILABLE: Self = Self::Error { + error: ResourcesError::ResourceObservationUnavailable, + }; +} + +impl From<&Result> for Resources { + /// Keep the snapshot, or only the error's code: never its message. + fn from(read: &Result) -> Self { + match read { + Ok(snapshot) => Self::Snapshot(snapshot.clone()), + Err(FabricError::ResourceObservationUnsupported { .. }) => Self::Error { + error: ResourcesError::ResourceObservationUnsupported, + }, + Err(_) => Self::UNAVAILABLE, + } + } +} + +/// Error codes for a snapshot that could not be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ResourcesError { + /// The runtime stopped or is stopping, its adapter host exited, a live + /// process in its tree or the process table cannot be read, or the + /// process tree kept changing during the read. + ResourceObservationUnavailable, + /// The adapter kind, environment provider, or operating system has no + /// resource reader. + ResourceObservationUnsupported, +} + /// The cgroup v2 `memory.events` `oom_kill` count of the container Fabric /// runs in, or why it could not be read. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 571584a4a..d02e45414 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -4,23 +4,27 @@ //! The opt-in per-invocation run record, `run-record.jsonl`. //! //! When `runtime.record.enabled` is set, core appends one start line before it -//! sends the request to the adapter host and one end line after the invocation -//! returns or fails. Recording is synchronous and best-effort: record I/O -//! errors do not replace the invocation result. A failed write stops the -//! record and becomes a `record_write_failed` event on the result, and slow -//! storage delays the invocation. +//! sends the request to the adapter host, a sample line about every 10 seconds +//! while the invocation runs, and one end line after it returns or fails. +//! Recording is synchronous and best-effort: record I/O errors do not replace +//! the invocation result. A failed write stops the record and becomes a +//! `record_write_failed` event on the result, and slow storage delays the +//! invocation. //! //! Everything here is crate-private. The line format is public only through //! the `run-record-line` JSON Schema, version `v1alpha1`. mod cgroup; +mod file; mod invocation; mod lines; +mod sampler; mod stop; mod writer; pub(crate) use cgroup::CgroupReader; pub(crate) use invocation::{HostExit, InvocationRecorder, RecordStart}; pub(crate) use lines::RecordLine; +pub(crate) use sampler::SampleTicker; #[cfg(test)] pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/record/sampler.rs b/crates/fabric-core/src/record/sampler.rs new file mode 100644 index 000000000..6c257a60c --- /dev/null +++ b/crates/fabric-core/src/record/sampler.rs @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One process-wide thread that appends a sample line to every active record. +//! +//! The thread starts with the first recorded invocation and serves every +//! runtime in the process, so N parallel runtimes cost one wake-up per +//! interval, not N threads. Each tick reads each record's runtime snapshot, +//! which never waits for the runtime's host lock. + +use std::collections::BTreeMap; +use std::panic::{self, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, LazyLock, Mutex, MutexGuard, Once, Weak}; +use std::thread; +use std::time::{Duration, Instant}; + +use super::file::{RecordFile, lock}; +use super::lines::{LineBody, Resources, SAMPLE_INTERVAL_MS, SampleLine}; +use crate::error::Result; +use crate::resource_snapshot::RuntimeResourceSnapshot; + +/// Reads a runtime's resource snapshot without waiting for its host lock. +pub(crate) type ReadResources = Arc Result + Send + Sync>; + +static GLOBAL: LazyLock> = + LazyLock::new(|| SampleTicker::new(Duration::from_millis(SAMPLE_INTERVAL_MS))); + +/// One active record the ticker samples. +struct ActiveRecord { + file: Arc>, + started: Instant, + read: ReadResources, +} + +impl ActiveRecord { + /// Read the snapshot, then append the sample line. The read runs without + /// the file lock, so a slow read never delays the end line. A read that + /// panics is written as unavailable, so the record keeps its heartbeat. + fn sample(&self) { + let resources = panic::catch_unwind(AssertUnwindSafe(|| Resources::from(&(self.read)()))) + .unwrap_or(Resources::UNAVAILABLE); + let sample = SampleLine { + elapsed_ms: u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX), + resources, + }; + lock(&self.file).append_sample(LineBody::Sample(sample)); + } +} + +/// Appends sample lines to active records at a fixed interval. +pub(crate) struct SampleTicker { + interval: Duration, + records: Mutex>>, + next_id: AtomicU64, + thread_started: Once, + running: AtomicBool, + /// The process that spawned the thread. A forked child inherits + /// `running` but not the thread. + spawned_by: AtomicU32, +} + +impl SampleTicker { + /// A ticker with its own interval; [`Self::start`] starts its thread. + pub(crate) fn new(interval: Duration) -> Arc { + Arc::new(Self { + interval, + records: Mutex::new(BTreeMap::new()), + next_id: AtomicU64::new(0), + thread_started: Once::new(), + running: AtomicBool::new(false), + spawned_by: AtomicU32::new(0), + }) + } + + /// The process-wide ticker, sampling every [`SAMPLE_INTERVAL_MS`]. + pub(crate) fn global() -> Arc { + GLOBAL.clone() + } + + /// Start the thread once, and return the sample interval when it runs + /// in this process. `None` means the operating system refused the thread, + /// or this process is a forked child of the one that started it, so + /// records get no sample lines. + pub(crate) fn start(self: &Arc) -> Option { + self.thread_started.call_once(|| { + // The thread holds a weak reference and ends when the ticker is + // dropped. The process-wide ticker is never dropped. + let ticker = Arc::downgrade(self); + let interval = self.interval; + let spawned = thread::Builder::new() + .name("fabric-record-samples".to_string()) + .spawn(move || run(&ticker, interval)); + self.spawned_by.store(std::process::id(), Ordering::Relaxed); + self.running.store(spawned.is_ok(), Ordering::Release); + }); + self.interval_in(std::process::id()) + } + + /// The sample interval when the thread runs in process `pid`. + fn interval_in(&self, pid: u32) -> Option { + (self.running.load(Ordering::Acquire) && self.spawned_by.load(Ordering::Relaxed) == pid) + .then_some(self.interval) + } + + /// Sample `file` until the returned registration is dropped. + pub(crate) fn register( + self: &Arc, + file: Arc>, + started: Instant, + read: ReadResources, + ) -> Registration { + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + self.lock_records().insert( + id, + Arc::new(ActiveRecord { + file, + started, + read, + }), + ); + Registration { + ticker: Arc::clone(self), + id, + } + } + + /// Append one sample line to every active record. + fn tick(&self) { + let active: Vec> = self.lock_records().values().cloned().collect(); + for record in active { + record.sample(); + } + } + + fn lock_records(&self) -> MutexGuard<'_, BTreeMap>> { + self.records + .lock() + .unwrap_or_else(|error| error.into_inner()) + } +} + +/// The ticker thread: tick on a fixed schedule until the ticker is gone. +/// Ticks are scheduled from the first deadline, so the time a tick takes +/// does not add up across ticks. +fn run(ticker: &Weak, interval: Duration) { + let mut next = Instant::now() + interval; + loop { + thread::sleep(next.saturating_duration_since(Instant::now())); + let Some(ticker) = ticker.upgrade() else { + return; + }; + ticker.tick(); + drop(ticker); + next += interval; + let now = Instant::now(); + if next < now { + // A tick longer than the interval skips the missed ticks. + next = now + interval; + } + } +} + +/// Keeps a record in the ticker; dropping it stops sampling that record. +pub(crate) struct Registration { + ticker: Arc, + id: u64, +} + +impl Drop for Registration { + fn drop(&mut self) { + self.ticker.lock_records().remove(&self.id); + } +} + +impl std::fmt::Debug for Registration { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("Registration") + .field("id", &self.id) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn start_returns_no_interval_in_a_process_that_did_not_spawn_the_thread() { + let ticker = SampleTicker::new(Duration::from_secs(3600)); + + assert_eq!(ticker.start(), Some(Duration::from_secs(3600))); + // A forked child keeps the parent's memory, so it sees a started + // thread, but it has another PID and no thread. + assert_eq!(ticker.interval_in(std::process::id().wrapping_add(1)), None); + } +} diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs index ea95049cf..4ac4f3bbd 100644 --- a/crates/fabric-core/src/record/writer.rs +++ b/crates/fabric-core/src/record/writer.rs @@ -86,6 +86,7 @@ pub(crate) mod test_hooks { match body { LineBody::Start(_) => fail_if(Fault::StartAppend), LineBody::End(_) => fail_if(Fault::EndAppend), + LineBody::Sample(_) => Ok(()), } } diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d6520830e..a4fba6bef 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -31,7 +31,7 @@ use crate::config::{ validate_tool_definitions, validate_workflow, }; use crate::error::{FabricError, Result}; -use crate::record::{CgroupReader, HostExit, InvocationRecorder, RecordStart}; +use crate::record::{CgroupReader, HostExit, InvocationRecorder, RecordStart, SampleTicker}; use crate::resource_snapshot::{LocalHostProcess, RuntimeResourceSnapshot, snapshot_local_host}; static NEXT_ID: AtomicU64 = AtomicU64::new(1); @@ -1876,6 +1876,11 @@ fn local_host_invocation_result( fabric_home: &fabric_home, timeout: invoke_timeout, cgroup: CgroupReader::system(), + read_resources: { + let runtime = runtime.clone(); + Arc::new(move || local_runtime_resource_snapshot(&runtime)) + }, + ticker: SampleTicker::global(), })); } adapter_invocation.runtime_context.deadline_millis = @@ -1921,17 +1926,19 @@ fn local_host_invocation_result( let ended_at = Instant::now(); // Read while the host lock is held and before a timed-out host is // stopped, so the end values count this invocation and no later one. + let end_snapshot = (exchange.is_ok() || recorder.is_some()) + .then(|| local_runtime_resource_snapshot(runtime)); let exchange_end = recorder .as_ref() - .map(|recorder| recorder.read_exchange_end(ended_at)); + .zip(end_snapshot.as_ref()) + .map(|(recorder, snapshot)| recorder.read_exchange_end(ended_at, snapshot)); match exchange { Ok(output) => { if let (Some(recorder), Some(end)) = (recorder.as_mut(), exchange_end) { recorder.exchange_ended(end); } let stderr = take_local_host_stderr(&mut host_guard); - // Read while the host lock is held, so no later invocation is counted. - let resource_snapshot = local_runtime_resource_snapshot(runtime).ok(); + let resource_snapshot = end_snapshot.and_then(Result::ok); Ok(( output, resource_snapshot, @@ -1951,7 +1958,8 @@ fn local_host_invocation_result( ) { invalidate_timed_out_local_host(&runtime.runtime_id, &host, &mut host_guard); } - // Only a record reads the exit status, so a run without one never waits. + // Only a record reads the exit status, so a run without one + // never waits. After a forced stop it is the signal core sent. if let (Some(recorder), Some(end)) = (recorder.as_mut(), exchange_end) { let crashed = matches!( &error, @@ -6291,4 +6299,49 @@ for line in sys.stdin: assert_eq!(stop["limit"]["value"], 3_600_000); assert!(stop["limit"]["usage"].is_u64(), "{stop}"); } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn record_start_and_end_lines_carry_the_runtime_snapshot() { + let (root, plan) = record_plan("success"); + + let result = run_plan(&plan, RunRequest::text("snapshot")).expect("run"); + let (_, lines) = only_record(&root); + + let (start, end) = (line_of_kind(&lines, "start"), line_of_kind(&lines, "end")); + assert_eq!(start["sample_interval_ms"], 10_000); + for line in [start, end] { + assert_eq!(line["resources"]["runtime_id"], result.runtime_id.as_str()); + assert!(line["resources"]["process_count"].as_u64().expect("count") >= 1); + } + assert_eq!( + end["resources"], + serde_json::to_value(result.resource_snapshot.expect("result snapshot")) + .expect("snapshot JSON") + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn record_end_line_of_a_timed_out_invocation_keeps_the_hung_host_snapshot() { + let (root, plan) = record_plan("invoke_timeout"); + let runtime = start_runtime(&plan).expect("start local host"); + + run_local_host_adapter_with_timeout( + &plan, + &runtime, + RunRequest::text("hang"), + Duration::from_millis(100), + ) + .expect_err("timeout raises"); + let (_, lines) = only_record(&root); + + let end = line_of_kind(&lines, "end"); + assert_eq!(end["error"]["code"], "host_timeout"); + assert_eq!(end["resources"]["runtime_id"], runtime.runtime_id.as_str()); + assert!(end["resources"]["process_count"].as_u64().expect("count") >= 1); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } } diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 016802bab..72000f93e 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1033,18 +1033,22 @@ result lists it as the `record` artifact with kind `run_record`: record = next(a for a in result.artifacts.artifacts if a.kind == "run_record") ``` -The file holds a start line and an end line. Every line carries `schema` +The file holds a start line, a sample line about every 10 seconds while the +invocation runs when the start line has `sample_interval_ms`, and an end +line. Every line carries `schema` (`nemo.fabric.run_record`), `version` (`v1alpha1`), `kind`, `runtime_id`, `invocation_id`, `request_id`, and `time`. - The `start` line is written before the request is sent. It holds the agent name, harness, adapter ID, each configured model's provider and model name, - the NeMo Fabric version, and the timeout in milliseconds. From + the NeMo Fabric version, the timeout and the sample interval in + milliseconds, and the runtime's resource snapshot. From `RunRequest.context`, it keeps the string or integer values of `task_id`, `task_revision`, `attempt`, `step`, `harbor_session_id`, `harbor_context_id`, and `harbor_trial_name`, cut to 256 characters. Every other context key appears by name only as `{"recorded": false}`. `override_keys` lists the top-level `RunRequest.overrides` key names, sorted, without their values. +- A `sample` line holds `elapsed_ms` and the runtime's resource snapshot. - The `end` line is written after the invocation returns or fails. It holds `status`, `elapsed_ms`, the error's `code`, `stage`, and `retryable` flag, the numeric `RunUsage` counters, and each result artifact's name, kind, @@ -1055,7 +1059,28 @@ The file holds a start line and an end line. Every line carries `schema` `host_io`, or `invalid_agent_run_result`. A result extension that the adapter's descriptor does not declare also reads `invalid_agent_run_result`. A failed write of `adapter-invocation.json` or a result artifact reads - `artifact_write_failed`, and any other error reads `fabric_error`. + `artifact_write_failed`, and any other error reads `fabric_error`. The end + line also holds the runtime's snapshot when the invocation ended, read right + after the adapter host answered or the timeout fired. + +The start, sample, and end lines store the snapshot as `resources`, with the +same fields as `RuntimeResourceSnapshot` (see +[Runtime Resource Snapshots](#runtime-resource-snapshots)). When no snapshot +can be read, `resources` is +`{"error": "resource_observation_unavailable"}` or +`{"error": "resource_observation_unsupported"}`. CPU time is cumulative since +the adapter host started. When both the start and end lines hold a snapshot, +the invocation's CPU time is the end line's `cpu_time_seconds` minus the start +line's. The difference leaves out descendants that left the process tree. + +One NeMo Fabric thread writes the sample lines for every recorded invocation +in the process. When the operating system refuses that thread, the start line +has no `sample_interval_ms` and the record gets no sample lines. A process +that forks after its first recorded invocation, such as Python +`multiprocessing` with the `fork` start method, has no sample thread in the +child, so in a forked child the start line has no `sample_interval_ms`. Use +the `spawn` or `forkserver` start method with recording; `fork` after threads +exist is unsafe. The end line describes the invocation. A one-shot `Fabric.run` stops the runtime after the invocation; when that stop fails, the result reads `failed` @@ -1135,9 +1160,14 @@ writes for every invocation whether or not recording is on; those files are not covered by this rule. Each line is written with one append, so a killed process keeps every line -whose write returned. NeMo Fabric syncs the file once, after the end line. A -record without an end line belongs to an invocation that is still running or -that never finished; treat its outcome as unknown, not as failed. +whose write returned. NeMo Fabric syncs the file once, after the end line. +When the start line has `sample_interval_ms`, a record without an end line +whose file changed within three sample intervals (30 seconds at the default) +belongs to a running invocation, and one that has not changed for longer +never finished, or its record stopped on a failed write (the result's +`record_write_failed` event says which). Without `sample_interval_ms`, a +record without an end line has an unknown outcome. Never count a record +without an end line as failed. Recording is synchronous and best-effort. If a write or the sync fails, the record stops. Record I/O errors do not replace the invocation result: a diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index 99839a0fb..7f3ec6f4b 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -86,7 +86,7 @@ }, "container_oom_kills": { "$ref": "#/$defs/OomKills", - "description": "The container's cgroup v2 `oom_kill` count when the invocation ended." + "description": "The container's cgroup v2 `oom_kill` count when the invocation ended,\nread with `resources`." }, "elapsed_ms": { "description": "Monotonic time from the start line to the end line, in milliseconds.", @@ -105,6 +105,10 @@ ], "description": "Error code, stage and retryable flag, without the message." }, + "resources": { + "$ref": "#/$defs/Resources", + "description": "The runtime's process values when the invocation ended, read under\nthe runtime's host lock right after the adapter host answered or the\ntimeout fired." + }, "status": { "$ref": "#/$defs/RunStatus", "description": "Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`." @@ -135,6 +139,7 @@ "required": [ "status", "elapsed_ms", + "resources", "container_oom_kills" ], "type": "object" @@ -348,6 +353,43 @@ } ] }, + "Resources": { + "anyOf": [ + { + "$ref": "#/$defs/RuntimeResourceSnapshot", + "description": "The snapshot." + }, + { + "description": "No snapshot could be read.", + "properties": { + "error": { + "$ref": "#/$defs/ResourcesError", + "description": "Why no snapshot could be read." + } + }, + "required": [ + "error" + ], + "type": "object" + } + ], + "description": "A runtime resource snapshot, or the error code that explains why none\ncould be read." + }, + "ResourcesError": { + "description": "Error codes for a snapshot that could not be read.", + "oneOf": [ + { + "const": "resource_observation_unavailable", + "description": "The runtime stopped or is stopping, its adapter host exited, a live\nprocess in its tree or the process table cannot be read, or the\nprocess tree kept changing during the read.", + "type": "string" + }, + { + "const": "resource_observation_unsupported", + "description": "The adapter kind, environment provider, or operating system has no\nresource reader.", + "type": "string" + } + ] + }, "RunStatus": { "description": "Runtime completion status.", "oneOf": [ @@ -368,6 +410,72 @@ } ] }, + "RuntimeResourceSnapshot": { + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", + "properties": { + "cpu_time_seconds": { + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", + "format": "double", + "minimum": 0.0, + "type": "number" + }, + "observed_at": { + "description": "When Fabric read these values, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + }, + "process_count": { + "description": "Live processes in the tree, including the adapter host. Exited processes\nthat are not yet reaped are not counted.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "resident_memory_bytes": { + "description": "Sum of the resident set sizes of the live processes, in bytes. Pages\nshared between processes count once per process.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "runtime_id": { + "description": "Runtime this snapshot describes.", + "type": "string" + }, + "started_at": { + "description": "When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + } + }, + "required": [ + "runtime_id", + "started_at", + "observed_at", + "resident_memory_bytes", + "process_count", + "cpu_time_seconds" + ], + "type": "object" + }, + "SampleLine": { + "description": "Fields of a sample line.", + "properties": { + "elapsed_ms": { + "description": "Monotonic time since the start line, in milliseconds.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "resources": { + "$ref": "#/$defs/Resources", + "description": "The runtime's process values now." + } + }, + "required": [ + "elapsed_ms", + "resources" + ], + "type": "object" + }, "StartLine": { "description": "Fields of the start line.", "properties": { @@ -419,6 +527,19 @@ }, "type": "array" }, + "resources": { + "$ref": "#/$defs/Resources", + "description": "The runtime's process values when the invocation started." + }, + "sample_interval_ms": { + "description": "Interval between sample lines, in milliseconds. Absent when this\nprocess has no sample thread, so the record gets no sample lines.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, "timeout_ms": { "description": "Host invocation timeout that applied, in milliseconds.", "format": "uint64", @@ -431,6 +552,7 @@ "harness", "fabric_version", "timeout_ms", + "resources", "exclusive_container", "container_oom_kills" ], @@ -583,6 +705,20 @@ ], "type": "object" }, + { + "$ref": "#/$defs/SampleLine", + "description": "Written about every 10 s while the invocation runs, when the start\nline has `sample_interval_ms`.", + "properties": { + "kind": { + "const": "sample", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, { "$ref": "#/$defs/EndLine", "description": "Written once, after the invocation returned a result or failed.", diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index d02191a48..aa559eb71 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -53,19 +53,26 @@ Set `RuntimeConfig(artifacts=..., record=RecordConfig(enabled=True))` to have NeMo Fabric write `run-record.jsonl` for each invocation under `/.fabric///`. Recording is off by default and needs an artifact root, the `local` environment provider, and a -`process` or `python` adapter. The -result lists the file as the artifact with kind `run_record`. The file has a -`start` line, written before the request, and an `end` line with status, -elapsed time, error code, numeric usage, artifact names, and `stop`, which +`process` or `python` adapter. The result lists the file as the artifact with +kind `run_record`. The file has a `start` line, written before the request, a +`sample` line about every 10 seconds when the start line has +`sample_interval_ms`, and an `end` line with status, elapsed time, error +code, numeric usage, artifact names, and `stop`, which gives NeMo Fabric's failure cause (`fabric_failed`, `sandbox_failed`, `agent_exhausted_budget`, `unclassified`, or `agent_failed`), the time limit for a timeout, and the adapter host's exit status. Only a `failed` result -reads `agent_failed`; an error NeMo Fabric raises never does. It holds no -prompts, output, or error messages. A record without an end line has an -unknown outcome; do not count it as failed. Recording is synchronous and -best-effort: a failed record write adds a `record_write_failed` event and does -not replace the result, and slow storage delays the invocation. The format is -the `v1alpha1` +reads `agent_failed`; an error NeMo Fabric raises never does. Each line +stores the runtime's resource snapshot as `resources`. When both the start and +end lines hold one, the invocation's CPU time is end minus start, leaving out +descendants that left the process tree. It holds no prompts, output, or error +messages. When the start line has `sample_interval_ms`, a record without an +end line whose file changed within three sample intervals belongs to a running +invocation, and one that has not changed for longer never finished, or its +record stopped on a failed write (the result's `record_write_failed` event +says which). Without `sample_interval_ms`, its outcome is unknown. Never count a record without an +end line as failed. Recording is synchronous and best-effort: a failed record +write adds a `record_write_failed` event and does not replace the result, and +slow storage delays the invocation. The format is the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) and may change until a reader ships. From bc7da5cc76f7611669e4c33bef9efaafe2b78b67 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:33:28 -0700 Subject: [PATCH 035/159] fix: close a record's sampling window when its exchange ends A record stayed registered with the sample ticker until finish, which runs after core releases the host lock and processes the result. In that window another caller can start the next invocation on the same runtime, and the earlier record kept sampling the runtime while the later invocation ran, so its sample lines described the wrong invocation. Core now takes the record's sampling registration under the host lock and drops it when the exchange ends. Dropping it removes the record from the ticker and marks the file closed for sampling, so a sample whose read finishes later is discarded. The registration is declared after the host lock guard, so an early return also closes the window before the lock is released. The regression test ends invocation A's exchange, runs invocation B on the same runtime before A's result is finished, and waits for three more ticks; before this change A's record gained a sample on every one. Tests can give records started on their thread a fast ticker. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/file.rs | 19 ++-- crates/fabric-core/src/record/invocation.rs | 7 ++ crates/fabric-core/src/record/mod.rs | 2 + crates/fabric-core/src/record/sampler.rs | 28 +++++- crates/fabric-core/src/runtime.rs | 104 ++++++++++++++++++++ 5 files changed, 152 insertions(+), 8 deletions(-) diff --git a/crates/fabric-core/src/record/file.rs b/crates/fabric-core/src/record/file.rs index 373d8908c..be2aa26ce 100644 --- a/crates/fabric-core/src/record/file.rs +++ b/crates/fabric-core/src/record/file.rs @@ -21,8 +21,9 @@ pub(crate) struct RecordFile { writer: Option, /// The first create, write or sync failure. pub failure: Option, - /// Set when the end line is written; no sample line follows it. - ended: bool, + /// Set when the sampling window closes, at the exchange's end or at the + /// end line; no sample line follows it. + sampling_closed: bool, } impl RecordFile { @@ -44,7 +45,7 @@ impl RecordFile { request_id, writer, failure, - ended: false, + sampling_closed: false, } } @@ -68,18 +69,24 @@ impl RecordFile { } } - /// Append a sample line, unless the end line is already written. + /// Append a sample line, unless the sampling window is closed. pub(crate) fn append_sample(&mut self, body: LineBody) { - if !self.ended { + if !self.sampling_closed { self.append(body); } } + /// Close the sampling window. A sample whose read finishes later is + /// discarded. + pub(crate) fn close_sampling(&mut self) { + self.sampling_closed = true; + } + /// Append the end line, sync the file and close it. No sample line /// follows, and a sample tick that still holds this record keeps no open /// file handle. pub(crate) fn append_end(&mut self, body: LineBody) { - self.ended = true; + self.sampling_closed = true; self.append(body); if let Some(writer) = self.writer.take() && let Err(error) = writer.sync() diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 042b332f1..a202a7f84 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -217,6 +217,13 @@ impl InvocationRecorder { ExchangeEnd { oom_kills, ..end } } + /// The record's sampling registration. Core holds it under the host lock + /// and drops it when the exchange ends, which closes the sampling window + /// before a later invocation on the runtime can start. + pub(crate) fn take_sampling(&mut self) -> Option { + self.sampling.take() + } + /// Keep what the exchange saw for the end line. pub(crate) fn exchange_ended(&mut self, end: ExchangeEnd) { self.exchange = Some(end); diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index d02e45414..5bf3dd1ee 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -27,4 +27,6 @@ pub(crate) use invocation::{HostExit, InvocationRecorder, RecordStart}; pub(crate) use lines::RecordLine; pub(crate) use sampler::SampleTicker; #[cfg(test)] +pub(crate) use sampler::test_hooks as sampler_test_hooks; +#[cfg(test)] pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/record/sampler.rs b/crates/fabric-core/src/record/sampler.rs index 6c257a60c..a5cdc487e 100644 --- a/crates/fabric-core/src/record/sampler.rs +++ b/crates/fabric-core/src/record/sampler.rs @@ -75,6 +75,10 @@ impl SampleTicker { /// The process-wide ticker, sampling every [`SAMPLE_INTERVAL_MS`]. pub(crate) fn global() -> Arc { + #[cfg(test)] + if let Some(ticker) = test_hooks::TICKER.with(|slot| slot.borrow().clone()) { + return ticker; + } GLOBAL.clone() } @@ -114,7 +118,7 @@ impl SampleTicker { self.lock_records().insert( id, Arc::new(ActiveRecord { - file, + file: Arc::clone(&file), started, read, }), @@ -122,6 +126,7 @@ impl SampleTicker { Registration { ticker: Arc::clone(self), id, + file, } } @@ -161,15 +166,19 @@ fn run(ticker: &Weak, interval: Duration) { } } -/// Keeps a record in the ticker; dropping it stops sampling that record. +/// Keeps a record in the ticker. Dropping it closes the record's sampling +/// window: the ticker stops sampling the record, and a sample whose read is +/// still running is discarded. pub(crate) struct Registration { ticker: Arc, id: u64, + file: Arc>, } impl Drop for Registration { fn drop(&mut self) { self.ticker.lock_records().remove(&self.id); + lock(&self.file).close_sampling(); } } @@ -182,6 +191,21 @@ impl std::fmt::Debug for Registration { } } +/// A per-thread ticker that replaces the process-wide one in tests. +#[cfg(test)] +pub(crate) mod test_hooks { + use std::cell::RefCell; + use std::sync::Arc; + + use super::SampleTicker; + + thread_local! { + /// Ticker for records started on this thread, instead of the + /// process-wide one. + pub(crate) static TICKER: RefCell>> = const { RefCell::new(None) }; + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index a4fba6bef..cfecec717 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1883,6 +1883,12 @@ fn local_host_invocation_result( ticker: SampleTicker::global(), })); } + // Dropping `sampling` closes the record's sampling window. It is + // declared after `host_guard`, so an early return also closes the + // window before the host lock is released. + let sampling = recorder + .as_mut() + .and_then(InvocationRecorder::take_sampling); adapter_invocation.runtime_context.deadline_millis = adapter_deadline_millis(invoke_timeout); let (lifecycle_request, adapter_payload) = match invocation_kind { @@ -1932,6 +1938,9 @@ fn local_host_invocation_result( .as_ref() .zip(end_snapshot.as_ref()) .map(|(recorder, snapshot)| recorder.read_exchange_end(ended_at, snapshot)); + // No later sample belongs to this invocation: the next invocation on + // this runtime can start as soon as the host lock is released. + drop(sampling); match exchange { Ok(output) => { if let (Some(recorder), Some(end)) = (recorder.as_mut(), exchange_end) { @@ -6344,4 +6353,99 @@ for line in sys.stdin: let _ = stop_runtime(&plan, &runtime); let _ = fs::remove_dir_all(root); } + + fn sample_lines(record: &Path) -> usize { + fs::read_to_string(record) + .expect("read record") + .lines() + .filter(|line| line.contains(r#""kind":"sample""#)) + .count() + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn record_gets_no_sample_after_its_exchange_ends_while_a_later_invocation_runs() { + let (root, plan) = record_plan("success"); + let ticker = SampleTicker::new(Duration::from_millis(10)); + crate::record::sampler_test_hooks::TICKER + .with(|slot| *slot.borrow_mut() = Some(Arc::clone(&ticker))); + let runtime = start_runtime(&plan).expect("start local host"); + + // Invocation A's exchange ends, and its result is not finished yet. + let mut first = None; + let first_outcome = local_host_invocation_result( + &plan, + &runtime, + RunRequest::text("first"), + LocalHostInvocation::Invoke, + Duration::from_secs(30), + &mut first, + ); + let first = first.expect("first record"); + let first_record = record_files(&root).pop().expect("first record file"); + let samples_at_exchange_end = sample_lines(&first_record); + // Invocation B runs on the same runtime while A is unfinished. + run_local_host_adapter_with_timeout( + &plan, + &runtime, + RunRequest::text("second"), + Duration::from_secs(30), + ) + .expect("second invocation"); + // A record that stays open on the same ticker shows that the ticker + // ticked after both exchanges. + let sentinel_home = root.join("sentinel"); + fs::create_dir_all(&sentinel_home).expect("sentinel folder"); + let sentinel_invocation = InvocationHandle { + invocation_id: "invocation-sentinel".to_string(), + request_id: "request-sentinel".to_string(), + runtime_id: runtime.runtime_id.clone(), + }; + let sentinel = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &sentinel_invocation, + request: &RunRequest::text("sentinel"), + fabric_home: &sentinel_home, + timeout: Duration::from_secs(30), + cgroup: CgroupReader::system(), + read_resources: { + let runtime = runtime.clone(); + Arc::new(move || local_runtime_resource_snapshot(&runtime)) + }, + ticker: Arc::clone(&ticker), + }); + let sentinel_record = sentinel_home.join("run-record.jsonl"); + let deadline = Instant::now() + Duration::from_secs(10); + while sample_lines(&sentinel_record) < 3 && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + assert!( + sample_lines(&sentinel_record) >= 3, + "the ticker never ticked" + ); + + first.finish(first_outcome).expect("first result"); + drop(sentinel); + crate::record::sampler_test_hooks::TICKER.with(|slot| *slot.borrow_mut() = None); + + let lines: Vec = fs::read_to_string(&first_record) + .expect("read record") + .lines() + .map(|line| serde_json::from_str(line).expect("record line")) + .collect(); + let sample_times: Vec<&Value> = lines + .iter() + .filter(|line| line["kind"] == "sample") + .map(|line| &line["elapsed_ms"]) + .collect(); + assert_eq!( + sample_times.len(), + samples_at_exchange_end, + "sample elapsed_ms: {sample_times:?}" + ); + assert_eq!(lines.last().expect("end line")["kind"], "end"); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } } From 8e7eda7269964accfb186c3297727042eb155ab5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:33:36 -0700 Subject: [PATCH 036/159] test: block a sample read across the end line instead of sleeping The shared-ticker test slept 60 ms after the end line and checked that no sample followed, which passes whenever the ticker happens not to tick in that window. The new test blocks the first sample read, writes the end line while the read is in flight, releases the read, and waits until a second record on the same ticker gets its next sample, so the blocked tick has finished before the record is read. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 45 ++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index a202a7f84..e693ac6bf 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1161,7 +1161,6 @@ mod tests { diagnostics: String::new(), }; first.finish(Err(error)).expect_err("error passes through"); - std::thread::sleep(Duration::from_millis(60)); let lines = read_lines(&first_path); let validator = record_line_validator(); @@ -1180,6 +1179,50 @@ mod tests { let _ = std::fs::remove_dir_all(second_folder); } + #[test] + fn a_sample_read_in_flight_when_the_record_ends_writes_nothing_after_the_end_line() { + use std::sync::Barrier; + use std::sync::atomic::{AtomicUsize, Ordering}; + let ticker = SampleTicker::new(Duration::from_millis(10)); + let (entered, release) = (Arc::new(Barrier::new(2)), Arc::new(Barrier::new(2))); + let calls = Arc::new(AtomicUsize::new(0)); + let read: ReadResources = { + let (entered, release) = (Arc::clone(&entered), Arc::clone(&release)); + Arc::new(move || { + // Call 0 is the start line's read; call 1 is the first sample + // read, which waits until the test has finished the record. + if calls.fetch_add(1, Ordering::SeqCst) == 1 { + entered.wait(); + release.wait(); + } + Ok(snapshot()) + }) + }; + let (folder, recorder) = start_recorder("in-flight", &ticker, read); + // A second record on the same ticker, sampled after the first in + // each tick, shows when the blocked tick has finished. + let (sentinel_folder, sentinel) = + start_recorder("in-flight-sentinel", &ticker, Arc::new(|| Ok(snapshot()))); + let path = folder.join(RECORD_FILE_NAME); + let sentinel_path = sentinel_folder.join(RECORD_FILE_NAME); + + entered.wait(); + let (_, _, invocation) = record_test_fixture(); + recorder + .finish(Ok(succeeded_result(&invocation))) + .expect("result"); + let sentinel_samples = samples(&sentinel_path).len(); + release.wait(); + wait_for_samples(&sentinel_path, sentinel_samples + 1); + + let lines = read_lines(&path); + assert_eq!(lines.last().expect("end line")["kind"], "end", "{lines:?}"); + assert!(samples(&path).is_empty(), "{lines:?}"); + drop(sentinel); + let _ = std::fs::remove_dir_all(folder); + let _ = std::fs::remove_dir_all(sentinel_folder); + } + #[test] fn sample_line_with_a_snapshot_error_keeps_only_the_error_code() { let ticker = SampleTicker::new(Duration::from_millis(20)); From f7c688684cf1cfda7b9e3f2a25bda8e99e0bf95e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:34:52 -0700 Subject: [PATCH 037/159] docs: say what a recent sample and a stale run record show The guides read a record whose file changed within three sample intervals as a running invocation and an older one as never finished. Neither follows: the one sample thread serves records one after another, so a slow snapshot read delays every record's samples, a suspended process can resume, and a sample says Fabric observed the record, not that the agent made progress. A recent sample now means recently observed; after three intervals the record is stale and its outcome is unknown until an end line exists, and a stale record is never failed. Also say that end minus start CPU time is a partial observed delta whose decrease reads as unknown, that the end oom_kill count read with the snapshot can be replaced by a larger count after a crashed host's exit and is best effort, that sampling does not take the host lock rather than that it costs nothing, that sample lines stop when the exchange ends, and that a forked child stops writing samples, without a general claim about fork safety. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/lines.rs | 12 ++++-- crates/fabric-core/src/record/mod.rs | 3 +- crates/fabric-core/src/record/sampler.rs | 5 ++- docs/sdk/python.mdx | 42 ++++++++++--------- schemas/sdk/run-record-line.schema.json | 4 +- .../references/results-and-errors.md | 24 ++++++----- 6 files changed, 52 insertions(+), 38 deletions(-) diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index 405d46f15..a7f70f5a9 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -71,8 +71,10 @@ pub(crate) enum RecordVersion { pub(crate) enum LineBody { /// Written once, before core sends the request to the adapter host. Start(StartLine), - /// Written about every 10 s while the invocation runs, when the start - /// line has `sample_interval_ms`. + /// Written about every 10 s until the exchange with the adapter host + /// ends, when the start line has `sample_interval_ms`. A recent sample + /// shows that Fabric observed the record recently, not that the agent is + /// making progress. Sample(SampleLine), /// Written once, after the invocation returned a result or failed. End(EndLine), @@ -169,8 +171,10 @@ pub(crate) struct EndLine { /// the runtime's host lock right after the adapter host answered or the /// timeout fired. pub resources: Resources, - /// The container's cgroup v2 `oom_kill` count when the invocation ended, - /// read with `resources`. + /// The container's cgroup v2 `oom_kill` count, read with `resources`. + /// After a crashed host's exit, a larger count read then replaces it. + /// Best effort: the kernel can count a kill after the host's exit is + /// seen. pub container_oom_kills: OomKills, /// Why the invocation stopped; absent when it holds nothing, as on success. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index 5bf3dd1ee..b205d64ad 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -5,7 +5,8 @@ //! //! When `runtime.record.enabled` is set, core appends one start line before it //! sends the request to the adapter host, a sample line about every 10 seconds -//! while the invocation runs, and one end line after it returns or fails. +//! until the exchange with the host ends, and one end line after the +//! invocation returns or fails. //! Recording is synchronous and best-effort: record I/O errors do not replace //! the invocation result. A failed write stops the record and becomes a //! `record_write_failed` event on the result, and slow storage delays the diff --git a/crates/fabric-core/src/record/sampler.rs b/crates/fabric-core/src/record/sampler.rs index a5cdc487e..23f4f9427 100644 --- a/crates/fabric-core/src/record/sampler.rs +++ b/crates/fabric-core/src/record/sampler.rs @@ -6,7 +6,8 @@ //! The thread starts with the first recorded invocation and serves every //! runtime in the process, so N parallel runtimes cost one wake-up per //! interval, not N threads. Each tick reads each record's runtime snapshot, -//! which never waits for the runtime's host lock. +//! which never takes the runtime's host lock. Records are sampled one after +//! another, so one slow read delays the samples of every record. use std::collections::BTreeMap; use std::panic::{self, AssertUnwindSafe}; @@ -36,7 +37,7 @@ struct ActiveRecord { impl ActiveRecord { /// Read the snapshot, then append the sample line. The read runs without /// the file lock, so a slow read never delays the end line. A read that - /// panics is written as unavailable, so the record keeps its heartbeat. + /// panics is written as unavailable, so the record keeps its sample lines. fn sample(&self) { let resources = panic::catch_unwind(AssertUnwindSafe(|| Resources::from(&(self.read)()))) .unwrap_or(Resources::UNAVAILABLE); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 72000f93e..11aa774a0 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1033,9 +1033,9 @@ result lists it as the `record` artifact with kind `run_record`: record = next(a for a in result.artifacts.artifacts if a.kind == "run_record") ``` -The file holds a start line, a sample line about every 10 seconds while the -invocation runs when the start line has `sample_interval_ms`, and an end -line. Every line carries `schema` +The file holds a start line, a sample line about every 10 seconds until the +adapter host answers or the timeout fires when the start line has +`sample_interval_ms`, and an end line. Every line carries `schema` (`nemo.fabric.run_record`), `version` (`v1alpha1`), `kind`, `runtime_id`, `invocation_id`, `request_id`, and `time`. @@ -1070,17 +1070,18 @@ can be read, `resources` is `{"error": "resource_observation_unavailable"}` or `{"error": "resource_observation_unsupported"}`. CPU time is cumulative since the adapter host started. When both the start and end lines hold a snapshot, -the invocation's CPU time is the end line's `cpu_time_seconds` minus the start -line's. The difference leaves out descendants that left the process tree. +the end line's `cpu_time_seconds` minus the start line's is a partial observed +CPU delta for the invocation: it leaves out descendants that left the process +tree before the end snapshot. Read a decrease as unknown, never as zero or a +negative CPU time. One NeMo Fabric thread writes the sample lines for every recorded invocation -in the process. When the operating system refuses that thread, the start line -has no `sample_interval_ms` and the record gets no sample lines. A process -that forks after its first recorded invocation, such as Python -`multiprocessing` with the `fork` start method, has no sample thread in the -child, so in a forked child the start line has no `sample_interval_ms`. Use -the `spawn` or `forkserver` start method with recording; `fork` after threads -exist is unsafe. +in the process, one record after another. Sampling does not take the host +lock. When the operating system refuses that thread, the start line has no +`sample_interval_ms` and the record gets no sample lines. A forked child stops +writing samples: in a process forked after its first recorded invocation, such +as a Python `multiprocessing` child with the `fork` start method, the start +line has no `sample_interval_ms` and the record gets no sample lines. The end line describes the invocation. A one-shot `Fabric.run` stops the runtime after the invocation; when that stop fails, the result reads `failed` @@ -1161,13 +1162,16 @@ not covered by this rule. Each line is written with one append, so a killed process keeps every line whose write returned. NeMo Fabric syncs the file once, after the end line. -When the start line has `sample_interval_ms`, a record without an end line -whose file changed within three sample intervals (30 seconds at the default) -belongs to a running invocation, and one that has not changed for longer -never finished, or its record stopped on a failed write (the result's -`record_write_failed` event says which). Without `sample_interval_ms`, a -record without an end line has an unknown outcome. Never count a record -without an end line as failed. +When the start line has `sample_interval_ms`, a recent sample shows that NeMo +Fabric observed the record recently, not that the agent is making progress. +After three sample intervals (30 seconds at the default) without an update, +the record is stale, and the invocation's outcome is unknown until an end +line exists. A stale record is never counted as failed: one slow snapshot read +delays samples for every record in the process, and a suspended process can +resume. A record also stops changing when a write fails; the result's +`record_write_failed` event says so. Without `sample_interval_ms`, a record +without an end line has an unknown outcome. Never count a record without an +end line as failed. Recording is synchronous and best-effort. If a write or the sync fails, the record stops. Record I/O errors do not replace the invocation result: a diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index 7f3ec6f4b..f14f319a7 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -86,7 +86,7 @@ }, "container_oom_kills": { "$ref": "#/$defs/OomKills", - "description": "The container's cgroup v2 `oom_kill` count when the invocation ended,\nread with `resources`." + "description": "The container's cgroup v2 `oom_kill` count, read with `resources`.\nAfter a crashed host's exit, a larger count read then replaces it.\nBest effort: the kernel can count a kill after the host's exit is\nseen." }, "elapsed_ms": { "description": "Monotonic time from the start line to the end line, in milliseconds.", @@ -707,7 +707,7 @@ }, { "$ref": "#/$defs/SampleLine", - "description": "Written about every 10 s while the invocation runs, when the start\nline has `sample_interval_ms`.", + "description": "Written about every 10 s until the exchange with the adapter host\nends, when the start line has `sample_interval_ms`. A recent sample\nshows that Fabric observed the record recently, not that the agent is\nmaking progress.", "properties": { "kind": { "const": "sample", diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index aa559eb71..a64699e3d 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -55,22 +55,26 @@ NeMo Fabric write `run-record.jsonl` for each invocation under default and needs an artifact root, the `local` environment provider, and a `process` or `python` adapter. The result lists the file as the artifact with kind `run_record`. The file has a `start` line, written before the request, a -`sample` line about every 10 seconds when the start line has -`sample_interval_ms`, and an `end` line with status, elapsed time, error +`sample` line about every 10 seconds until the adapter host answers or the +timeout fires, when the start line has `sample_interval_ms`, and an `end` line with status, elapsed time, error code, numeric usage, artifact names, and `stop`, which gives NeMo Fabric's failure cause (`fabric_failed`, `sandbox_failed`, `agent_exhausted_budget`, `unclassified`, or `agent_failed`), the time limit for a timeout, and the adapter host's exit status. Only a `failed` result reads `agent_failed`; an error NeMo Fabric raises never does. Each line stores the runtime's resource snapshot as `resources`. When both the start and -end lines hold one, the invocation's CPU time is end minus start, leaving out -descendants that left the process tree. It holds no prompts, output, or error -messages. When the start line has `sample_interval_ms`, a record without an -end line whose file changed within three sample intervals belongs to a running -invocation, and one that has not changed for longer never finished, or its -record stopped on a failed write (the result's `record_write_failed` event -says which). Without `sample_interval_ms`, its outcome is unknown. Never count a record without an -end line as failed. Recording is synchronous and best-effort: a failed record +end lines hold one, end minus start is a partial observed CPU delta that +leaves out descendants that left the process tree; read a decrease as +unknown, never as zero or negative. It holds no prompts, output, or error +messages. A recent sample shows that NeMo Fabric observed the record +recently, not that the agent is making progress. After three sample intervals +without an update, the record is stale and the invocation's outcome is +unknown until an end line exists; never count a stale record as failed. One +slow snapshot read delays samples for every record in the process, and a +suspended process can resume. A record also stops changing when a write +fails; the result's `record_write_failed` event says so. Without +`sample_interval_ms`, a record without an end line has an unknown outcome. +Recording is synchronous and best-effort: a failed record write adds a `record_write_failed` event and does not replace the result, and slow storage delays the invocation. The format is the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) From 7e2be97645241fe1c162dfd27e638e5540297f7c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:44:46 -0700 Subject: [PATCH 038/159] test: build the sampling window test helpers only where the test runs The regression test for the sampling window runs on Linux and macOS, so on Windows its record helper and the ticker hook re-export were unused and clippy warned. Gate both to the same platforms. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/mod.rs | 2 +- crates/fabric-core/src/runtime.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index b205d64ad..8bf3551b0 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -27,7 +27,7 @@ pub(crate) use cgroup::CgroupReader; pub(crate) use invocation::{HostExit, InvocationRecorder, RecordStart}; pub(crate) use lines::RecordLine; pub(crate) use sampler::SampleTicker; -#[cfg(test)] +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] pub(crate) use sampler::test_hooks as sampler_test_hooks; #[cfg(test)] pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index cfecec717..52737acc4 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -6354,6 +6354,7 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[cfg(any(target_os = "linux", target_os = "macos"))] fn sample_lines(record: &Path) -> usize { fs::read_to_string(record) .expect("read record") From 24407bb942fb71d65e07542d0dd194b84752ab00 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:59:43 -0700 Subject: [PATCH 039/159] docs: name NeMo Fabric in the resource snapshot docs The RuntimeResourceSnapshot overview, the started_at and observed_at descriptions, the LocalHostProcess::new doc comment and the Python docstring used a standalone "Fabric" for the product. The repository's naming rule asks for "NeMo Fabric" after first use. The two snapshot schemas are regenerated with just schemas; the TypeScript contract does not change. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 10 +++++----- schemas/sdk/run-result.schema.json | 6 +++--- schemas/sdk/runtime-resource-snapshot.schema.json | 6 +++--- .../nemo-fabric-runtime/src/nemo_fabric/types.py | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 61755580b..7e07a1c99 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -39,16 +39,16 @@ const NANOS_PER_SECOND: f64 = 1_000_000_000.0; /// /// Memory, process count, and CPU time cover the adapter host process and its /// live descendants. For a runtime started on a service, the shared service -/// process is not included. Values Fabric cannot read are returned as an error, -/// never as zero. +/// process is not included. Values NeMo Fabric cannot read are returned as an +/// error, never as zero. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct RuntimeResourceSnapshot { /// Runtime this snapshot describes. pub runtime_id: String, - /// When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. + /// When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. #[schemars(extend("format" = "date-time"))] pub started_at: String, - /// When Fabric read these values, as an RFC 3339 UTC timestamp. + /// When NeMo Fabric read these values, as an RFC 3339 UTC timestamp. #[schemars(extend("format" = "date-time"))] pub observed_at: String, /// Sum of the resident set sizes of the live processes, in bytes. Pages @@ -75,7 +75,7 @@ pub(crate) struct LocalHostProcess { } impl LocalHostProcess { - /// Record the host `pid`, started by Fabric at `started_at`. + /// Record the host `pid`, started by NeMo Fabric at `started_at`. pub(crate) fn new(pid: u32, started_at: SystemTime) -> Self { Self { pid, diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 504d6b129..7d5e1ef80 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -280,7 +280,7 @@ "type": "object" }, "RuntimeResourceSnapshot": { - "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values NeMo Fabric cannot read are returned as an\nerror, never as zero.", "properties": { "cpu_time_seconds": { "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", @@ -289,7 +289,7 @@ "type": "number" }, "observed_at": { - "description": "When Fabric read these values, as an RFC 3339 UTC timestamp.", + "description": "When NeMo Fabric read these values, as an RFC 3339 UTC timestamp.", "format": "date-time", "type": "string" }, @@ -310,7 +310,7 @@ "type": "string" }, "started_at": { - "description": "When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "description": "When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", "format": "date-time", "type": "string" } diff --git a/schemas/sdk/runtime-resource-snapshot.schema.json b/schemas/sdk/runtime-resource-snapshot.schema.json index 0b3ac2916..3b3a8747b 100644 --- a/schemas/sdk/runtime-resource-snapshot.schema.json +++ b/schemas/sdk/runtime-resource-snapshot.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values NeMo Fabric cannot read are returned as an\nerror, never as zero.", "properties": { "cpu_time_seconds": { "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", @@ -9,7 +9,7 @@ "type": "number" }, "observed_at": { - "description": "When Fabric read these values, as an RFC 3339 UTC timestamp.", + "description": "When NeMo Fabric read these values, as an RFC 3339 UTC timestamp.", "format": "date-time", "type": "string" }, @@ -30,7 +30,7 @@ "type": "string" }, "started_at": { - "description": "When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "description": "When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", "format": "date-time", "type": "string" } diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index c6a3cb682..001e060a9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1843,8 +1843,8 @@ class RuntimeResourceSnapshot(FabricMapping): Attributes: runtime_id: Runtime this snapshot describes. - started_at: When Fabric started the adapter host, as RFC 3339 UTC. - observed_at: When Fabric read these values, as RFC 3339 UTC. + started_at: When NeMo Fabric started the adapter host, as RFC 3339 UTC. + observed_at: When NeMo Fabric read these values, as RFC 3339 UTC. resident_memory_bytes: Sum of the live processes' resident set sizes. Pages shared between processes count once per process. process_count: Live processes in the tree, including the adapter host. From 778d8d8b50e0d9337161aaab828cb18cdace9ac3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 07:59:44 -0700 Subject: [PATCH 040/159] docs: regenerate the references for the snapshot product name Regenerated the Rust reference with Rust 1.94.0 and the Python reference with lazydocs; only the RuntimeResourceSnapshot page and the RuntimeResourceSnapshot attributes in the types page change. Signed-off-by: mnajafian-nv --- .../api/python-library-reference/nemo_fabric.types.md | 4 ++-- .../resource-snapshot/struct-runtimeresourcesnapshot.mdx | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 59c7e7c06..3d0201fac 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1255,8 +1255,8 @@ Memory, process count, and CPU time cover the adapter host process and its live **Attributes:** - `runtime_id`: Runtime this snapshot describes. - - `started_at`: When Fabric started the adapter host, as RFC 3339 UTC. - - `observed_at`: When Fabric read these values, as RFC 3339 UTC. + - `started_at`: When NeMo Fabric started the adapter host, as RFC 3339 UTC. + - `observed_at`: When NeMo Fabric read these values, as RFC 3339 UTC. - `resident_memory_bytes`: Sum of the live processes' resident set sizes. Pages shared between processes count once per process. - `process_count`: Live processes in the tree, including the adapter host. - `cpu_time_seconds`: CPU time used since the adapter host started, including exited descendants. It can drop when a descendant leaves the tree. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx index aff244954..9c5acd32d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/struct-runtimeresourcesnapshot.mdx @@ -14,7 +14,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Timestamped resource usage of one runtime's local adapter host process tree. -Memory, process count, and CPU time cover the adapter host process and its live descendants. For a runtime started on a service, the shared service process is not included. Values Fabric cannot read are returned as an error, never as zero. +Memory, process count, and CPU time cover the adapter host process and its live descendants. For a runtime started on a service, the shared service process is not included. Values NeMo Fabric cannot read are returned as an error, never as zero. ## Fields @@ -24,11 +24,11 @@ Runtime this snapshot describes. ### `started_at: String` -When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. +When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp. ### `observed_at: String` -When Fabric read these values, as an RFC 3339 UTC timestamp. +When NeMo Fabric read these values, as an RFC 3339 UTC timestamp. ### `resident_memory_bytes: u64` From 395efa67e5c3a22217269a0180602cdf85f19d28 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 08:16:33 -0700 Subject: [PATCH 041/159] test: say the exited-child threshold is two ticks only on Linux The comment called the 20 ms threshold two 10 ms clock ticks, but the test also runs on macOS, where proc_pid_rusage reports CPU time in nanoseconds. It now says 20 ms is two ticks on Linux. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/resource_snapshot.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 7e07a1c99..5029cdf1a 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -834,9 +834,9 @@ mod tests { assert_eq!(running.process_count, 2, "child never seen: {running:?}"); assert_eq!(exited.process_count, 1, "child still running: {exited:?}"); - // The host only sleeps, so two 10 ms clock ticks can come only from - // the exited child. A fast shell can finish the loop in well under - // 100 ms. + // The host only sleeps, so 20 ms of CPU can come only from the exited + // child. That is two clock ticks on Linux (10 ms each), and a fast + // shell can finish the loop in well under 100 ms. assert!( exited.cpu_time_seconds >= 0.02, "exited child's CPU (at least two clock ticks) missing: {exited:?}" From f18b0edc559b61fdf8e5a199e679d32715c1211b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 08:28:45 -0700 Subject: [PATCH 042/159] chore: regenerate the run record line schema for the snapshot wording The merged snapshot docs now say NeMo Fabric, and the run record line schema embeds those descriptions. Signed-off-by: mnajafian-nv --- schemas/sdk/run-record-line.schema.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index f14f319a7..37ea73f27 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -411,7 +411,7 @@ ] }, "RuntimeResourceSnapshot": { - "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values Fabric cannot read are returned as an error,\nnever as zero.", + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values NeMo Fabric cannot read are returned as an\nerror, never as zero.", "properties": { "cpu_time_seconds": { "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", @@ -420,7 +420,7 @@ "type": "number" }, "observed_at": { - "description": "When Fabric read these values, as an RFC 3339 UTC timestamp.", + "description": "When NeMo Fabric read these values, as an RFC 3339 UTC timestamp.", "format": "date-time", "type": "string" }, @@ -441,7 +441,7 @@ "type": "string" }, "started_at": { - "description": "When Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "description": "When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", "format": "date-time", "type": "string" } From ddff2da28d8eed519366454a27dee5a42ce92ce2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 08:50:34 -0700 Subject: [PATCH 043/159] docs: call end minus start CPU time an estimate The run record docs said end minus start CPU time is a partial count that only leaves out descendants. It can also overcount: a start snapshot can read a parent before it reaps a child, and the end snapshot then counts that child's earlier CPU through the parent's waited-children total. The Python SDK guide and the integration skill now call the difference an estimate that can overcount or undercount, and say to treat a negative difference as unknown. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 7 +++---- .../references/results-and-errors.md | 10 +++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 11aa774a0..3adafc440 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1070,10 +1070,9 @@ can be read, `resources` is `{"error": "resource_observation_unavailable"}` or `{"error": "resource_observation_unsupported"}`. CPU time is cumulative since the adapter host started. When both the start and end lines hold a snapshot, -the end line's `cpu_time_seconds` minus the start line's is a partial observed -CPU delta for the invocation: it leaves out descendants that left the process -tree before the end snapshot. Read a decrease as unknown, never as zero or a -negative CPU time. +the end line's `cpu_time_seconds` minus the start line's is an estimate of the +invocation's CPU time. Reaping and changes to the process tree can make it +overcount or undercount. Treat a negative difference as unknown. One NeMo Fabric thread writes the sample lines for every recorded invocation in the process, one record after another. Sampling does not take the host diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index a64699e3d..7409c0038 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -63,11 +63,11 @@ gives NeMo Fabric's failure cause (`fabric_failed`, `sandbox_failed`, for a timeout, and the adapter host's exit status. Only a `failed` result reads `agent_failed`; an error NeMo Fabric raises never does. Each line stores the runtime's resource snapshot as `resources`. When both the start and -end lines hold one, end minus start is a partial observed CPU delta that -leaves out descendants that left the process tree; read a decrease as -unknown, never as zero or negative. It holds no prompts, output, or error -messages. A recent sample shows that NeMo Fabric observed the record -recently, not that the agent is making progress. After three sample intervals +end lines hold one, end minus start CPU time is an estimate. Reaping and +changes to the process tree can make it overcount or undercount the +invocation's CPU time. Treat a negative difference as unknown. The file holds +no prompts, output, or error messages. A recent sample shows that NeMo Fabric +observed the record recently, not that the agent is making progress. After three sample intervals without an update, the record is stale and the invocation's outcome is unknown until an end line exists; never count a stale record as failed. One slow snapshot read delays samples for every record in the process, and a From e1dd296ce286fed769f53eb85ad0ff89137b330f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 08:52:00 -0700 Subject: [PATCH 044/159] test: drop a sample read that finishes after the exchange ends The existing in-flight test finishes the record, so the end line closes the writer and a late sample has no file to write to. It passes even without the sampling window check. Core drops the sampling registration earlier, when the exchange ends, while the file stays open for the end line. The new test blocks a sample read, drops the registration without finishing the record, releases the read, and checks that the file still holds only the start line. With the sampling_closed check in RecordFile::append_sample removed, the late sample is appended and the test fails. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 51 +++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index e693ac6bf..bf76679ff 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1223,6 +1223,57 @@ mod tests { let _ = std::fs::remove_dir_all(sentinel_folder); } + #[test] + fn a_sample_read_in_flight_when_the_exchange_ends_writes_nothing_to_the_open_record() { + use std::sync::Barrier; + use std::sync::atomic::{AtomicUsize, Ordering}; + let ticker = SampleTicker::new(Duration::from_millis(10)); + let (entered, release) = (Arc::new(Barrier::new(2)), Arc::new(Barrier::new(2))); + let calls = Arc::new(AtomicUsize::new(0)); + let read: ReadResources = { + let (entered, release) = (Arc::clone(&entered), Arc::clone(&release)); + Arc::new(move || { + // Call 0 is the start line's read; call 1 is the first sample + // read, which waits until the test has closed the window. + if calls.fetch_add(1, Ordering::SeqCst) == 1 { + entered.wait(); + release.wait(); + } + Ok(snapshot()) + }) + }; + let (folder, mut recorder) = start_recorder("window-closed", &ticker, read); + // A second record on the same ticker, sampled after the first in + // each tick, shows when the blocked tick has finished. + let (sentinel_folder, sentinel) = start_recorder( + "window-closed-sentinel", + &ticker, + Arc::new(|| Ok(snapshot())), + ); + let path = folder.join(RECORD_FILE_NAME); + let sentinel_path = sentinel_folder.join(RECORD_FILE_NAME); + + entered.wait(); + // What core does when the exchange ends: drop the registration while + // the record file stays open for the end line. + drop(recorder.take_sampling()); + let sentinel_samples = samples(&sentinel_path).len(); + release.wait(); + wait_for_samples(&sentinel_path, sentinel_samples + 1); + + let lines = read_lines(&path); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert_eq!(lines[0]["kind"], "start"); + let (_, _, invocation) = record_test_fixture(); + recorder + .finish(Ok(succeeded_result(&invocation))) + .expect("result"); + assert!(samples(&path).is_empty(), "{:?}", read_lines(&path)); + drop(sentinel); + let _ = std::fs::remove_dir_all(folder); + let _ = std::fs::remove_dir_all(sentinel_folder); + } + #[test] fn sample_line_with_a_snapshot_error_keeps_only_the_error_code() { let ticker = SampleTicker::new(Duration::from_millis(20)); From 6bb233f5e4f1cd6647cc1bd3d3e5b32c66df3730 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:03:21 -0700 Subject: [PATCH 045/159] docs: tie the run record format's stability to nemo-fabric report The record line's version doc, the schema catalog, the Python SDK guide and the integration skill said the v1alpha1 format may change until a record reader ships. nemo-fabric watch also reads records and is planned to ship first, so that wording would end the alpha period too early. They now say the format may change until the nemo-fabric report command ships. The run record line schema is regenerated with just schemas; the TypeScript contract does not change. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/lines.rs | 3 ++- docs/sdk/python.mdx | 3 ++- schemas/SCHEMA.md | 4 ++-- schemas/sdk/run-record-line.schema.json | 2 +- skills/nemo-fabric-integrate/references/results-and-errors.md | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index a7f70f5a9..f2a43e577 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -33,7 +33,8 @@ const UNRECOGNIZED_CODE: &str = "unrecognized"; pub(crate) struct RecordLine { /// Always `nemo.fabric.run_record`. pub schema: RecordSchema, - /// Line format version. `v1alpha1` may change until a record reader ships. + /// Line format version. `v1alpha1` may change until the `nemo-fabric report` + /// command ships. pub version: RecordVersion, /// Runtime id. pub runtime_id: String, diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 3adafc440..c190b4936 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1185,7 +1185,8 @@ delay. The file is created with mode `0600` on Linux and macOS. Creation refuses an existing file or symbolic link at `run-record.jsonl`. The line format is published as `schemas/sdk/run-record-line.schema.json` at -version `v1alpha1`, which may change until a reader for run records ships. +version `v1alpha1`, which may change until the `nemo-fabric report` command +ships. ## Events diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 48de38fee..153efb228 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -128,8 +128,8 @@ descriptor schema. - `sdk/error-info`: structured runtime or adapter error metadata. - `sdk/fabric-event`: NeMo Fabric lifecycle or progress event. - `sdk/run-record-line`: one line of the opt-in per-invocation run record, - `run-record.jsonl`. The schema is `v1alpha1` and may change until a reader - for run records ships. + `run-record.jsonl`. The schema is `v1alpha1` and may change until the + `nemo-fabric report` command ships. - `sdk/runtime-resource-snapshot`: timestamped memory, process count, and CPU time of a runtime's local adapter host and its descendants. diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index 37ea73f27..20da40125 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -758,7 +758,7 @@ }, "version": { "$ref": "#/$defs/RecordVersion", - "description": "Line format version. `v1alpha1` may change until a record reader ships." + "description": "Line format version. `v1alpha1` may change until the `nemo-fabric report`\ncommand ships." } }, "required": [ diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 7409c0038..ab84d20b7 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -78,7 +78,7 @@ Recording is synchronous and best-effort: a failed record write adds a `record_write_failed` event and does not replace the result, and slow storage delays the invocation. The format is the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) -and may change until a reader ships. +and may change until the `nemo-fabric report` command ships. ## Runtime Resource Snapshots From baf3566374dee184bf711e446d393ac60394a048 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:24:13 -0700 Subject: [PATCH 046/159] test: bound the sample race test handshakes The two in-flight sample tests paired the test thread and the blocked sample read with Barrier::wait, which has no timeout. A regression that stopped the ticker from reading, or that made the test wait on the blocked read, would hang the test run instead of failing it. Both tests now share one helper whose handshakes are mpsc channels: the test waits up to 10 seconds for the read to start and the read waits up to 10 seconds for release, each with a message that names the missing step. With the ticker's sample loop removed, both tests fail after the timeout with "the ticker never started the first sample read". With the sampling_closed check in RecordFile::append_sample removed, the exchange-end test still fails. With that check removed and append_end keeping the writer open, the record-end test still fails. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 86 ++++++++++++--------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index bf76679ff..102cbd45c 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -518,6 +518,8 @@ fn end_artifacts(manifest: &ArtifactManifest) -> Vec { #[cfg(test)] mod tests { + use std::sync::mpsc::{self, Receiver, Sender}; + use super::*; use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; use crate::record::lines::{RecordLine, RecordSchema, RecordVersion}; @@ -1137,6 +1139,42 @@ mod tests { ); } + /// How long a race test's handshake waits before it fails. + const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); + + /// A read that blocks its first sample read. Call 0 is the start line's + /// read. Call 1 signals the returned receiver, then waits until the test + /// sends on the returned sender. Both waits time out, so a regression + /// fails the test instead of hanging it. + fn read_blocked_at_first_sample() -> (ReadResources, Receiver<()>, Sender<()>) { + use std::sync::atomic::{AtomicUsize, Ordering}; + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel::<()>(); + let release_rx = Mutex::new(release_rx); + let calls = AtomicUsize::new(0); + let read: ReadResources = Arc::new(move || { + if calls.fetch_add(1, Ordering::SeqCst) == 1 { + entered_tx + .send(()) + .expect("the test waits for the sample read"); + release_rx + .lock() + .expect("release receiver lock") + .recv_timeout(HANDSHAKE_TIMEOUT) + .expect("the test never released the blocked sample read"); + } + Ok(snapshot()) + }); + (read, entered_rx, release_tx) + } + + /// Wait until the blocked sample read has started. + fn wait_until_entered(entered: &Receiver<()>) { + entered + .recv_timeout(HANDSHAKE_TIMEOUT) + .expect("the ticker never started the first sample read"); + } + #[test] fn two_records_share_one_ticker_and_samples_stop_at_the_end_line() { let ticker = SampleTicker::new(Duration::from_millis(20)); @@ -1181,23 +1219,9 @@ mod tests { #[test] fn a_sample_read_in_flight_when_the_record_ends_writes_nothing_after_the_end_line() { - use std::sync::Barrier; - use std::sync::atomic::{AtomicUsize, Ordering}; let ticker = SampleTicker::new(Duration::from_millis(10)); - let (entered, release) = (Arc::new(Barrier::new(2)), Arc::new(Barrier::new(2))); - let calls = Arc::new(AtomicUsize::new(0)); - let read: ReadResources = { - let (entered, release) = (Arc::clone(&entered), Arc::clone(&release)); - Arc::new(move || { - // Call 0 is the start line's read; call 1 is the first sample - // read, which waits until the test has finished the record. - if calls.fetch_add(1, Ordering::SeqCst) == 1 { - entered.wait(); - release.wait(); - } - Ok(snapshot()) - }) - }; + // The first sample read waits until the test has finished the record. + let (read, entered, release) = read_blocked_at_first_sample(); let (folder, recorder) = start_recorder("in-flight", &ticker, read); // A second record on the same ticker, sampled after the first in // each tick, shows when the blocked tick has finished. @@ -1206,13 +1230,15 @@ mod tests { let path = folder.join(RECORD_FILE_NAME); let sentinel_path = sentinel_folder.join(RECORD_FILE_NAME); - entered.wait(); + wait_until_entered(&entered); let (_, _, invocation) = record_test_fixture(); recorder .finish(Ok(succeeded_result(&invocation))) .expect("result"); let sentinel_samples = samples(&sentinel_path).len(); - release.wait(); + release + .send(()) + .expect("the blocked sample read waits for release"); wait_for_samples(&sentinel_path, sentinel_samples + 1); let lines = read_lines(&path); @@ -1225,23 +1251,9 @@ mod tests { #[test] fn a_sample_read_in_flight_when_the_exchange_ends_writes_nothing_to_the_open_record() { - use std::sync::Barrier; - use std::sync::atomic::{AtomicUsize, Ordering}; let ticker = SampleTicker::new(Duration::from_millis(10)); - let (entered, release) = (Arc::new(Barrier::new(2)), Arc::new(Barrier::new(2))); - let calls = Arc::new(AtomicUsize::new(0)); - let read: ReadResources = { - let (entered, release) = (Arc::clone(&entered), Arc::clone(&release)); - Arc::new(move || { - // Call 0 is the start line's read; call 1 is the first sample - // read, which waits until the test has closed the window. - if calls.fetch_add(1, Ordering::SeqCst) == 1 { - entered.wait(); - release.wait(); - } - Ok(snapshot()) - }) - }; + // The first sample read waits until the test has closed the window. + let (read, entered, release) = read_blocked_at_first_sample(); let (folder, mut recorder) = start_recorder("window-closed", &ticker, read); // A second record on the same ticker, sampled after the first in // each tick, shows when the blocked tick has finished. @@ -1253,12 +1265,14 @@ mod tests { let path = folder.join(RECORD_FILE_NAME); let sentinel_path = sentinel_folder.join(RECORD_FILE_NAME); - entered.wait(); + wait_until_entered(&entered); // What core does when the exchange ends: drop the registration while // the record file stays open for the end line. drop(recorder.take_sampling()); let sentinel_samples = samples(&sentinel_path).len(); - release.wait(); + release + .send(()) + .expect("the blocked sample read waits for release"); wait_for_samples(&sentinel_path, sentinel_samples + 1); let lines = read_lines(&path); From c723c68b6ee3ea6c9019ae22a657275ba29f66d8 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:39:57 -0700 Subject: [PATCH 047/159] fix: require RFC 3339 UTC resource snapshot timestamps RuntimeResourceSnapshot documents started_at and observed_at as RFC 3339 UTC, but from_mapping accepted any non-empty string, so values such as "yesterday" or a +02:00 offset passed validation. Parse both fields and reject values without the date-T-time form or with a nonzero UTC offset. The original string is kept unchanged. Signed-off-by: mnajafian-nv --- .../src/nemo_fabric/types.py | 23 +++++++++++++++++-- .../python/test_runtime_resource_snapshot.py | 18 +++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 001e060a9..ecad2667d 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -10,6 +10,8 @@ from collections.abc import Mapping from collections.abc import Sequence from copy import deepcopy +from datetime import datetime +from datetime import timedelta from pathlib import Path from types import MappingProxyType from typing import Any @@ -68,6 +70,22 @@ def _required_text(value: Any, name: str) -> str: return value +def _utc_timestamp(value: Any, name: str) -> str: + text = _required_text(value, name) + message = f"{name} must be an RFC 3339 UTC timestamp" + # fromisoformat also accepts a space separator, date-only, and basic ISO + # 8601 forms, so require the RFC 3339 "YYYY-MM-DDT" prefix first. + if text[10:11] != "T": + raise FabricConfigError(message) + try: + parsed = datetime.fromisoformat(text) + except ValueError: + raise FabricConfigError(message) from None + if parsed.utcoffset() != timedelta(0): + raise FabricConfigError(message) + return text + + def _boolean(value: Any, name: str) -> bool: if not isinstance(value, bool): raise FabricConfigError(f"{name} must be a boolean") @@ -1872,8 +1890,9 @@ class RuntimeResourceSnapshot(FabricMapping): @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: - for field in ("runtime_id", "started_at", "observed_at"): - data[field] = _required_text(data.get(field), field.replace("_", " ")) + data["runtime_id"] = _required_text(data.get("runtime_id"), "runtime id") + for field in ("started_at", "observed_at"): + data[field] = _utc_timestamp(data.get(field), field.replace("_", " ")) for field in ("resident_memory_bytes", "process_count"): value = data.get(field) if ( diff --git a/tests/python/test_runtime_resource_snapshot.py b/tests/python/test_runtime_resource_snapshot.py index d14f4354a..7afe7cae6 100644 --- a/tests/python/test_runtime_resource_snapshot.py +++ b/tests/python/test_runtime_resource_snapshot.py @@ -106,6 +106,24 @@ def test_runtime_resource_snapshot_round_trips_and_rejects_invalid_fields(): RuntimeResourceSnapshot.from_mapping({**_SNAPSHOT, field: value}) +@pytest.mark.parametrize("field", ["started_at", "observed_at"]) +def test_runtime_resource_snapshot_timestamps_accept_core_utc_format(field): + snapshot = RuntimeResourceSnapshot.from_mapping( + {**_SNAPSHOT, field: "2026-10-14T18:01:10.205Z"} + ) + + assert getattr(snapshot, field) == "2026-10-14T18:01:10.205Z" + + +@pytest.mark.parametrize("field", ["started_at", "observed_at"]) +@pytest.mark.parametrize( + "value", ["yesterday", "2026-10-14", "2026-10-14T18:01:10+02:00"] +) +def test_runtime_resource_snapshot_timestamps_reject_non_rfc3339_utc(field, value): + with pytest.raises(FabricConfigError, match="RFC 3339 UTC timestamp"): + RuntimeResourceSnapshot.from_mapping({**_SNAPSHOT, field: value}) + + def test_run_result_resource_snapshot_is_typed_or_none(): base = { "agent_name": "demo", From 6b10b68da2aadb575dec30db086283f459d714ed Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:45:40 -0700 Subject: [PATCH 048/159] docs: limit the record write failure event to returned results When the invocation raises, the run record cannot add an event to a result, and the raised error passes through unchanged. Signed-off-by: mnajafian-nv --- .../nemo-fabric-integrate/references/results-and-errors.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 1c06367a3..abe962f88 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -58,8 +58,10 @@ result lists the file as the artifact with kind `run_record`. The file has a elapsed time, error code, numeric usage, and artifact names. It holds no prompts, output, or error messages. A record without an end line has an unknown outcome; do not count it as failed. Recording is synchronous and -best-effort: a failed record write adds a `record_write_failed` event and does -not replace the result, and slow storage delays the invocation. The format is +best-effort: when the invocation returns a result, a failed record write adds +a `record_write_failed` event to it and changes nothing else; when the +invocation raises, the error stays the same error. Slow storage delays the +invocation. The format is the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) and may change until a reader ships. From 5b0033b92aa74a0877a179dca240e7c0dfc7eee5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:52:31 -0700 Subject: [PATCH 049/159] fix: require the full RFC 3339 shape for snapshot timestamps datetime.fromisoformat also accepts basic ISO 8601 forms such as 2026-10-14T180110Z, so check the RFC 3339 date-time shape before parsing. Signed-off-by: mnajafian-nv --- .../nemo-fabric-runtime/src/nemo_fabric/types.py | 12 +++++++++--- tests/python/test_runtime_resource_snapshot.py | 8 +++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index ecad2667d..3987fe121 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -6,6 +6,7 @@ from __future__ import annotations import math +import re from collections.abc import Iterator from collections.abc import Mapping from collections.abc import Sequence @@ -70,12 +71,17 @@ def _required_text(value: Any, name: str) -> str: return value +# RFC 3339 date-time shape. fromisoformat also accepts a space separator, +# date-only values and basic ISO 8601 forms, so check the shape first. +_RFC3339_DATE_TIME = re.compile( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})" +) + + def _utc_timestamp(value: Any, name: str) -> str: text = _required_text(value, name) message = f"{name} must be an RFC 3339 UTC timestamp" - # fromisoformat also accepts a space separator, date-only, and basic ISO - # 8601 forms, so require the RFC 3339 "YYYY-MM-DDT" prefix first. - if text[10:11] != "T": + if _RFC3339_DATE_TIME.fullmatch(text) is None: raise FabricConfigError(message) try: parsed = datetime.fromisoformat(text) diff --git a/tests/python/test_runtime_resource_snapshot.py b/tests/python/test_runtime_resource_snapshot.py index 7afe7cae6..c0dbeaf8b 100644 --- a/tests/python/test_runtime_resource_snapshot.py +++ b/tests/python/test_runtime_resource_snapshot.py @@ -117,7 +117,13 @@ def test_runtime_resource_snapshot_timestamps_accept_core_utc_format(field): @pytest.mark.parametrize("field", ["started_at", "observed_at"]) @pytest.mark.parametrize( - "value", ["yesterday", "2026-10-14", "2026-10-14T18:01:10+02:00"] + "value", + [ + "yesterday", + "2026-10-14", + "2026-10-14T18:01:10+02:00", + "2026-10-14T180110Z", + ], ) def test_runtime_resource_snapshot_timestamps_reject_non_rfc3339_utc(field, value): with pytest.raises(FabricConfigError, match="RFC 3339 UTC timestamp"): From ac9f02c1272b9882114741f98787737600f03d36 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:52:42 -0700 Subject: [PATCH 050/159] docs: name the result fields a record write failure leaves unchanged A returned result can still gain the run_record artifact, so list the fields that stay the same instead of saying nothing else changes. Signed-off-by: mnajafian-nv --- .../nemo-fabric-integrate/references/results-and-errors.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index abe962f88..0c2b8add4 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -59,10 +59,9 @@ elapsed time, error code, numeric usage, and artifact names. It holds no prompts, output, or error messages. A record without an end line has an unknown outcome; do not count it as failed. Recording is synchronous and best-effort: when the invocation returns a result, a failed record write adds -a `record_write_failed` event to it and changes nothing else; when the -invocation raises, the error stays the same error. Slow storage delays the -invocation. The format is -the `v1alpha1` +a `record_write_failed` event to it, and its status, output, error, and usage +stay the same; when the invocation raises, the error stays the same error. +Slow storage delays the invocation. The format is the `v1alpha1` [run record line schema](https://github.com/NVIDIA/NeMo-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) and may change until a reader ships. From f83521b602a41e28a8aa2bc8ab7de1bbfcbf5a83 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 09:54:34 -0700 Subject: [PATCH 051/159] fix: record a schema-invalid result extension as invalid_agent_run_result The result extension check raises InvalidAdapterExtension for a value that breaks the declared schema; the end line recorded it as fabric_error, so one fault had two codes. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 34 +++++++++++++++++++-- docs/sdk/python.mdx | 3 +- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 45451bd2c..2f41f53f6 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -241,9 +241,13 @@ fn result_end_line(result: &RunResult, elapsed_ms: u64) -> EndLine { fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { let (code, stage) = match error { FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), - // After the record starts, only the check of the adapter's result - // extensions raises this, the same fault as a rejected result. - FabricError::AdapterCompatibility { .. } => (INVALID_AGENT_RUN_RESULT, ErrorStage::Invoke), + // After the record starts, the check of the adapter's result + // extensions raises these: an extension the descriptor does not + // declare, or one that breaks its declared schema. Both are the same + // fault as a rejected result. + FabricError::AdapterCompatibility { .. } | FabricError::InvalidAdapterExtension { .. } => { + (INVALID_AGENT_RUN_RESULT, ErrorStage::Invoke) + } FabricError::Write { .. } => (ARTIFACT_WRITE_FAILED, ErrorStage::Artifact), _ => (FABRIC_ERROR, ErrorStage::Invoke), }; @@ -302,6 +306,30 @@ mod tests { use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; use crate::record::test_hooks::{self, Fault}; + #[test] + fn rejected_result_extensions_read_invalid_agent_run_result() { + let undeclared = FabricError::AdapterCompatibility { + adapter_id: "demo".to_string(), + field: "result.extensions".to_string(), + reason: "not declared".to_string(), + }; + let schema_invalid = FabricError::InvalidAdapterExtension { + adapter_id: "demo".to_string(), + descriptor_source: crate::config::DescriptorSource::Bundled, + descriptor_path: std::path::PathBuf::from("adapter.json"), + extension_path: "result.extensions.demo".to_string(), + reason: "breaks the declared schema".to_string(), + }; + for error in [undeclared, schema_invalid] { + let line = error_end_line(&error, 1); + assert_eq!( + line.error.expect("end line error").code, + INVALID_AGENT_RUN_RESULT, + "{error}" + ); + } + } + fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { let config: FabricConfig = serde_json::from_value(serde_json::json!({ "schema_version": "fabric.agent/v1alpha1", diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 3fa5363a9..a7dce1079 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -987,7 +987,8 @@ The file holds a start line and an end line. Every line carries `schema` raises instead of returning a result, the end line reads `failed`. An adapter host error keeps its code, such as `host_timeout`, `host_crashed`, `host_io`, or `invalid_agent_run_result`. A result extension that the - adapter's descriptor does not declare also reads `invalid_agent_run_result`. + adapter's descriptor does not declare, or that breaks its declared schema, + also reads `invalid_agent_run_result`. A failed write of `adapter-invocation.json` or a result artifact reads `artifact_write_failed`, and any other error reads `fabric_error`. From fd56894dec8ba98754b1c03e0cac6e8225dd11fc Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 10:05:39 -0700 Subject: [PATCH 052/159] docs: mention sample lines in the RecordConfig description The record also gets sample lines while the adapter host works on the request; the description feeds the agent and run plan schemas. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 6 ++++-- .../nemo-fabric-core/config/struct-recordconfig.mdx | 2 +- schemas/sdk/agent.schema.json | 2 +- schemas/sdk/run-plan.schema.json | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 5cf456ae5..26adbd1a2 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -1074,8 +1074,10 @@ impl RuntimeConfig { /// Per-invocation run record settings. /// -/// When enabled, core appends a start line and an end line for each local -/// invocation to `/.fabric///run-record.jsonl`. +/// When enabled, core appends a start line, sample lines about every 10 +/// seconds while the adapter host works on the request, and an end line for +/// each local invocation to +/// `/.fabric///run-record.jsonl`. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RecordConfig { diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx index b72fc30ee..97ff3f08d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx @@ -13,7 +13,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Per-invocation run record settings. -When enabled, core appends a start line and an end line for each local invocation to `/.fabric///run-record.jsonl`. +When enabled, core appends a start line, sample lines about every 10 seconds while the adapter host works on the request, and an end line for each local invocation to `/.fabric///run-record.jsonl`. ## Fields diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index 6ba4ad3ee..54648362e 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -560,7 +560,7 @@ }, "RecordConfig": { "additionalProperties": false, - "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line, sample lines about every 10\nseconds while the adapter host works on the request, and an end line for\neach local invocation to\n`/.fabric///run-record.jsonl`.", "properties": { "enabled": { "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index 03499b493..504966e9a 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -1928,7 +1928,7 @@ }, "RecordConfig": { "additionalProperties": false, - "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line, sample lines about every 10\nseconds while the adapter host works on the request, and an end line for\neach local invocation to\n`/.fabric///run-record.jsonl`.", "properties": { "enabled": { "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", From bc75c3982c4d4b38fd4af86cba16765b8ad2e010 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 14:22:11 -0700 Subject: [PATCH 053/159] docs: regenerate references after merging main Regenerate the Rust library reference with Rust 1.94.0 so it lists main's environment lifecycle items next to the resource snapshot items, with updated page positions. Signed-off-by: mnajafian-nv --- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 70 ++++++++++++++++++- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../resource-snapshot/index.mdx | 2 +- .../runtime/enum-errorstage.mdx | 2 +- .../enum-openaichatcompletionchunkobject.mdx | 2 +- .../runtime/enum-openaistreamhost.mdx | 2 +- .../runtime/enum-openaistreamprofile.mdx | 2 +- .../enum-openaistreamprotocolversion.mdx | 2 +- .../runtime/enum-openaistreamrecord.mdx | 2 +- .../runtime/enum-runstatus.mdx | 2 +- .../runtime/enum-serviceownership.mdx | 2 +- .../runtime/fn-attach-service.mdx | 2 +- .../runtime/fn-invoke-openai-stream.mdx | 2 +- .../runtime/fn-invoke-runtime.mdx | 2 +- .../runtime/fn-prepare-service.mdx | 2 +- .../runtime/fn-release-service.mdx | 2 +- .../nemo-fabric-core/runtime/fn-run-plan.mdx | 2 +- .../runtime/fn-runtime-resource-snapshot.mdx | 2 +- .../runtime/fn-start-runtime-in.mdx | 2 +- .../runtime/fn-start-runtime-with-service.mdx | 2 +- .../runtime/fn-start-runtime.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 3 +- .../runtime/struct-environmenthandle.mdx | 4 +- .../runtime/struct-errorinfo.mdx | 2 +- .../runtime/struct-fabricevent.mdx | 2 +- .../runtime/struct-invocationhandle.mdx | 2 +- .../struct-openaichatcompletionchunk.mdx | 2 +- ...struct-openaichatcompletionchunkchoice.mdx | 2 +- .../struct-openaichatcompletionchunkdelta.mdx | 2 +- .../runtime/struct-openaistreaminvocation.mdx | 2 +- .../runtime/struct-openaistreamsink.mdx | 2 +- .../runtime/struct-openaistreamtransport.mdx | 2 +- .../runtime/struct-runrequest.mdx | 2 +- .../runtime/struct-runresult.mdx | 2 +- .../runtime/struct-runtimecontext.mdx | 2 +- .../runtime/struct-runtimehandle.mdx | 2 +- .../struct-runtimetelemetrycontext.mdx | 2 +- .../runtime/struct-runusage.mdx | 2 +- .../runtime/struct-servicehandle.mdx | 2 +- .../runtime/struct-servicereference.mdx | 2 +- .../runtime/struct-telemetryref.mdx | 2 +- .../schema/enum-schemaname.mdx | 2 +- .../nemo-fabric-core/schema/index.mdx | 2 +- 50 files changed, 120 insertions(+), 51 deletions(-) diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 6c276ae6b..71771291a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 127 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index fa45b1f21..a6a947127 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 128 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 218c1dde6..630b1a01b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 129 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index 7a918b620..936364ce9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 130 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index f0f77cf94..916915ab1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 131 +position: 135 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 75900423a..fe01749e7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -497,6 +497,42 @@ Environment provider. Adapter kind. +### `UnsupportedEnvironmentRelease` + +
+ +An environment provider does not implement release. + +#### Fields + +##### `provider: String` + +Environment provider. + +### `EnvironmentProviderOperation` + +
+ +An environment provider process could not complete an operation. + +#### Fields + +##### `provider: String` + +Environment provider. + +##### `operation: String` + +Lifecycle operation. + +##### `code: String` + +Stable provider or host error code. + +##### `message: String` + +Human-readable message that excludes credentials and raw diagnostics. + ### `EnvironmentHandleRequired`
@@ -533,6 +569,38 @@ Actual value from the environment handle. Environment handle id. +### `ResourceObservationUnavailable` + +
+ +A runtime's resources cannot be read now: the runtime stopped, is stopping, its adapter host process exited, a live process in its tree or the process table cannot be read, or the process tree kept changing during the read. + +#### Fields + +##### `runtime_id: String` + +Runtime handle id. + +##### `reason: String` + +Why the runtime cannot be observed. + +### `ResourceObservationUnsupported` + +
+ +The runtime's adapter kind, environment provider, or operating system has no resource reader. + +#### Fields + +##### `runtime_id: String` + +Runtime handle id. + +##### `reason: String` + +What has no resource reader. + ### `InvalidProcessSettings`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index 0bf1ac774..ecc3b0451 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 134 +position: 136 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 6b00f4d18..da30f61c4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 136 +position: 140 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index affbe0d0f..a0348db94 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -3,7 +3,7 @@ title: "Module resource_snapshot" sidebar-title: "resource_snapshot" slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" description: "Resource snapshots of a runtime's local adapter host and its descendants." -position: 133 +position: 137 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx index eb802320b..cdaccd776 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx @@ -2,7 +2,7 @@ title: "Enum Error Stage" sidebar-title: "ErrorStage" description: "NeMo Fabric lifecycle stage associated with an error." -position: 23 +position: 24 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx index cca22a793..83505d3b2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Chat Completion Chunk Object" sidebar-title: "OpenAiChatCompletionChunkObject" description: "Exact OpenAI object discriminator accepted by the v1 chunk profile." -position: 24 +position: 25 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx index f2d130b85..5e337a0e0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Host" sidebar-title: "OpenAiStreamHost" description: "Supported native-streaming listener host." -position: 25 +position: 26 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx index 1c3f839ae..eb3f8fe51 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Profile" sidebar-title: "OpenAiStreamProfile" description: "Supported OpenAI-compatible chunk profile." -position: 26 +position: 27 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx index 86fddd94c..ee69f77c3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Protocol Version" sidebar-title: "OpenAiStreamProtocolVersion" description: "Supported southbound native-streaming protocol version." -position: 27 +position: 28 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx index a3a933cdf..d02adc8bf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Record" sidebar-title: "OpenAiStreamRecord" description: "One correlated NDJSON record on the adapter-native stream channel." -position: 28 +position: 29 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx index 97954ce13..f7acd827c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx @@ -2,7 +2,7 @@ title: "Enum RunStatus" sidebar-title: "RunStatus" description: "Runtime completion status." -position: 29 +position: 30 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx index c5050a301..310cf8983 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx @@ -2,7 +2,7 @@ title: "Enum Service Ownership" sidebar-title: "ServiceOwnership" description: "Ownership of a service exposed through NeMo Fabric." -position: 30 +position: 31 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx index d8ff25825..494089eb6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx @@ -2,7 +2,7 @@ title: "Function attach_service" sidebar-title: "attach_service" description: "Attach to a caller-owned long-lived service for a resolved run plan." -position: 31 +position: 33 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx index b81268897..1fa28e020 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx @@ -2,7 +2,7 @@ title: "Function invoke_openai_stream" sidebar-title: "invoke_openai_stream" description: "Invoke a started harness runtime and pass through native OpenAI chat-completion chunks." -position: 32 +position: 34 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx index e1471de73..28b005c5d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx @@ -2,7 +2,7 @@ title: "Function invoke_runtime" sidebar-title: "invoke_runtime" description: "Invoke a started harness runtime." -position: 33 +position: 35 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx index a59f0067c..49e08a5bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx @@ -2,7 +2,7 @@ title: "Function prepare_service" sidebar-title: "prepare_service" description: "Prepare a Fabric-owned long-lived service for a resolved run plan." -position: 35 +position: 37 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx index b356d2db0..72ca1b83e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx @@ -2,7 +2,7 @@ title: "Function release_service" sidebar-title: "release_service" description: "Release a Fabric-owned service or detach from a caller-owned service." -position: 36 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 55a5a47cb..331c08b3c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -2,7 +2,7 @@ title: "Function run_plan" sidebar-title: "run_plan" description: "Invoke a NeMo Fabric run plan." -position: 37 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx index 4275ce2f6..87bb90002 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-runtime-resource-snapshot.mdx @@ -2,7 +2,7 @@ title: "Function runtime_resource_snapshot" sidebar-title: "runtime_resource_snapshot" description: "Read a timestamped resource snapshot of a started runtime." -position: 38 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-in.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-in.mdx index d31455b12..ccec560e4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-in.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-in.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_in" sidebar-title: "start_runtime_in" description: "Start a runtime in an explicitly prepared environment." -position: 42 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx index c61b166c3..96b3675dc 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_with_service" sidebar-title: "start_runtime_with_service" description: "Start a runtime connected to an active service." -position: 40 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx index 5b10ad19a..0d7562938 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime" sidebar-title: "start_runtime" description: "Start or connect to a harness runtime in its configured local environment." -position: 41 +position: 42 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index a55e5bd91..77db99a02 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 41 +position: 45 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 5fa3f62d9..acfe954c9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 135 +position: 138 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -59,6 +59,7 @@ Runtime invocation helpers. - [release_environment](fn-release-environment.mdx): Release or detach an environment through its provider. - [release_service](fn-release-service.mdx): Release a Fabric-owned service or detach from a caller-owned service. - [run_plan](fn-run-plan.mdx): Invoke a NeMo Fabric run plan. +- [runtime_resource_snapshot](fn-runtime-resource-snapshot.mdx): Read a timestamped resource snapshot of a started runtime. - [start_runtime](fn-start-runtime.mdx): Start or connect to a harness runtime in its configured local environment. - [start_runtime_in](fn-start-runtime-in.mdx): Start a runtime in an explicitly prepared environment. - [start_runtime_with_service](fn-start-runtime-with-service.mdx): Start a runtime connected to an active service. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx index 55d47378d..67102fa82 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx @@ -29,11 +29,11 @@ Where NeMo Fabric control code runs. ### `workspace: Option` -Workspace visible to the harness runtime. +Workspace visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `artifacts: Option` -Artifact root visible to the harness runtime. +Artifact root visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `env: BTreeMap` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx index 9468739a0..376449e6a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx @@ -2,7 +2,7 @@ title: "Struct Error Info" sidebar-title: "ErrorInfo" description: "Normalized error metadata." -position: 5 +position: 6 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx index 5330e632a..5ec70a8ad 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx @@ -2,7 +2,7 @@ title: "Struct Fabric Event" sidebar-title: "FabricEvent" description: "NeMo Fabric lifecycle or progress event." -position: 6 +position: 7 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx index 3f3ac0d42..1df9629e6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx @@ -2,7 +2,7 @@ title: "Struct Invocation Handle" sidebar-title: "InvocationHandle" description: "One request sent to a runtime." -position: 7 +position: 8 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx index 026b2de72..96e965402 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk" sidebar-title: "OpenAiChatCompletionChunk" description: "OpenAI Chat Completions chunk accepted by the native streaming profile." -position: 8 +position: 9 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx index 94b2e70ee..f3dbf5424 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Choice" sidebar-title: "OpenAiChatCompletionChunkChoice" description: "One choice within an OpenAI Chat Completions streaming chunk." -position: 9 +position: 10 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx index 845afed45..53b0a6972 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Delta" sidebar-title: "OpenAiChatCompletionChunkDelta" description: "Incremental assistant message fields carried by one OpenAI choice." -position: 10 +position: 11 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx index 49b4c9c49..b5a3b7fcb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Invocation" sidebar-title: "OpenAiStreamInvocation" description: "One adapter-native OpenAI streaming invocation." -position: 11 +position: 12 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx index 8ad3827d5..a178c8113 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Sink" sidebar-title: "OpenAiStreamSink" description: "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric." -position: 12 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx index 2ed1e5c62..4a7f2e2f9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Transport" sidebar-title: "OpenAiStreamTransport" description: "SDK-owned loopback transport for one native OpenAI streaming invocation." -position: 13 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index 8dcc6950f..d0200756d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -2,7 +2,7 @@ title: "Struct RunRequest" sidebar-title: "RunRequest" description: "A request passed to a NeMo Fabric-managed harness runtime." -position: 14 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 3e05964a5..acde0de61 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -2,7 +2,7 @@ title: "Struct RunResult" sidebar-title: "RunResult" description: "Result from a NeMo Fabric-managed harness invocation." -position: 15 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 28af565b2..625527702 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Context" sidebar-title: "RuntimeContext" description: "Context generated for one invocation of a started runtime." -position: 17 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx index b0392902d..0671b565e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Handle" sidebar-title: "RuntimeHandle" description: "Active or resumable harness runtime." -position: 18 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx index 1525ed89d..890abc95b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Telemetry Context" sidebar-title: "RuntimeTelemetryContext" description: "Runtime telemetry config passed to adapters." -position: 19 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx index 6f16e387b..864be9b27 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx @@ -2,7 +2,7 @@ title: "Struct RunUsage" sidebar-title: "RunUsage" description: "Normalized invocation usage exposed to NeMo Fabric consumers." -position: 16 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx index 8c36ff5de..dead906b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Service Handle" sidebar-title: "ServiceHandle" description: "Public identity and binding for one prepared or attached service." -position: 20 +position: 21 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx index e28440353..4a249074a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx @@ -2,7 +2,7 @@ title: "Struct Service Reference" sidebar-title: "ServiceReference" description: "Caller-supplied reference to an already-running service." -position: 21 +position: 22 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx index 2d2bc6240..15d28ef49 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Ref" sidebar-title: "TelemetryRef" description: "Reference to telemetry emitted by Relay or another configured telemetry path." -position: 22 +position: 23 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index e45e2090f..38c553dc6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -187,7 +187,7 @@ Runtime resource snapshot schema. #### `ALL` -
22]"}} />
+
23]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 45c3ca38e..ccd80095b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 136 +position: 139 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} From c233cf43dfac52833dfe7e225cc6879e365062b5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 14:25:39 -0700 Subject: [PATCH 054/159] test: reject the run record on the OpenShell provider Main now ships OpenShell as a real environment provider whose task paths pass config validation. Check that an OpenShell environment with valid task paths still fails planning when the run record is on, next to the existing docker case. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 111ede01d..49abb07b2 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -7479,18 +7479,25 @@ mod tests { #[test] fn record_enabled_with_a_non_local_provider_fails_validation() { - let mut config = typed_config("nvidia.fabric.hermes"); - config.runtime.artifacts = Some(PathBuf::from("runs")); - config.runtime.record = Some(RecordConfig { enabled: true }); - config.environment.as_mut().expect("environment").provider = "docker".to_string(); + for provider in ["docker", "openshell"] { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + let environment = config.environment.as_mut().expect("environment"); + environment.provider = provider.to_string(); + // Valid OpenShell task paths, so only the record check can fail. + environment.workspace = Some(PathBuf::from("/sandbox/workspace")); + environment.artifacts = Some(PathBuf::from("/sandbox/artifacts")); - assert_eq!( - record_error(&config), - Some(( - "runtime.record.enabled".to_string(), - "supported only with the local environment provider".to_string() - )) - ); + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "supported only with the local environment provider".to_string() + )), + "{provider}" + ); + } } #[test] From caa72f5fdb408d5534882aa259b1fdd67b5f6d04 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 14:26:37 -0700 Subject: [PATCH 055/159] docs: regenerate references after merging main Regenerate the Rust library reference with Rust 1.94.0 so it lists main's environment lifecycle items next to the run record items, with updated page positions. Signed-off-by: mnajafian-nv --- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 38 ++++++++++++++++++- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../runtime/enum-errorstage.mdx | 2 +- .../enum-openaichatcompletionchunkobject.mdx | 2 +- .../runtime/enum-openaistreamhost.mdx | 2 +- .../runtime/enum-openaistreamprofile.mdx | 2 +- .../enum-openaistreamprotocolversion.mdx | 2 +- .../runtime/enum-openaistreamrecord.mdx | 2 +- .../runtime/enum-runstatus.mdx | 2 +- .../runtime/enum-serviceownership.mdx | 2 +- .../runtime/fn-attach-service.mdx | 2 +- .../runtime/fn-invoke-openai-stream.mdx | 2 +- .../runtime/fn-invoke-runtime.mdx | 2 +- .../runtime/fn-prepare-service.mdx | 2 +- .../runtime/fn-release-service.mdx | 2 +- .../nemo-fabric-core/runtime/fn-run-plan.mdx | 2 +- .../runtime/fn-start-runtime-with-service.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 2 +- .../runtime/struct-environmenthandle.mdx | 4 +- .../runtime/struct-errorinfo.mdx | 2 +- .../runtime/struct-fabricevent.mdx | 2 +- .../runtime/struct-invocationhandle.mdx | 2 +- .../struct-openaichatcompletionchunk.mdx | 2 +- ...struct-openaichatcompletionchunkchoice.mdx | 2 +- .../struct-openaichatcompletionchunkdelta.mdx | 2 +- .../runtime/struct-openaistreaminvocation.mdx | 2 +- .../runtime/struct-openaistreamsink.mdx | 2 +- .../runtime/struct-openaistreamtransport.mdx | 2 +- .../runtime/struct-runrequest.mdx | 2 +- .../runtime/struct-runresult.mdx | 2 +- .../runtime/struct-runtimecontext.mdx | 2 +- .../runtime/struct-runtimehandle.mdx | 2 +- .../struct-runtimetelemetrycontext.mdx | 2 +- .../runtime/struct-runusage.mdx | 2 +- .../runtime/struct-servicehandle.mdx | 2 +- .../runtime/struct-servicereference.mdx | 2 +- .../runtime/struct-telemetryref.mdx | 2 +- .../schema/enum-schemaname.mdx | 2 +- .../nemo-fabric-core/schema/index.mdx | 2 +- 46 files changed, 83 insertions(+), 47 deletions(-) diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index feabb8df6..b0aba3459 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 126 +position: 130 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 4c4e21b44..e62476ee1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 127 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index de4682271..74aa25a6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 128 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index 7f45f6f6e..359d8fe9e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 129 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index 4f6f26b00..548c11e07 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 130 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index d504f49ef..6462f4179 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -497,6 +497,42 @@ Environment provider. Adapter kind. +### `UnsupportedEnvironmentRelease` + +
+ +An environment provider does not implement release. + +#### Fields + +##### `provider: String` + +Environment provider. + +### `EnvironmentProviderOperation` + +
+ +An environment provider process could not complete an operation. + +#### Fields + +##### `provider: String` + +Environment provider. + +##### `operation: String` + +Lifecycle operation. + +##### `code: String` + +Stable provider or host error code. + +##### `message: String` + +Human-readable message that excludes credentials and raw diagnostics. + ### `EnvironmentHandleRequired`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index 0bf1ac774..239052d7c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 134 +position: 135 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index f9c667fa9..3a81be390 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 134 +position: 138 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx index eb802320b..cdaccd776 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx @@ -2,7 +2,7 @@ title: "Enum Error Stage" sidebar-title: "ErrorStage" description: "NeMo Fabric lifecycle stage associated with an error." -position: 23 +position: 24 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx index cca22a793..83505d3b2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Chat Completion Chunk Object" sidebar-title: "OpenAiChatCompletionChunkObject" description: "Exact OpenAI object discriminator accepted by the v1 chunk profile." -position: 24 +position: 25 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx index f2d130b85..5e337a0e0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Host" sidebar-title: "OpenAiStreamHost" description: "Supported native-streaming listener host." -position: 25 +position: 26 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx index 1c3f839ae..eb3f8fe51 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Profile" sidebar-title: "OpenAiStreamProfile" description: "Supported OpenAI-compatible chunk profile." -position: 26 +position: 27 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx index 86fddd94c..ee69f77c3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Protocol Version" sidebar-title: "OpenAiStreamProtocolVersion" description: "Supported southbound native-streaming protocol version." -position: 27 +position: 28 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx index a3a933cdf..d02adc8bf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Record" sidebar-title: "OpenAiStreamRecord" description: "One correlated NDJSON record on the adapter-native stream channel." -position: 28 +position: 29 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx index 97954ce13..f7acd827c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx @@ -2,7 +2,7 @@ title: "Enum RunStatus" sidebar-title: "RunStatus" description: "Runtime completion status." -position: 29 +position: 30 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx index c5050a301..310cf8983 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx @@ -2,7 +2,7 @@ title: "Enum Service Ownership" sidebar-title: "ServiceOwnership" description: "Ownership of a service exposed through NeMo Fabric." -position: 30 +position: 31 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx index d8ff25825..494089eb6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx @@ -2,7 +2,7 @@ title: "Function attach_service" sidebar-title: "attach_service" description: "Attach to a caller-owned long-lived service for a resolved run plan." -position: 31 +position: 33 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx index b81268897..1fa28e020 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx @@ -2,7 +2,7 @@ title: "Function invoke_openai_stream" sidebar-title: "invoke_openai_stream" description: "Invoke a started harness runtime and pass through native OpenAI chat-completion chunks." -position: 32 +position: 34 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx index e1471de73..28b005c5d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx @@ -2,7 +2,7 @@ title: "Function invoke_runtime" sidebar-title: "invoke_runtime" description: "Invoke a started harness runtime." -position: 33 +position: 35 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx index a59f0067c..49e08a5bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx @@ -2,7 +2,7 @@ title: "Function prepare_service" sidebar-title: "prepare_service" description: "Prepare a Fabric-owned long-lived service for a resolved run plan." -position: 35 +position: 37 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx index b356d2db0..72ca1b83e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx @@ -2,7 +2,7 @@ title: "Function release_service" sidebar-title: "release_service" description: "Release a Fabric-owned service or detach from a caller-owned service." -position: 36 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 55a5a47cb..331c08b3c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -2,7 +2,7 @@ title: "Function run_plan" sidebar-title: "run_plan" description: "Invoke a NeMo Fabric run plan." -position: 37 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx index 547b47760..5acac20bd 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_with_service" sidebar-title: "start_runtime_with_service" description: "Start a runtime connected to an active service." -position: 39 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index 71febb037..ce8835391 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 40 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 5fa3f62d9..2cc36bc2f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 135 +position: 136 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx index 55d47378d..67102fa82 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx @@ -29,11 +29,11 @@ Where NeMo Fabric control code runs. ### `workspace: Option` -Workspace visible to the harness runtime. +Workspace visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `artifacts: Option` -Artifact root visible to the harness runtime. +Artifact root visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `env: BTreeMap` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx index 9468739a0..376449e6a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx @@ -2,7 +2,7 @@ title: "Struct Error Info" sidebar-title: "ErrorInfo" description: "Normalized error metadata." -position: 5 +position: 6 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx index 5330e632a..5ec70a8ad 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx @@ -2,7 +2,7 @@ title: "Struct Fabric Event" sidebar-title: "FabricEvent" description: "NeMo Fabric lifecycle or progress event." -position: 6 +position: 7 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx index 3f3ac0d42..1df9629e6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx @@ -2,7 +2,7 @@ title: "Struct Invocation Handle" sidebar-title: "InvocationHandle" description: "One request sent to a runtime." -position: 7 +position: 8 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx index 026b2de72..96e965402 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk" sidebar-title: "OpenAiChatCompletionChunk" description: "OpenAI Chat Completions chunk accepted by the native streaming profile." -position: 8 +position: 9 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx index 94b2e70ee..f3dbf5424 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Choice" sidebar-title: "OpenAiChatCompletionChunkChoice" description: "One choice within an OpenAI Chat Completions streaming chunk." -position: 9 +position: 10 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx index 845afed45..53b0a6972 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Delta" sidebar-title: "OpenAiChatCompletionChunkDelta" description: "Incremental assistant message fields carried by one OpenAI choice." -position: 10 +position: 11 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx index 49b4c9c49..b5a3b7fcb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Invocation" sidebar-title: "OpenAiStreamInvocation" description: "One adapter-native OpenAI streaming invocation." -position: 11 +position: 12 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx index 8ad3827d5..a178c8113 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Sink" sidebar-title: "OpenAiStreamSink" description: "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric." -position: 12 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx index 2ed1e5c62..4a7f2e2f9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Transport" sidebar-title: "OpenAiStreamTransport" description: "SDK-owned loopback transport for one native OpenAI streaming invocation." -position: 13 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index 8dcc6950f..d0200756d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -2,7 +2,7 @@ title: "Struct RunRequest" sidebar-title: "RunRequest" description: "A request passed to a NeMo Fabric-managed harness runtime." -position: 14 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 1cb5bd052..f74b804b8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -2,7 +2,7 @@ title: "Struct RunResult" sidebar-title: "RunResult" description: "Result from a NeMo Fabric-managed harness invocation." -position: 15 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 28af565b2..625527702 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Context" sidebar-title: "RuntimeContext" description: "Context generated for one invocation of a started runtime." -position: 17 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx index b0392902d..0671b565e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Handle" sidebar-title: "RuntimeHandle" description: "Active or resumable harness runtime." -position: 18 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx index 1525ed89d..890abc95b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Telemetry Context" sidebar-title: "RuntimeTelemetryContext" description: "Runtime telemetry config passed to adapters." -position: 19 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx index 6f16e387b..864be9b27 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx @@ -2,7 +2,7 @@ title: "Struct RunUsage" sidebar-title: "RunUsage" description: "Normalized invocation usage exposed to NeMo Fabric consumers." -position: 16 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx index 8c36ff5de..dead906b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Service Handle" sidebar-title: "ServiceHandle" description: "Public identity and binding for one prepared or attached service." -position: 20 +position: 21 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx index e28440353..4a249074a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx @@ -2,7 +2,7 @@ title: "Struct Service Reference" sidebar-title: "ServiceReference" description: "Caller-supplied reference to an already-running service." -position: 21 +position: 22 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx index 2d2bc6240..15d28ef49 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Ref" sidebar-title: "TelemetryRef" description: "Reference to telemetry emitted by Relay or another configured telemetry path." -position: 22 +position: 23 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index a0442791d..282ef841d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -187,7 +187,7 @@ One line of an opt-in per-invocation run record, version `v1alpha1`. #### `ALL` -
22]"}} />
+
23]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 45c3ca38e..52a9671f8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 136 +position: 137 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} From 874731d52141eeed91f9723a971e999e8431f722 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 16 Oct 2026 14:36:00 -0700 Subject: [PATCH 056/159] docs: regenerate references after merging main Regenerate the Rust library reference with Rust 1.94.0 after the merges: the config index lists RecordConfig, SchemaName::ALL shows 24 entries, and page positions follow main's environment lifecycle items. Signed-off-by: mnajafian-nv --- .../nemo-fabric-core/adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../nemo-fabric-core/agent-execution/index.mdx | 2 +- .../rust-library-reference/nemo-fabric-core/config/index.mdx | 3 ++- .../rust-library-reference/nemo-fabric-core/doctor/index.mdx | 2 +- .../nemo-fabric-core/error/enum-fabricerror.mdx | 2 +- .../rust-library-reference/nemo-fabric-core/error/index.mdx | 2 +- .../api/rust-library-reference/nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/resource-snapshot/index.mdx | 2 +- .../rust-library-reference/nemo-fabric-core/runtime/index.mdx | 2 +- .../nemo-fabric-core/schema/enum-schemaname.mdx | 2 +- .../rust-library-reference/nemo-fabric-core/schema/index.mdx | 2 +- 12 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 71771291a..3ba18310c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 131 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index a6a947127..a7c60bd2d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 132 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 630b1a01b..782999cf5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 133 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index 936364ce9..5159ab6bd 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 134 +position: 135 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -53,6 +53,7 @@ NeMo Fabric config models and loading helpers. - [McpServerPlan](struct-mcpserverplan.mdx): Resolved MCP server exposure. - [MetadataConfig](struct-metadataconfig.mdx): Human-readable metadata. - [ModelConfig](struct-modelconfig.mdx): Model configuration. +- [RecordConfig](struct-recordconfig.mdx): Per-invocation run record settings. - [RelayAtifConfig](struct-relayatifconfig.mdx): Relay ATIF export configuration. - [RelayAtofConfig](struct-relayatofconfig.mdx): Relay ATOF export configuration. - [RelayComponentConfig](struct-relaycomponentconfig.mdx): Generic NeMo Relay plugin component configuration. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index 916915ab1..84cffbff2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 135 +position: 136 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index fe01749e7..f1e2fe312 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -2,7 +2,7 @@ title: "Enum Fabric Error" sidebar-title: "FabricError" description: "Errors raised by NeMo Fabric config loading and validation." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index ecc3b0451..cf7a42463 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 136 +position: 137 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index da30f61c4..c450d80cb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 140 +position: 141 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index a0348db94..48167fea9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -3,7 +3,7 @@ title: "Module resource_snapshot" sidebar-title: "resource_snapshot" slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" description: "Resource snapshots of a runtime's local adapter host and its descendants." -position: 137 +position: 138 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index acfe954c9..2b8e9d342 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 138 +position: 139 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index d4789a3c6..4c3932563 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -194,7 +194,7 @@ Runtime resource snapshot schema. #### `ALL` -
23]"}} />
+
24]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index ccd80095b..de3b12928 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 139 +position: 140 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} From 1619b1d379c6e3057ab66d9c5fd003d4c73366a3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sat, 17 Oct 2026 21:49:08 -0700 Subject: [PATCH 057/159] test: build race-test fixtures before the sampling window opens The two in-flight sample tests resolved the run plan, created folders and built the result while the blocked sample read waited 10 s for its release. On a slow file system that work outran the handshake and the tests failed with a timeout. Both tests now prepare the fixture, folders and result before the first record starts, and keep the same race and assertions. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 59 ++++++++++++++------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 48e60beee..01a946912 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1128,13 +1128,23 @@ mod tests { ticker: &Arc, read: ReadResources, ) -> (PathBuf, InvocationRecorder) { - let (plan, runtime, invocation) = record_test_fixture(); - let folder = temp_folder(name); + start_recorder_in(temp_folder(name), &record_test_fixture(), ticker, read) + } + + /// Start a record in `folder` from a fixture the test already built, so + /// a race test does no plan resolution or folder setup while a blocked + /// sample read waits for it. + fn start_recorder_in( + folder: PathBuf, + (plan, runtime, invocation): &(RunPlan, RuntimeHandle, InvocationHandle), + ticker: &Arc, + read: ReadResources, + ) -> (PathBuf, InvocationRecorder) { let request = RunRequest::text("input"); let recorder = InvocationRecorder::start(RecordStart { - plan: &plan, - runtime: &runtime, - invocation: &invocation, + plan, + runtime, + invocation, request: &request, fabric_home: &folder, timeout: Duration::from_secs(5), @@ -1254,21 +1264,28 @@ mod tests { #[test] fn a_sample_read_in_flight_when_the_record_ends_writes_nothing_after_the_end_line() { let ticker = SampleTicker::new(Duration::from_millis(10)); + // Build everything before the first record starts: once it does, the + // blocked sample read waits only HANDSHAKE_TIMEOUT for the release. + let fixture = record_test_fixture(); + let result = succeeded_result(&fixture.2); + let folder = temp_folder("in-flight"); + let sentinel_folder = temp_folder("in-flight-sentinel"); // The first sample read waits until the test has finished the record. let (read, entered, release) = read_blocked_at_first_sample(); - let (folder, recorder) = start_recorder("in-flight", &ticker, read); + let (folder, recorder) = start_recorder_in(folder, &fixture, &ticker, read); // A second record on the same ticker, sampled after the first in // each tick, shows when the blocked tick has finished. - let (sentinel_folder, sentinel) = - start_recorder("in-flight-sentinel", &ticker, Arc::new(|| Ok(snapshot()))); + let (sentinel_folder, sentinel) = start_recorder_in( + sentinel_folder, + &fixture, + &ticker, + Arc::new(|| Ok(snapshot())), + ); let path = folder.join(RECORD_FILE_NAME); let sentinel_path = sentinel_folder.join(RECORD_FILE_NAME); wait_until_entered(&entered); - let (_, _, invocation) = record_test_fixture(); - recorder - .finish(Ok(succeeded_result(&invocation))) - .expect("result"); + recorder.finish(Ok(result)).expect("result"); let sentinel_samples = samples(&sentinel_path).len(); release .send(()) @@ -1286,13 +1303,20 @@ mod tests { #[test] fn a_sample_read_in_flight_when_the_exchange_ends_writes_nothing_to_the_open_record() { let ticker = SampleTicker::new(Duration::from_millis(10)); + // Build everything before the first record starts: once it does, the + // blocked sample read waits only HANDSHAKE_TIMEOUT for the release. + let fixture = record_test_fixture(); + let result = succeeded_result(&fixture.2); + let folder = temp_folder("window-closed"); + let sentinel_folder = temp_folder("window-closed-sentinel"); // The first sample read waits until the test has closed the window. let (read, entered, release) = read_blocked_at_first_sample(); - let (folder, mut recorder) = start_recorder("window-closed", &ticker, read); + let (folder, mut recorder) = start_recorder_in(folder, &fixture, &ticker, read); // A second record on the same ticker, sampled after the first in // each tick, shows when the blocked tick has finished. - let (sentinel_folder, sentinel) = start_recorder( - "window-closed-sentinel", + let (sentinel_folder, sentinel) = start_recorder_in( + sentinel_folder, + &fixture, &ticker, Arc::new(|| Ok(snapshot())), ); @@ -1312,10 +1336,7 @@ mod tests { let lines = read_lines(&path); assert_eq!(lines.len(), 1, "{lines:?}"); assert_eq!(lines[0]["kind"], "start"); - let (_, _, invocation) = record_test_fixture(); - recorder - .finish(Ok(succeeded_result(&invocation))) - .expect("result"); + recorder.finish(Ok(result)).expect("result"); assert!(samples(&path).is_empty(), "{:?}", read_lines(&path)); drop(sentinel); let _ = std::fs::remove_dir_all(folder); From 9d6bc218bf4debd918ea49ad44eb8e7789fba6e4 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 15:28:16 -0700 Subject: [PATCH 058/159] fix: count every Deep Agents model call in token usage The Deep Agents adapter summed usage_metadata from the AI messages that reached graph state through the updates stream. Three cases came out wrong: - Deep Agents adds SummarizationMiddleware to every agent and subagent. Its summary call returns only text, so the summary model's message and usage never enter graph state, and an invocation that summarized under-reported input, output and total tokens. - When the invocation raised, including GraphRecursionError, the adapter returned no messages, so usage was None even though model calls had finished. - A count was summed over the calls that reported it and returned as known. ChatOpenAI also turns an omitted Chat Completions count into 0 and derives a missing total, so a partial provider report came out as measured zeros. The adapter now records usage from LangChain's chat-model callbacks, which fire for every chat-model call that runs in the adapter process: the main agent, declarative subagents and summarization. Agent Protocol subagents run remotely and are not included. A new recorder is created for each invocation, so earlier turns are never counted again. It outlives an exception, so a failed invocation still reports the fields that every counted call reported. Each token count and the cost are known only when every counted call in the invocation reported them, and an explicit zero stays a known zero. A request the provider rejects with a 4xx status (other than 408) contributes no usage. A 5xx response, timeout, cancellation or unfinished call leaves the invocation's counts and cost unknown, because the provider may have done work without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the recorder, and only the final response's reported usage counts. When a response carries OpenAI-shaped raw usage, the raw counts decide what was reported: an omitted count stays unknown, the provider's total is used when present, and a derived total counts only when input and output were both reported. The graph tests run the real Deep Agents graph and ChatOpenAI against a scripted Chat Completions endpoint for summarization, parallel task subagents, 400, 408, 503 and timeout failures, the recursion limit, partial reports and explicit zeros. Separate ChatOpenAI.with_retry tests cover 429 then success and 503 then success. The existing usage tests now produce their messages through a LangChain chat-model call so the callbacks fire. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 28 +- .../deepagents/adapter.py | 254 +++++---- tests/adapters/test_deepagents.py | 491 +++++++++++++++--- tests/e2e/test_deepagents.py | 1 - tests/integrations/test_relay_scope_leak.py | 4 +- 5 files changed, 618 insertions(+), 160 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 2090da4cd..78e94ce51 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -227,10 +227,30 @@ service. The normalized result includes the final response, buffered messages and per-step events, LangGraph thread id, token usage (and cost when the provider -reports it), and errors. Usage aggregates the current turn across the main agent -and any delegated subagents (streamed with `subgraphs=True`). Configuration and -preflight failures (a missing credential, an absent `deepagents` package, or an -invalid MCP server) fail runtime start before an invocation is accepted. +reports it), and errors. Configuration and preflight failures (a missing +credential, an absent `deepagents` package, or an invalid MCP server) fail +runtime start before an invocation is accepted. + +Usage sums every chat-model call that runs in the adapter process for the +invocation: the main agent, declarative subagents, and Deep Agents +summarization. Agent Protocol subagents (entries with `graph_id`) run in their +remote service, and their tokens are not included. A token count or cost is +reported only when every counted call reported it; otherwise it stays unknown. +This also holds for a failed invocation. A request the provider rejects with a +4xx status (other than 408) contributes no usage. A 5xx response, timeout, +cancellation or unfinished call leaves the invocation's counts and cost +unknown, because the provider may have done work without reporting usage. +OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries +to success is not visible to the adapter, and only the final response's +reported usage counts. Usage replayed from a configured LangChain model cache +would be counted as reported; Fabric configures no such cache. + +When a response carries OpenAI-shaped usage in +`response_metadata.token_usage`, as `ChatOpenAI` responses without streaming +do, the raw counts decide what was reported: an omitted count stays unknown, +and the provider's total is used when present. Other providers and streamed +responses use LangChain's `usage_metadata`, which some integrations fill with 0 +for an omitted count. ## Runtime Lifecycle diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 618957f1f..37771f419 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -21,9 +21,12 @@ from pathlib import Path from typing import Any from typing import NamedTuple +from uuid import UUID from langchain.agents.middleware import AgentMiddleware +from langchain_core.callbacks import BaseCallbackHandler from langchain_core.messages import ToolMessage +from langchain_core.outputs import LLMResult from langgraph.errors import GraphRecursionError from nemo_fabric_adapter_contract.models import AgentConfig from nemo_fabric_adapter_contract.models import AgentMcpServerConfig @@ -750,7 +753,7 @@ async def invoke( resumed=resumed, result_state=outcome.result_state, events=outcome.events or [], - turn_messages=outcome.turn_messages or [], + model_calls=outcome.model_calls or [], error=outcome.error, telemetry_runtime=telemetry_runtime, relay_artifacts=relay_artifacts, @@ -874,24 +877,26 @@ async def _invoke_agent( ) -> TurnOutcome: """Run one agent turn, normalizing an invocation failure into an error string.""" + model_usage = ModelCallUsage() try: - result_state, events, turn_messages = await invoke_compiled_agent( + result_state, events = await invoke_compiled_agent( self._agent, user_message, self._thread_id, - callbacks=callbacks, + callbacks=[*(callbacks or []), model_usage], ) except GraphRecursionError: return TurnOutcome( + model_calls=model_usage.calls(), error="Deep Agents reached its graph recursion limit.", error_code="deepagents_recursion_limit_reached", ) except Exception as exc: # normalized adapter failure - return TurnOutcome(error=_error_text(exc)) + return TurnOutcome(model_calls=model_usage.calls(), error=_error_text(exc)) return TurnOutcome( result_state=result_state, events=events, - turn_messages=turn_messages, + model_calls=model_usage.calls(), ) def _telemetry_output( @@ -966,18 +971,65 @@ def _apply_callbacks( return config +class ModelCallUsage(BaseCallbackHandler): + """Record what each chat-model call of one invocation reports. + + LangChain fires these callbacks for every chat-model call in the run, + including calls whose response never enters graph state, such as the Deep + Agents summarization call. The records live on the handler, so calls that + finished before an invocation raised are still reported. + + A call is recorded as unreported when it starts, and its usage replaces that + record when it ends. A request the provider rejects with a 4xx status (other + than 408) contributes no usage and is dropped. A 5xx response, timeout, + cancellation or unfinished call stays unreported and makes the invocation's + counts and cost unknown, because the provider may have done work without + reporting usage. + """ + + run_inline = True + + def __init__(self) -> None: + super().__init__() + self._calls: dict[UUID, dict[str, Any]] = {} + + def on_chat_model_start( + self, serialized: Any, messages: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self._calls[run_id] = {} + + def on_llm_start( + self, serialized: Any, prompts: Any, *, run_id: UUID, **kwargs: Any + ) -> None: + self._calls[run_id] = {} + + def on_llm_end(self, response: LLMResult, *, run_id: UUID, **kwargs: Any) -> None: + generations = response.generations[0] if response.generations else [] + message = getattr(generations[0], "message", None) if generations else None + self._calls[run_id] = {} if message is None else _message_to_dict(message) + + def on_llm_error( + self, error: BaseException, *, run_id: UUID, **kwargs: Any + ) -> None: + # Provider SDKs (OpenAI, Anthropic) raise errors carrying the HTTP status + # code when the provider answered with an error response. A 4xx other + # than 408 (request timeout) means the provider rejected the request. + status_code = getattr(error, "status_code", None) + if isinstance(status_code, int) and 400 <= status_code < 500: + if status_code != 408: + self._calls.pop(run_id, None) + + def calls(self) -> list[dict[str, Any]]: + return list(self._calls.values()) + + async def invoke_compiled_agent( agent: Any, user_message: str, thread_id: str | None, callbacks: list[Any] | None = None, -) -> tuple[Any, list[dict[str, Any]], list[dict[str, Any]]]: - """Run a compiled agent and return state, events, and current-turn messages. - - On a later runtime invocation the final ``values`` state also replays - prior-turn messages, so usage/cost must be aggregated from the messages - emitted *this* turn — the ``updates`` deltas — rather than the full final - state. +) -> tuple[Any, list[dict[str, Any]]]: + """Run a compiled agent and return its final state and per-step events. ``callbacks`` (e.g. the NeMo Relay callback handler) are appended to the LangGraph run config; any callbacks already present are preserved rather than @@ -992,12 +1044,10 @@ async def invoke_compiled_agent( config = config or None events: list[dict[str, Any]] = [] - turn_messages: list[Any] = [] final: Any = None # subgraphs=True surfaces subagent (delegated ``task``) steps as 3-tuples whose - # namespace identifies the subgraph. Folding their message deltas into this turn - # keeps usage/cost accurate for delegating runs; the final ``values`` state is - # taken from the main graph only (empty namespace). + # namespace identifies the subgraph, so they appear in ``events``; the final + # ``values`` state is taken from the main graph only (empty namespace). stream = ( agent.astream(inputs, config, stream_mode=["updates", "values"], subgraphs=True) if config @@ -1009,16 +1059,9 @@ async def invoke_compiled_agent( if namespace: event["subgraph"] = "/".join(str(part) for part in namespace) events.append(event) - for node_output in chunk.values(): - if isinstance(node_output, dict): - new = node_output.get("messages") - if isinstance(new, list): - turn_messages.extend(new) - elif new is not None: - turn_messages.append(new) elif mode == "values" and not namespace: final = chunk - return final, events, _dedup_messages(_messages_to_dicts(turn_messages)) + return final, events class Observability(NamedTuple): @@ -1034,7 +1077,7 @@ class TurnOutcome(NamedTuple): result_state: Any = None events: list[dict[str, Any]] | None = None - turn_messages: list[dict[str, Any]] | None = None + model_calls: list[dict[str, Any]] | None = None error: str | None = None error_code: str | None = None telemetry_error: str | None = None @@ -1151,7 +1194,7 @@ def normalize_output( resumed: bool, result_state: Any, events: list[dict[str, Any]], - turn_messages: list[dict[str, Any]], + model_calls: list[dict[str, Any]], error: str | None, telemetry_runtime: dict[str, Any] | None, relay_artifacts: list[dict[str, str]] | None, @@ -1160,9 +1203,9 @@ def normalize_output( ) -> dict[str, Any]: messages = _extract_messages(result_state) response = _final_response(messages) - # Aggregate usage/cost from this turn's messages only; the replayed final state - # replays prior-turn messages that must not be re-counted. - usage = _aggregate_usage(turn_messages) + # Usage comes from this invocation's model calls, not from the final state, + # which also replays earlier turns and omits internal calls like summarization. + usage = _aggregate_usage(model_calls) output: dict[str, Any] = { "harness": HARNESS, @@ -1210,26 +1253,6 @@ def _messages_to_dicts(raw: Any) -> list[dict[str, Any]]: return [item if isinstance(item, dict) else _message_to_dict(item) for item in raw] -def _dedup_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Drop duplicate messages by id, preserving first-seen order. - - With ``subgraphs=True`` the same message can surface in both a subgraph's - updates and the parent stream; de-duplicating by id keeps usage aggregation - from counting a message twice. Messages without an id are always kept. - """ - - seen: set[str] = set() - unique: list[dict[str, Any]] = [] - for message in messages: - message_id = message.get("id") - if isinstance(message_id, str): - if message_id in seen: - continue - seen.add(message_id) - unique.append(message) - return unique - - def _message_to_dict(message: Any) -> dict[str, Any]: role = ( getattr(message, "type", None) or getattr(message, "role", None) or "assistant" @@ -1259,56 +1282,103 @@ def _final_response(messages: list[dict[str, Any]]) -> Any: return messages[-1].get("content") if messages else None -def _aggregate_usage(messages: list[dict[str, Any]]) -> dict[str, Any] | None: +def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: + """Sum model-call usage; a count is known only when every call reported it. + + One call without a count makes the invocation's sum for that count a + partial value, so it stays unknown rather than under-reported. + """ + + if not calls: + return None totals: dict[str, Any] = {} - cost = 0.0 - has_cost = False - for message in messages: - usage = message.get("usage") if isinstance(message.get("usage"), dict) else {} - for source, target in ( - ("input_tokens", "prompt_tokens"), - ("output_tokens", "completion_tokens"), - ("total_tokens", "total_tokens"), - ): - value = usage.get(source) - if ( - isinstance(value, int) - and not isinstance(value, bool) - and 0 <= value <= (1 << 64) - 1 - ): - totals[target] = int(totals.get(target, 0)) + value - # Cost is not part of LangChain's UsageMetadata; surface it only when a - # model/provider reports it on the usage or response metadata. - candidate = ( - usage.get("total_cost") or usage.get("cost") or _metadata_cost(message) - ) - if ( - isinstance(candidate, (int, float)) - and not isinstance(candidate, bool) - and math.isfinite(candidate) - and candidate >= 0 - ): - cost += float(candidate) - has_cost = True - if has_cost: - totals["cost"] = cost + for source, target in ( + ("input_tokens", "prompt_tokens"), + ("output_tokens", "completion_tokens"), + ("total_tokens", "total_tokens"), + ): + values = [_reported_tokens(call, source) for call in calls] + if all(value is not None for value in values): + totals[target] = sum(values) + costs = [_reported_cost(call) for call in calls] + if all(cost is not None for cost in costs): + totals["cost"] = math.fsum(costs) return totals or None -def _metadata_cost(message: dict[str, Any]) -> float | None: - metadata = message.get("response_metadata") - if isinstance(metadata, dict): - value = metadata.get("cost") - if ( - isinstance(value, (int, float)) - and not isinstance(value, bool) - and math.isfinite(value) - and value >= 0 - ): - return float(value) +# Raw OpenAI Chat Completions usage keys behind each LangChain usage counter. +_OPENAI_USAGE_KEYS = { + "input_tokens": "prompt_tokens", + "output_tokens": "completion_tokens", + "total_tokens": "total_tokens", +} + + +def _reported_tokens(call: dict[str, Any], name: str) -> int | None: + """Return one call's token count, or ``None`` when the provider omitted it. + + ``ChatOpenAI`` turns an omitted Chat Completions count into 0 and derives a + missing or zero total from input plus output. When a response carries + OpenAI-shaped raw usage in ``response_metadata.token_usage``, the raw counts + are used: an omitted count is unknown, and a derived total counts only when + both of its parts were reported. Other responses use ``usage_metadata``. + """ + + usage = call.get("usage") + value = usage.get(name) if isinstance(usage, dict) else None + if not _is_token_count(value): + return None + metadata = call.get("response_metadata") + raw = metadata.get("token_usage") if isinstance(metadata, dict) else None + if not isinstance(raw, dict): + return value + reported = raw.get(_OPENAI_USAGE_KEYS[name]) + if _is_token_count(reported): + return reported + if name == "total_tokens" and reported is None: + parts = (raw.get("prompt_tokens"), raw.get("completion_tokens")) + if all(_is_token_count(part) for part in parts): + return value return None +def _is_token_count(value: Any) -> bool: + return ( + isinstance(value, int) + and not isinstance(value, bool) + and 0 <= value <= (1 << 64) - 1 + ) + + +def _reported_cost(call: dict[str, Any]) -> float | None: + """Return a provider-reported cost for one call, or ``None`` when absent. + + Cost is not part of LangChain's UsageMetadata; some providers report it on the + usage or response metadata. + """ + + usage = call.get("usage") if isinstance(call.get("usage"), dict) else {} + metadata = call.get("response_metadata") + candidates = ( + usage.get("total_cost"), + usage.get("cost"), + metadata.get("cost") if isinstance(metadata, dict) else None, + ) + for value in candidates: + if value is not None: + return float(value) if _is_cost(value) else None + return None + + +def _is_cost(value: Any) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and value >= 0 + ) + + def _supported_kwargs(callable_obj: Any, kwargs: dict[str, Any]) -> dict[str, Any]: try: signature = inspect.signature(callable_obj) diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 23304be4d..c916cb2a3 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -25,6 +25,9 @@ from unittest.mock import MagicMock import pytest +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.language_models.fake_chat_models import GenericFakeChatModel +from langchain_core.messages import AIMessage from langchain_openai import ChatOpenAI as InstalledChatOpenAI from langgraph.errors import GraphRecursionError from nemo_fabric_adapter_contract.codec import ContractValidationError @@ -34,6 +37,7 @@ from nemo_fabric_adapter_contract.models import AgentRunRequest from nemo_fabric_adapter_contract.models import AgentRunResult from nemo_fabric_adapter_contract.models import AgentRunStatus +from nemo_fabric_adapter_contract.models import AgentUsage from nemo_fabric_adapter_contract.models import McpOAuth2Config from nemo_fabric_adapter_contract.models import McpServiceAccountConfig from nemo_fabric_adapter_contract.models import RuntimeContext @@ -49,6 +53,23 @@ def _activation(*diagnostics: dict[str, object]) -> types.SimpleNamespace: ROOT = Path(__file__).resolve().parents[2] +async def call_model(config: dict[str, Any] | None, message: AIMessage) -> AIMessage: + """Return ``message`` from a real LangChain chat-model call under ``config``. + + The call fires the run's LangChain callbacks the way a Deep Agents model call + does, which is where the adapter reads usage. Test stand-ins for the Relay + handler are not LangChain handlers, so they are left out. + """ + + callbacks = [ + callback + for callback in (config or {}).get("callbacks") or [] + if isinstance(callback, BaseCallbackHandler) + ] + model = GenericFakeChatModel(messages=iter([message])) + return await model.ainvoke("prompt", config={"callbacks": callbacks}) + + def test_descriptor_declares_supported_normalized_config(): descriptor = json.loads( (ROOT / "adapters/python/deepagents/deepagents.fabric-adapter.json").read_text( @@ -192,11 +213,17 @@ async def stream( recorder["subgraphs"] = subgraphs recorder["checkpointer"] = kwargs.get("checkpointer") user = inputs["messages"][-1]["content"] - ai = { - "role": "ai", - "content": f"reply to {user}", - "usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12}, - } + ai = await call_model( + config, + AIMessage( + content=f"reply to {user}", + usage_metadata={ + "input_tokens": 5, + "output_tokens": 7, + "total_tokens": 12, + }, + ), + ) # subgraphs=True yields 3-tuples ``(namespace, mode, chunk)``; the main # graph has an empty namespace. ``updates`` carries the message produced # this turn; ``values`` is the full replayed state. @@ -1445,7 +1472,7 @@ def test_native_telemetry_requires_mapping(tmp_path, make_payload): ) -async def test_relay_disabled_adds_no_scope_or_callbacks( +async def test_relay_disabled_adds_no_scope_or_relay_callbacks( tmp_path, make_payload, fake_sdks ): # With telemetry disabled the invocation runs without a Relay scope, callback @@ -1454,7 +1481,9 @@ async def test_relay_disabled_adds_no_scope_or_callbacks( assert output["completed"] is True assert "telemetry" not in output - assert (fake_sdks["config"] or {}).get("callbacks") is None + # The only run callback is the adapter's own model-usage recorder. + callbacks = fake_sdks["config"]["callbacks"] + assert [type(callback) for callback in callbacks] == [adapter.ModelCallUsage] assert "relay-mw" not in (fake_sdks["create_kwargs"].get("middleware") or []) @@ -2070,14 +2099,19 @@ async def test_cost_is_extracted_from_response_metadata( ): import deepagents - message = { - "role": "ai", - "content": "done", - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, - "response_metadata": {"cost": 0.0025}, - } - async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): + message = await call_model( + config, + AIMessage( + content="done", + usage_metadata={ + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + }, + response_metadata={"cost": 0.0025}, + ), + ) yield ((), "updates", {"agent": {"messages": [message]}}) yield ((), "values", {"messages": [message]}) @@ -2108,6 +2142,23 @@ def test_aggregate_usage_discards_tokens_larger_than_uint64(): assert adapter._aggregate_usage([{"usage": {"input_tokens": 1 << 64}}]) is None +@pytest.mark.parametrize( + ("costs", "expected"), + [ + pytest.param([0.0, 0.0], 0.0, id="explicit-zero"), + pytest.param([0.5, 0.25], 0.75, id="every-call"), + pytest.param([0.5, None], None, id="one-call-without-cost"), + ], +) +def test_aggregate_usage_reports_cost_only_when_every_call_did(costs, expected): + calls = [ + {"usage": {"input_tokens": 1}, "response_metadata": {"cost": cost}} + for cost in costs + ] + + assert adapter._aggregate_usage(calls).get("cost") == expected + + async def test_replayed_state_usage_counts_current_turn_only( tmp_path, make_payload, monkeypatch ): @@ -2115,20 +2166,25 @@ async def test_replayed_state_usage_counts_current_turn_only( # cost must reflect only the message emitted this turn, not the replayed one. import deepagents - prior = { - "role": "ai", - "content": "prior", - "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, - "response_metadata": {"cost": 0.001}, - } - current = { - "role": "ai", - "content": "now", - "usage": {"input_tokens": 2, "output_tokens": 2, "total_tokens": 4}, - "response_metadata": {"cost": 0.002}, - } + prior = AIMessage( + content="prior", + usage_metadata={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + response_metadata={"cost": 0.001}, + ) async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): + current = await call_model( + config, + AIMessage( + content="now", + usage_metadata={ + "input_tokens": 2, + "output_tokens": 2, + "total_tokens": 4, + }, + response_metadata={"cost": 0.002}, + ), + ) # Only the current turn's message is emitted as an update... yield ((), "updates", {"agent": {"messages": [current]}}) # ...but the final state also replays the prior turn. @@ -2427,47 +2483,360 @@ def test_deepagents_passthrough_rejects_fabric_owned_option(): adapter._validated_deepagents_settings({"model": "untrusted-model"}) -async def test_subagent_usage_folded_from_subgraph(tmp_path, make_payload, monkeypatch): - # Usage from a delegated subagent is emitted under a subgraph namespace; folding - # it into this turn keeps usage/cost accurate. Duplicate ids are counted once. - import deepagents - - sub_ai = { - "role": "ai", - "content": "subagent work", - "id": "sub-1", - "usage": {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, - } - main_ai = { - "role": "ai", - "content": "final", - "id": "main-1", - "usage": {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5}, +def chat_completion( + usage: dict[str, Any] | None, + *, + tool_calls: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Return an OpenAI Chat Completions response body with the given usage.""" + + message: dict[str, Any] = {"role": "assistant", "content": "done"} + if tool_calls: + message["content"] = None + message["tool_calls"] = tool_calls + body: dict[str, Any] = { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 0, + "model": "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": "tool_calls" if tool_calls else "stop", + } + ], } + if usage is not None: + body["usage"] = usage + return body - async def astream(inputs, config=None, *, stream_mode=None, subgraphs=False): - assert subgraphs is True - # subagent step under a subgraph namespace, emitted twice (dedup by id) - yield (("task:researcher",), "updates", {"agent": {"messages": [sub_ai]}}) - yield (("task:researcher",), "updates", {"agent": {"messages": [sub_ai]}}) - # main graph step + replayed final state - yield ((), "updates", {"agent": {"messages": [main_ai]}}) - yield ((), "values", {"messages": [main_ai]}) - agent = MagicMock() - agent.astream = astream - monkeypatch.setattr(deepagents, "create_deep_agent", MagicMock(return_value=agent)) +LS_TOOL_CALL = { + "id": "call-1", + "type": "function", + "function": {"name": "ls", "arguments": '{"path": "/"}'}, +} - output = await invoke_once(make_payload(tmp_path)) - # subagent (10/20/30) + main (2/3/5), the duplicate subagent message counted once - assert output["usage"] == { - "prompt_tokens": 12, - "completion_tokens": 23, - "total_tokens": 35, +@pytest.fixture(name="openai_endpoint") +def openai_endpoint_fixture(fake_sdks, monkeypatch) -> dict[str, Any]: + """Run the real Deep Agents graph and ChatOpenAI against a scripted endpoint. + + Tests append response bodies, ``httpx.Response`` objects, or exceptions to + raise to ``replies``; each model request takes the next one, and its JSON + body is kept in ``requests``. ``profile`` sets the chat model profile, which Deep Agents uses + to choose its summarization trigger. + """ + + import httpx + + for name in ( + "deepagents", + "deepagents.backends", + "deepagents.middleware", + "deepagents.middleware.subagents", + "langchain.chat_models", + "langgraph", + "langgraph.checkpoint", + "langgraph.checkpoint.sqlite", + "langgraph.checkpoint.sqlite.aio", + ): + monkeypatch.delitem(sys.modules, name, raising=False) + endpoint: dict[str, Any] = {"replies": [], "requests": [], "profile": None} + + def respond(request: httpx.Request) -> httpx.Response: + endpoint["requests"].append(json.loads(request.content)) + reply = endpoint["replies"].pop(0) + if isinstance(reply, Exception): + raise reply + if isinstance(reply, httpx.Response): + return reply + return httpx.Response(200, json=reply) + + def build_chat_openai(**kwargs: Any) -> InstalledChatOpenAI: + return InstalledChatOpenAI( + **kwargs, + max_retries=0, + profile=endpoint["profile"], + http_async_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), + ) + + fake_sdks["chat_openai"].side_effect = build_chat_openai + return endpoint + + +async def run_invocations(payload: dict[str, Any], count: int) -> list[AgentRunResult]: + runtime = adapter.DeepAgentsRuntime() + await runtime.start(lifecycle_start_payload(payload)) + try: + return [ + await runtime.invoke(*lifecycle_invocation(payload)) for _ in range(count) + ] + finally: + await runtime.stop() + + +async def test_summarization_model_call_usage_is_counted( + tmp_path, make_payload, openai_endpoint +): + # Deep Agents summarizes older history with a separate model call whose response + # never enters graph state. Its usage belongs to the invocation that made it. + openai_endpoint["profile"] = {"max_input_tokens": 1000} + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 1500, "completion_tokens": 10, "total_tokens": 1510} + ), + # Second invocation: the summary call, then the agent's model call. + chat_completion( + {"prompt_tokens": 1400, "completion_tokens": 60, "total_tokens": 1460} + ), + chat_completion( + {"prompt_tokens": 300, "completion_tokens": 5, "total_tokens": 305} + ), + ] + payload = make_payload(tmp_path) + payload["request"]["input"] = "word " * 6000 + + first, second = await run_invocations(payload, 2) + + summary_request, agent_request = openai_endpoint["requests"][1:] + assert "tools" not in summary_request + assert "tools" in agent_request + assert first.usage == AgentUsage( + input_tokens=1500, output_tokens=10, total_tokens=1510 + ) + assert second.status is AgentRunStatus.SUCCEEDED + assert second.usage == AgentUsage( + input_tokens=1700, output_tokens=65, total_tokens=1765 + ) + + +async def test_subagent_model_calls_are_counted( + tmp_path, make_payload, openai_endpoint +): + # Two parallel ``task`` calls run the general-purpose subagent twice; each + # subagent's model call belongs to this invocation. + task_calls = [ + { + "id": f"task-{index}", + "type": "function", + "function": { + "name": "task", + "arguments": json.dumps( + { + "description": f"Echo the word {index}.", + "subagent_type": "general-purpose", + } + ), + }, + } + for index in (1, 2) + ] + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + tool_calls=task_calls, + ), + chat_completion( + {"prompt_tokens": 10, "completion_tokens": 10, "total_tokens": 20} + ), + chat_completion( + {"prompt_tokens": 100, "completion_tokens": 100, "total_tokens": 200} + ), + chat_completion( + {"prompt_tokens": 1000, "completion_tokens": 1000, "total_tokens": 2000} + ), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert len(openai_endpoint["requests"]) == 4 + subgraphs = { + event["subgraph"] for event in result.output["events"] if "subgraph" in event } - # the subgraph step is recorded with its namespace label - assert any(evt.get("subgraph") == "task:researcher" for evt in output["events"]) + assert len(subgraphs) == 2 + assert result.usage == AgentUsage( + input_tokens=1111, output_tokens=1111, total_tokens=2222 + ) + + +async def test_recursion_limit_failure_keeps_usage_of_completed_model_calls( + tmp_path, make_payload, openai_endpoint +): + tool_call_reply = chat_completion( + {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, + tool_calls=[LS_TOOL_CALL], + ) + openai_endpoint["replies"] += [tool_call_reply] * 20 + payload = make_payload(tmp_path) + payload["config"]["runtime"] = {"max_turns": 6} + + (result,) = await run_invocations(payload, 1) + + calls = len(openai_endpoint["requests"]) + assert calls > 1 + assert result.status is AgentRunStatus.FAILED + assert result.error.code == "deepagents_recursion_limit_reached" + assert result.usage == AgentUsage( + input_tokens=10 * calls, output_tokens=calls, total_tokens=11 * calls + ) + + +def provider_failure(kind: str) -> Any: + """Return a scripted endpoint reply that fails a model call.""" + + import httpx + + if kind == "timeout": + return httpx.ReadTimeout("provider did not answer in time") + return httpx.Response(int(kind), json={"error": {"message": "request failed"}}) + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + # A 4xx rejection other than 408 spends nothing, so the finished call's + # usage stays known. + pytest.param( + "400", + AgentUsage(input_tokens=120, output_tokens=8, total_tokens=128), + id="400-rejection", + ), + # The provider may have done work without reporting usage. + pytest.param("408", None, id="408-request-timeout"), + pytest.param("503", None, id="503-server-error"), + pytest.param("timeout", None, id="read-timeout"), + ], +) +async def test_failed_model_call_usage_follows_provider_response( + tmp_path, make_payload, openai_endpoint, failure, expected +): + openai_endpoint["replies"] += [ + chat_completion( + {"prompt_tokens": 120, "completion_tokens": 8, "total_tokens": 128}, + tool_calls=[LS_TOOL_CALL], + ), + provider_failure(failure), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage == expected + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + pytest.param( + "429", + {"prompt_tokens": 30, "completion_tokens": 2, "total_tokens": 32}, + id="429-then-success", + ), + pytest.param("503", None, id="503-then-success"), + ], +) +async def test_retried_model_call_usage_follows_failed_attempt( + fake_sdks, openai_endpoint, failure, expected +): + # LangChain's with_retry runs each attempt as its own chat-model call, as the + # Deep Agents summarization model does. + openai_endpoint["replies"] += [ + provider_failure(failure), + chat_completion( + {"prompt_tokens": 30, "completion_tokens": 2, "total_tokens": 32} + ), + ] + model = fake_sdks["chat_openai"]( + model="nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + api_key="test123", + base_url="https://integrate.api.nvidia.com/v1", + ) + model_usage = adapter.ModelCallUsage() + + await model.with_retry(stop_after_attempt=2, wait_exponential_jitter=False).ainvoke( + "summarize", config={"callbacks": [model_usage]} + ) + + assert adapter._aggregate_usage(model_usage.calls()) == expected + + +async def test_recorder_keeps_a_cancelled_call_unreported(): + # Recorder unit test: a cancelled call ends without on_llm_end and, when + # LangChain gathers it, without on_llm_error; its start record keeps the + # invocation's counts unknown. + model_usage = adapter.ModelCallUsage() + await call_model( + {"callbacks": [model_usage]}, + AIMessage( + content="done", + usage_metadata={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ), + ) + model_usage.on_chat_model_start({}, [], run_id=uuid.uuid4()) + + assert adapter._aggregate_usage(model_usage.calls()) is None + + +@pytest.mark.parametrize( + ("usages", "expected"), + [ + pytest.param( + [{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}], + AgentUsage(input_tokens=0, output_tokens=0, total_tokens=0), + id="explicit-zero", + ), + pytest.param( + [{"prompt_tokens": 100}], + AgentUsage(input_tokens=100), + id="input-only", + ), + pytest.param( + [{"completion_tokens": 5}], + AgentUsage(output_tokens=5), + id="output-only", + ), + pytest.param( + [{"prompt_tokens": 100, "completion_tokens": 5}], + AgentUsage(input_tokens=100, output_tokens=5, total_tokens=105), + id="total-omitted", + ), + pytest.param( + [{"completion_tokens": 5, "total_tokens": 105}], + AgentUsage(output_tokens=5, total_tokens=105), + id="provider-total-with-input-omitted", + ), + pytest.param( + [{"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 0}], + AgentUsage(input_tokens=7, output_tokens=5, total_tokens=0), + id="provider-total-zero-kept", + ), + pytest.param( + [ + {"prompt_tokens": 100, "completion_tokens": 5, "total_tokens": 105}, + None, + ], + None, + id="one-call-without-usage", + ), + ], +) +async def test_partial_usage_reports_keep_missing_counts_unknown( + tmp_path, make_payload, openai_endpoint, usages, expected +): + # ChatOpenAI turns an omitted count into 0; a count is known only when every + # model call in the invocation reported it. + openai_endpoint["replies"] += [ + *(chat_completion(usage, tool_calls=[LS_TOOL_CALL]) for usage in usages[:-1]), + chat_completion(usages[-1]), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage == expected async def test_bad_mcp_transport_fails_runtime_start(tmp_path, make_payload): diff --git a/tests/e2e/test_deepagents.py b/tests/e2e/test_deepagents.py index 30b07e9ef..08155598c 100644 --- a/tests/e2e/test_deepagents.py +++ b/tests/e2e/test_deepagents.py @@ -486,7 +486,6 @@ async def test_deepagents_builtin_subagent_delegation(): call["args"]["subagent_type"] == "general-purpose" for call in task_calls ), result.to_mapping() assert any("subgraph" in event for event in output["events"]), result.to_mapping() - # delegated steps are folded into this turn's usage aggregation assert output["resumed"] is False, result.to_mapping() diff --git a/tests/integrations/test_relay_scope_leak.py b/tests/integrations/test_relay_scope_leak.py index 9dabfdcd5..5d5dbc58a 100644 --- a/tests/integrations/test_relay_scope_leak.py +++ b/tests/integrations/test_relay_scope_leak.py @@ -168,7 +168,7 @@ async def test_uuid_request_id_seeds_real_relay_parent( monkeypatch, make_runtime_context ): async def fake_invoke(agent, user_message, thread_id, callbacks=None): - return {"messages": []}, [], [] + return {"messages": []}, [] monkeypatch.setattr(adapter, "invoke_compiled_agent", fake_invoke) @@ -210,7 +210,7 @@ async def test_overlapping_turns_stay_telemetry_clean( async def fake_invoke(agent, user_message, thread_id, callbacks=None): if callbacks: await _overlapping_chain_runs(callbacks[0]) - return {"messages": []}, [], [] + return {"messages": []}, [] monkeypatch.setattr(adapter, "invoke_compiled_agent", fake_invoke) recording_scope = _RecordingScope() From 5de490480561c05c6bc2f7b7b3d6d439735262ff Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sat, 17 Oct 2026 22:22:18 -0700 Subject: [PATCH 059/159] feat: add what_ran provenance to run results A RunResult says which agent and adapter ran, but not which Fabric version, models or MCP servers were behind it, so a rerun weeks later cannot tell what changed. Add an optional, versioned what_ran object. Each fact holds a value, or a state and a reason, plus how Fabric learned it: its source, collector and method. Core fills the facts it holds before it sends the request: its own version, the adapter ID, the requested models, the configured MCP servers with secret-free endpoints, and the environment provider. Core reads no adapter report yet, so adapter-reported facts read unsupported with reason key_not_declared. On the OpenShell host path, the host relabels the task core's facts as task_core, records its own version as host_core_version and sets the environment provider to openshell. A task result without a readable what_ran keeps what_ran absent. The Rust and Python readers apply the same structural checks, tested against one shared vector file. A malformed what_ran reads as absent and never fails the enclosing result. Core gains a direct dependency on sha2 0.10, already in the lockfile, for the string length rule. Rust code that builds RunResult with a struct literal must add the what_ran field. Python's RunResult.to_mapping() now always includes it. Signed-off-by: mnajafian-nv --- Cargo.lock | 1 + Cargo.toml | 4 + crates/fabric-core/Cargo.toml | 1 + crates/fabric-core/src/endpoint.rs | 232 +++++ crates/fabric-core/src/lib.rs | 2 + crates/fabric-core/src/runtime.rs | 59 ++ crates/fabric-core/src/runtime/openshell.rs | 102 ++ crates/fabric-core/src/what_ran.rs | 984 ++++++++++++++++++ schemas/sdk/run-result.schema.json | 512 +++++++++ .../src/nemo_fabric/types.py | 141 ++- tests/fixtures/what_ran/reader-vectors.json | 809 ++++++++++++++ tests/python/test_native_sdk.py | 4 + tests/python/test_sdk_contract.py | 74 ++ 13 files changed, 2924 insertions(+), 1 deletion(-) create mode 100644 crates/fabric-core/src/endpoint.rs create mode 100644 crates/fabric-core/src/what_ran.rs create mode 100644 tests/fixtures/what_ran/reader-vectors.json diff --git a/Cargo.lock b/Cargo.lock index 9e6be276f..8eaa11a9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1345,6 +1345,7 @@ dependencies = [ "schemars", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "windows-sys 0.61.2", ] diff --git a/Cargo.toml b/Cargo.toml index 76ee54284..9cdefbdd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,10 @@ libc = "0.2.186" schemars = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +# SHA-256 for the `what_ran` string length rule, which ends a cut value with a +# digest prefix. Std has no SHA-256, and `sha2` 0.10 is already in the lockfile +# through the OpenShell provider. +sha2 = { version = "0.10.9", default-features = false } thiserror = "2" windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_System_Diagnostics_ToolHelp", "Win32_System_JobObjects", "Win32_System_Threading"] } pyo3 = "0.28.2" diff --git a/crates/fabric-core/Cargo.toml b/crates/fabric-core/Cargo.toml index 440de80fa..1572f92e3 100644 --- a/crates/fabric-core/Cargo.toml +++ b/crates/fabric-core/Cargo.toml @@ -18,6 +18,7 @@ jsonschema.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true thiserror.workspace = true [target.'cfg(unix)'.dependencies] diff --git a/crates/fabric-core/src/endpoint.rs b/crates/fabric-core/src/endpoint.rs new file mode 100644 index 000000000..cddabac29 --- /dev/null +++ b/crates/fabric-core/src/endpoint.rs @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Secret-free identities for configured endpoints. + +use std::net::Ipv6Addr; + +/// The endpoint identity of `url`: its lowercase scheme and host, and its +/// port as a number. An omitted port becomes the scheme's default, 80 for +/// `http` and `ws` and 443 for `https` and `wss`; other schemes keep a port +/// only when one is written. An IPv6 host keeps its brackets in canonical, +/// compressed lowercase form, such as `[2001:db8::1]`. The user info, path, +/// query and fragment never enter it, because a hosted service can carry a +/// key in any of them. +/// +/// `None` when `url` has no `scheme://host`, when an `@` follows the +/// authority (a password holding an unencoded `/`, `\`, `?` or `#` ends +/// the authority early, so the host cannot be told apart from the user +/// info), when the host holds a character other than ASCII letters, digits, +/// `-`, `.`, `_` and `~`, when a bracketed host is not an IPv6 address +/// (a scoped address with a zone ID is rejected too), or when the port is +/// not a number below 65,536. +pub(crate) fn endpoint_identity(url: &str) -> Option { + let (scheme, rest) = url.trim().split_once("://")?; + if !is_scheme(scheme) { + return None; + } + let authority_end = rest.find(['/', '\\', '?', '#']).unwrap_or(rest.len()); + let (authority, tail) = rest.split_at(authority_end); + if tail.contains('@') { + return None; + } + let host_port = authority + .rsplit_once('@') + .map_or(authority, |(_, host_port)| host_port); + let (host, port) = split_host_port(host_port)?; + let scheme = scheme.to_ascii_lowercase(); + let mut identity = format!("{scheme}://{host}"); + if let Some(port) = port.or_else(|| default_port(&scheme)) { + identity.push_str(&format!(":{port}")); + } + Some(identity) +} + +/// Whether `scheme` is a URL scheme: a letter, then letters, digits, `+`, +/// `-` or `.`. +fn is_scheme(scheme: &str) -> bool { + let mut characters = scheme.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic()) + && characters.all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '+' | '-' | '.') + }) +} + +/// Split `host[:port]` into the lowercase host, with a bracketed IPv6 +/// address in canonical form, and the port. `None` when the host is empty, +/// holds a character no host name holds, or is bracketed but not an IPv6 +/// address, or when the port is not a number below 65,536. An empty port +/// reads as no port. +fn split_host_port(host_port: &str) -> Option<(String, Option)> { + let (host, port) = match host_port.strip_prefix('[') { + Some(bracketed) => { + let (address, after) = bracketed.split_once(']')?; + let address: Ipv6Addr = address.parse().ok()?; + let port = match after { + "" => None, + after => Some(after.strip_prefix(':')?), + }; + (format!("[{address}]"), port) + } + None => { + let (host, port) = match host_port.split_once(':') { + Some((host, port)) => (host, Some(port)), + None => (host_port, None), + }; + let host_character = |character: char| { + character.is_ascii_alphanumeric() || matches!(character, '-' | '.' | '_' | '~') + }; + if host.is_empty() || !host.chars().all(host_character) { + return None; + } + (host.to_ascii_lowercase(), port) + } + }; + match port.filter(|port| !port.is_empty()) { + None => Some((host, None)), + Some(port) if port.bytes().all(|byte| byte.is_ascii_digit()) => { + port.parse::().ok().map(|port| (host, Some(port))) + } + Some(_) => None, + } +} + +fn default_port(scheme: &str) -> Option { + match scheme { + "http" | "ws" => Some(80), + "https" | "wss" => Some(443), + _ => None, + } +} + +/// The file name of a `stdio` server's executable: the part after the last +/// `/` or `\\`. +/// +/// `None` when the executable holds an `=` or any whitespace anywhere, or +/// when the file name is empty or holds a character other than ASCII +/// letters, digits, `.`, `_`, `+` and `-`. The plan keeps arguments in their +/// own field, so whitespace in the executable field means arguments may +/// have been written into it, and an argument can put a secret after a `/` +/// (`npx --token a/SECRET`). A leading `NAME=value` can do the same +/// (`TOKEN=a/SECRET server`). Both checks run before the folder is dropped. +/// A path with a space in a folder is rejected too. This is a conservative +/// extractor, not a shell parser. +pub(crate) fn executable_name(executable: &str) -> Option { + if executable.contains('=') || executable.chars().any(char::is_whitespace) { + return None; + } + let name = executable.rsplit(['/', '\\']).next().unwrap_or_default(); + let plain = |character: char| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | '+' | '-') + }; + (!name.is_empty() && name.chars().all(plain)).then(|| name.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn endpoint_identity_keeps_scheme_host_and_port_only() { + for (url, identity) in [ + ( + "HTTP://user:pw@10.0.0.5:8000/v1/?key=abc#frag", + "http://10.0.0.5:8000", + ), + ("https://API.example.com/", "https://api.example.com:443"), + ("http://[::1]/v1", "http://[::1]:80"), + ( + "https://[2001:DB8:0:0::1]:8443/", + "https://[2001:db8::1]:8443", + ), + ("https://example.com", "https://example.com:443"), + ("https://example.com:443", "https://example.com:443"), + ( + "https://actions.zapier.com/mcp/sk-ak-SECRET/sse", + "https://actions.zapier.com:443", + ), + ("https://u%40x:p%2Fq@host:8443/v1", "https://host:8443"), + ("https://host:/v1", "https://host:443"), + ("https://host:0443/v1", "https://host:443"), + ("WSS://Host.Example/s", "wss://host.example:443"), + ("grpc://host:9000", "grpc://host:9000"), + ] { + assert_eq!(endpoint_identity(url).as_deref(), Some(identity), "{url}"); + } + } + + /// A URL whose host cannot be told apart from its user info, or that is + /// not a URL, has no identity, so no part of a password becomes a host. + #[test] + fn endpoint_identity_rejects_ambiguous_or_invalid_urls() { + for url in [ + "https://user:123/SECRET@host/x", + "https://u:p#SECRET@host/x", + "https://u:p?SECRET@host/x", + "https://user:pw@host/p@x", + r"https://api.example.com\SECRET@secret-host", + "https://ärzte.example/mcp", + "https://server.example.com/@owner/server/mcp", + "https://u:p/x", + "https://host:99999", + "https://host:8a", + "https://", + "1http://host", + "http://ho st/", + "http://[::1/", + "http://[]/", + "https://[SECRET]/mcp", + "http://[::::]/", + "http://[fe80::1%25eth0]/", + "http://[fe80::1%eth0]/", + "http://[::1]x/", + "localhost:8000", + "not a url", + ] { + assert_eq!(endpoint_identity(url), None, "{url}"); + } + } + + #[test] + fn executable_name_is_the_file_name_without_its_folder() { + for (executable, name) in [ + ("db-mcp", Some("db-mcp")), + ("/home/alice/.venv/bin/python", Some("python")), + (r"C:\tools\npx.cmd", Some("npx.cmd")), + (r"C:\MCP\server.exe", Some("server.exe")), + ] { + assert_eq!(executable_name(executable).as_deref(), name, "{executable}"); + } + } + + /// An `=` or whitespace anywhere rejects the executable before its + /// folder is dropped, so no part of an assignment or an argument + /// becomes the name. + #[test] + fn executable_name_rejects_assignments_and_unplain_names() { + for executable in [ + "OPENAI_API_KEY=SECRET npx", + "TOKEN=prefix/SECRET", + r"TOKEN=prefix\SECRET", + "TOKEN=https://SECRET", + "/opt/server --token=SECRET", + "npx --api-key SECRET", + "npx -y @org/server-github --token SECRET", + "/opt/My Tools/db server", + "npx --token a/SECRET", + r"npx --token a\SECRET", + r"C:\Program Files\MCP\server.exe", + "/opt/My Tools/db-server", + "db-mcp\t", + " db-mcp", + "'quoted server'", + "/opt/tools/ärzte", + "", + "/opt/tools/", + ] { + assert_eq!(executable_name(executable), None, "{executable}"); + } + } +} diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index af3493a60..98853977d 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -8,11 +8,13 @@ pub mod agent_config; pub mod agent_execution; pub mod config; pub mod doctor; +mod endpoint; mod environment; pub mod error; pub mod runtime; pub mod schema; mod task_command; +pub mod what_ran; pub use adapter_contract::{ADAPTER_CONTRACT_VERSION, AdapterExtensionPoint}; pub use agent_execution::{ diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 3c3463596..893914ff9 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -32,6 +32,7 @@ use crate::config::{ }; use crate::environment::resolve_environment_provider; use crate::error::{FabricError, Result}; +use crate::what_ran::WhatRan; mod openshell; use openshell::OpenShellRuntimeAdapter; @@ -152,6 +153,15 @@ pub struct RunResult { /// Adapter-specific metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, + /// What executed for this invocation, and how NeMo Fabric learned each + /// fact. Absent when the producing core predates it, or when this core + /// finds it malformed or of another schema or version. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::what_ran::deserialize_lenient" + )] + pub what_ran: Option, } /// Runtime completion status. @@ -1913,6 +1923,11 @@ fn run_local_host_invocation_with_timeout( request_id: request.request_id.clone(), runtime_id: runtime.runtime_id.clone(), }; + let what_ran = crate::what_ran::collect_before_request( + plan, + adapter_id(plan), + &runtime.environment.provider, + ); let host = local_hosts() .get(&runtime.runtime_id) .cloned() @@ -2167,6 +2182,7 @@ fn run_local_host_invocation_with_timeout( .collect(), events, metadata, + what_ran: Some(what_ran), }) } @@ -4418,6 +4434,7 @@ for line in sys.stdin: ], events: Vec::new(), metadata: BTreeMap::new(), + what_ran: None, }; let encoded = serde_json::to_value(&result).expect("serialize SDK run result"); @@ -5442,6 +5459,48 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn local_host_result_carries_what_ran_without_changing_the_outcome() { + let run_result_schema = + crate::schema::generate_schema(crate::schema::SchemaName::RunResult) + .expect("run result schema"); + let validator = jsonschema::validator_for(&run_result_schema).expect("valid schema"); + let not_declared = serde_json::json!({ + "state": "unsupported", "reason": "key_not_declared", "source": "reported", + "collected_by": "adapter", "method": "adapter_report" + }); + for (mode, status) in [ + ("success", RunStatus::Succeeded), + ("adapter_reported_failure", RunStatus::Failed), + ] { + let (root, plan) = local_host_plan(mode); + + let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, status); + let result_json = serde_json::to_value(&result).expect("result JSON"); + assert!(validator.is_valid(&result_json), "{result_json}"); + let what_ran = &result_json["what_ran"]; + assert_eq!(what_ran["schema"], "nemo.fabric.what_ran"); + assert_eq!(what_ran["version"], "v1alpha1"); + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!(what_ran["adapter"]["id"], "acme.fabric.local-host"); + assert_eq!(what_ran["adapter"]["package_version"], not_declared); + assert_eq!(what_ran["harness"]["name"], not_declared); + assert_eq!(what_ran["harness"]["version"], not_declared); + assert_eq!(what_ran["models"], serde_json::json!({})); + assert_eq!(what_ran["mcp"], serde_json::json!({"servers": {}})); + assert_eq!( + what_ran["environment"], + serde_json::json!({"provider": "local"}) + ); + let _ = fs::remove_dir_all(root); + } + } + #[test] fn local_host_preserves_detailed_usage_in_metadata() { let (root, plan) = local_host_plan("detailed_usage_result"); diff --git a/crates/fabric-core/src/runtime/openshell.rs b/crates/fabric-core/src/runtime/openshell.rs index 75dc78bc3..b8e7f5e80 100644 --- a/crates/fabric-core/src/runtime/openshell.rs +++ b/crates/fabric-core/src/runtime/openshell.rs @@ -433,6 +433,7 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { "", )); } + adopt_task_what_ran(&mut result, &runtime.environment); for telemetry in &mut result.telemetry { telemetry.relay_enabled = telemetry .metadata @@ -516,6 +517,16 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { } } +/// Attribute the task core's `what_ran` to the task core, this host's +/// version and this host's environment provider. A task result without a +/// readable `what_ran`, from an older task core or one this host cannot +/// decode, keeps `what_ran` absent and is otherwise unchanged. +fn adopt_task_what_ran(result: &mut RunResult, environment: &EnvironmentHandle) { + if let Some(what_ran) = result.what_ran.as_mut() { + crate::what_ran::attribute_to_task_core(what_ran, &environment.provider); + } +} + pub(super) fn release(environment: &EnvironmentHandle) -> Result<()> { let key = resource_key(environment)?; let previous = { @@ -847,6 +858,97 @@ mod tests { } } + /// A task result as the task core returns it, with `what_ran` from the + /// shared reader vectors when given. + fn task_result(what_ran: Option) -> RunResult { + let mut result = serde_json::json!({ + "agent_name": "agent", "harness": "nvidia.fabric.claude", "adapter_kind": "python", + "runtime_id": "runtime-task", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "succeeded", "output": "done" + }); + if let Some(what_ran) = what_ran { + result["what_ran"] = what_ran; + } + serde_json::from_value(result).expect("a task result decodes whatever what_ran holds") + } + + fn task_what_ran() -> Value { + let vectors: Value = serde_json::from_str(include_str!( + "../../../../tests/fixtures/what_ran/reader-vectors.json" + )) + .expect("reader vectors JSON"); + vectors["base"].clone() + } + + fn facts_collected_by(value: &Value, collector: &str) -> usize { + match value { + Value::Object(map) if map.contains_key("source") => { + usize::from(map.get("collected_by") == Some(&Value::from(collector))) + } + Value::Object(map) => map + .values() + .map(|value| facts_collected_by(value, collector)) + .sum(), + _ => 0, + } + } + + #[test] + fn task_what_ran_is_attributed_to_the_task_core_and_this_host() { + for task_version in [crate::version(), "0.5.0"] { + let mut what_ran = task_what_ran(); + what_ran["fabric"]["core_version"]["value"] = Value::from(task_version); + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + let what_ran = serde_json::to_value(result.what_ran).expect("what_ran JSON"); + assert_task_facts_are_attributed(&what_ran, task_version); + } + } + + fn assert_task_facts_are_attributed(what_ran: &Value, task_version: &str) { + let host_version = serde_json::json!({"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}); + let task_core_version = serde_json::json!({"value": task_version, "source": "observed", + "collected_by": "task_core", "method": "crate_version"}); + assert_eq!(what_ran["fabric"]["core_version"], task_core_version); + assert_eq!(what_ran["fabric"]["host_core_version"], host_version); + assert_eq!( + facts_collected_by(what_ran, "host_core"), + 1, + "only host_core_version is the host's fact" + ); + assert_eq!(what_ran["environment"]["provider"], "openshell"); + assert_eq!( + what_ran["models"]["default"]["requested"]["collected_by"], + "task_core" + ); + assert_eq!( + what_ran["mcp"]["servers"]["docs"]["endpoint"]["collected_by"], + "task_core" + ); + assert_eq!( + what_ran["harness"]["version"]["collected_by"], "adapter", + "adapter facts keep their collector" + ); + } + + #[test] + fn task_result_without_readable_what_ran_is_kept_without_it() { + let mut undecodable = task_what_ran(); + undecodable["harness"]["version"]["state"] = Value::from("withheld"); + for what_ran in [None, Some(undecodable)] { + let mut result = task_result(what_ran); + + adopt_task_what_ran(&mut result, &environment()); + + assert_eq!(result.what_ran, None); + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.output, Value::from("done")); + } + } + #[test] fn unknown_binding_survives_a_new_host_environment_handle() { let original = environment(); diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs new file mode 100644 index 000000000..ef5686ee4 --- /dev/null +++ b/crates/fabric-core/src/what_ran.rs @@ -0,0 +1,984 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! What executed for one invocation, and how NeMo Fabric learned each fact. +//! +//! [`WhatRan`] rides on [`RunResult`](crate::RunResult). Each fact is a +//! [`Fact`]: a value, or a [`FactState`] with a reason, and in both cases +//! the [`FactSource`], the component that collected it and the method it +//! used. A few names fixed by the run plan stay plain strings: the adapter +//! ID, each model role's provider, the environment provider, and each MCP +//! server's origin and transport. +//! +//! A string value longer than 256 characters, in a string fact or as an +//! element of a string-list fact, keeps its first 240 characters, then `~`, +//! then the first 15 hex digits of its SHA-256. Map keys and the plain names +//! above are never cut. +//! +//! `what_ran` is evidence. It never decides which runs are compared. +//! +//! Compatibility rules for readers: +//! +//! - `source` and `state` are closed sets. A new value changes +//! [`WHAT_RAN_VERSION`]. +//! - `collected_by`, `method` and `reason` are open. Readers accept values +//! they do not know. +//! - Readers accept unknown properties. Adding an optional property keeps +//! [`WHAT_RAN_VERSION`]; removing one or changing its meaning changes it. +//! The Rust types drop unknown properties when they decode, so a host +//! that relays a task core's result drops them too. +//! - A fact holds `value`, or `state` and `reason`, never both. A +//! `RunResult` reader that finds a fact breaking this, any other value +//! the schema rejects, or another schema or version reads `what_ran` as +//! absent and still decodes the result. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::de::{DeserializeOwned, Error as _}; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::config::{McpServerPlan, McpTransport, ModelConfig, RunPlan}; +use crate::endpoint::{endpoint_identity, executable_name}; + +/// Schema name written into every [`WhatRan`]. +pub const WHAT_RAN_SCHEMA: &str = "nemo.fabric.what_ran"; + +/// Contract version written into every [`WhatRan`]. +pub const WHAT_RAN_VERSION: &str = "v1alpha1"; + +/// How NeMo Fabric learned a fact. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FactSource { + /// NeMo Fabric core saw it itself. + Observed, + /// The adapter, the harness, an MCP server or a remote service said so. + Reported, + /// It comes from the resolved run plan. + Configured, +} + +/// Why a fact has no value. A collector picks the first state that matches, +/// in declaration order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FactState { + /// The fact cannot exist for this run. + NotApplicable, + /// The adapter or environment provider has no source for the fact. + Unsupported, + /// A source exists, but it failed, timed out, returned an invalid report + /// or never returned. + CollectionFailed, + /// NeMo Fabric holds no value and no evidence for the other states. + Unavailable, +} + +/// Pattern of the open codes `collected_by`, `method` and `reason`. +const CODE_PATTERN: &str = "^[a-z0-9_.:#+/-]{1,128}$"; + +/// Who collected a fact and how. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FactOrigin { + /// How NeMo Fabric learned the fact. + pub source: FactSource, + /// Component that collected the fact, such as `host_core`, `task_core` + /// or `adapter`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub collected_by: String, + /// How the collector got the fact, such as `crate_version`, + /// `resolved_plan` or `adapter_report`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub method: String, +} + +/// A fact with a value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars( + rename = "KnownFact_for_{T}", + extend("not" = {"anyOf": [{"required": ["state"]}, {"required": ["reason"]}]}) +)] +pub struct KnownFact { + /// The fact itself: a string, a boolean or a list of strings. + pub value: T, + /// Who collected the fact and how. + #[serde(flatten)] + pub origin: FactOrigin, +} + +/// A fact without a value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend("not" = {"required": ["value"]}))] +pub struct MissingFact { + /// Why the fact has no value. + pub state: FactState, + /// Short code that explains the state, such as `key_not_declared`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub reason: String, + /// Who would have collected the fact and how. + #[serde(flatten)] + pub origin: FactOrigin, +} + +/// One fact: a `value`, or a `state` with a `reason`, never both. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(untagged)] +#[schemars(rename = "Fact_for_{T}")] +pub enum Fact { + /// The fact has a value. + Known(KnownFact), + /// The fact has no value. + Missing(MissingFact), +} + +impl<'de, T: DeserializeOwned> Deserialize<'de> for Fact { + /// Decode a fact, rejecting one that holds `value` together with + /// `state` or `reason`, or an open code outside its pattern. Unknown + /// properties are ignored. + fn deserialize>(deserializer: D) -> Result { + let fact = serde_json::Map::::deserialize(deserializer)?; + let fact = if fact.contains_key("value") { + if fact.contains_key("state") || fact.contains_key("reason") { + return Err(D::Error::custom( + "a fact holds `value`, or `state` and `reason`, never both", + )); + } + KnownFact::deserialize(Value::Object(fact)).map(Self::Known) + } else { + MissingFact::deserialize(Value::Object(fact)).map(Self::Missing) + } + .map_err(D::Error::custom)?; + let origin = fact.origin(); + let reason = match &fact { + Self::Known(_) => None, + Self::Missing(missing) => Some(missing.reason.as_str()), + }; + if [origin.collected_by.as_str(), origin.method.as_str()] + .into_iter() + .chain(reason) + .all(is_code) + { + Ok(fact) + } else { + Err(D::Error::custom(format!( + "`collected_by`, `method` and `reason` must match {CODE_PATTERN}" + ))) + } + } +} + +/// Whether `code` matches [`CODE_PATTERN`]. +fn is_code(code: &str) -> bool { + (1..=128).contains(&code.len()) + && code.bytes().all(|byte| { + byte.is_ascii_lowercase() + || byte.is_ascii_digit() + || matches!(byte, b'_' | b'.' | b':' | b'#' | b'+' | b'/' | b'-') + }) +} + +/// What executed for one invocation. +/// +/// Decoding a `WhatRan` directly checks the shape of every section and +/// fact. `RunResult` also checks the schema name and version, and that +/// every `plan` MCP entry names its transport. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct WhatRan { + /// Schema name, `nemo.fabric.what_ran`. + #[schemars(extend("const" = WHAT_RAN_SCHEMA))] + pub schema: String, + /// Contract version, `v1alpha1`. + #[schemars(extend("const" = WHAT_RAN_VERSION))] + pub version: String, + /// NeMo Fabric core that ran the adapter. + pub fabric: FabricFacts, + /// Adapter that ran. + pub adapter: AdapterFacts, + /// Harness behind the adapter. + pub harness: HarnessFacts, + /// Model facts keyed by the role names of the plan's `models` settings. + pub models: BTreeMap, + /// MCP server facts. + pub mcp: McpFacts, + /// Environment the runtime ran in. + pub environment: EnvironmentFacts, +} + +/// NeMo Fabric core facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct FabricFacts { + /// Version of the core that ran the adapter. + pub core_version: Fact, + /// Version of the host core that relayed the result from a task core + /// in another environment. `not_applicable` for a local run. + pub host_core_version: Fact, +} + +/// Adapter facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct AdapterFacts { + /// Adapter ID from the plan, when an adapter was resolved. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + /// Version of the adapter package, as the adapter reports it. + pub package_version: Fact, +} + +/// Harness facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct HarnessFacts { + /// Name of the harness program that ran, as the adapter reports it. + /// Core never derives it from the adapter ID. + pub name: Fact, + /// Version of the harness program that ran, as the adapter reports it. + pub version: Fact, +} + +/// Facts about one model role. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ModelFacts { + /// Provider from the plan. + pub provider: String, + /// Model the plan requested. + pub requested: Fact, + /// Model that served, as the harness or provider reports it. + pub served: Fact, +} + +/// MCP facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct McpFacts { + /// Servers keyed by name. Empty when the plan configures no MCP server + /// and the harness reports none. + pub servers: BTreeMap, +} + +/// Facts about one MCP server. +/// +/// An entry with origin `harness` is a server the harness reports and the +/// plan does not configure. It has the same shape: its `endpoint` reads +/// `not_applicable` with reason `not_configured`, its `transport` may be +/// absent, and runtime facts the adapter did not collect read +/// `unavailable` with reason `not_collected`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend( + "if" = {"properties": {"origin": {"const": ORIGIN_PLAN}}}, + "then" = {"required": ["transport"], "properties": {"transport": {"type": "string"}}} +))] +pub struct McpServerFacts { + /// Where the entry comes from: `plan` for a server the plan configures, + /// `harness` for one only the harness reports. + pub origin: String, + /// Transport, such as `stdio` or `streamable-http`. Always present for + /// a `plan` entry. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transport: Option, + /// Configured endpoint without secrets: the lowercase scheme and host + /// and the port for a network server, or the executable's file name for + /// a `stdio` server. + pub endpoint: Fact, + /// What the harness learned when it connected, as the adapter reports it. + pub runtime: McpRuntimeFacts, +} + +/// What the harness learned about one MCP server when it connected. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct McpRuntimeFacts { + /// Whether the harness connected to the server. + pub connected: Fact, + /// MCP protocol version negotiated with the server. + pub protocol_version: Fact, + /// Name the server reports for itself. + pub server_name: Fact, + /// Version the server reports for itself. + pub server_version: Fact, + /// Raw tool names the server lists, each bounded by the string length + /// rule. + pub tool_names: Fact>, +} + +/// Environment facts. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct EnvironmentFacts { + /// Environment provider, such as `local`. + pub provider: String, +} + +const HOST_CORE: &str = "host_core"; +const TASK_CORE: &str = "task_core"; +const ADAPTER: &str = "adapter"; +const ORIGIN_PLAN: &str = "plan"; +const METHOD_CRATE_VERSION: &str = "crate_version"; +const METHOD_RESOLVED_PLAN: &str = "resolved_plan"; +const METHOD_ADAPTER_REPORT: &str = "adapter_report"; +const REASON_KEY_NOT_DECLARED: &str = "key_not_declared"; +const REASON_LOCAL_RUN: &str = "local_run"; +const REASON_NOT_CONFIGURED: &str = "not_configured"; +const REASON_ENDPOINT_UNPARSED: &str = "endpoint_unparsed"; +const REASON_EXECUTABLE_UNPARSED: &str = "executable_unparsed"; + +/// Longest string value kept whole, in characters. +const MAX_VALUE_CHARS: usize = 256; +/// Characters a longer string value keeps before `~` and its digest prefix. +const TRUNCATED_PREFIX_CHARS: usize = 240; +/// Hex digits of the SHA-256 that end a truncated string value. +const TRUNCATED_DIGEST_HEX_DIGITS: usize = 15; +const _: () = assert!(TRUNCATED_PREFIX_CHARS + 1 + TRUNCATED_DIGEST_HEX_DIGITS == MAX_VALUE_CHARS); + +impl Fact { + /// A known string fact, with the string length rule applied. + fn text(value: &str, source: FactSource, collected_by: &str, method: &str) -> Self { + Self::Known(KnownFact { + value: bounded_text(value), + origin: FactOrigin { + source, + collected_by: collected_by.to_string(), + method: method.to_string(), + }, + }) + } +} + +impl Fact { + fn origin(&self) -> &FactOrigin { + match self { + Self::Known(known) => &known.origin, + Self::Missing(missing) => &missing.origin, + } + } + + fn origin_mut(&mut self) -> &mut FactOrigin { + match self { + Self::Known(known) => &mut known.origin, + Self::Missing(missing) => &mut missing.origin, + } + } + + fn missing( + state: FactState, + reason: &str, + source: FactSource, + collected_by: &str, + method: &str, + ) -> Self { + Self::Missing(MissingFact { + state, + reason: reason.to_string(), + origin: FactOrigin { + source, + collected_by: collected_by.to_string(), + method: method.to_string(), + }, + }) + } + + /// An adapter-reported fact while core reads no adapter report key. + fn report_key_not_declared() -> Self { + Self::missing( + FactState::Unsupported, + REASON_KEY_NOT_DECLARED, + FactSource::Reported, + ADAPTER, + METHOD_ADAPTER_REPORT, + ) + } +} + +impl WhatRan { + /// The origin of every fact, so a host can relabel the facts a task + /// core collected. + fn origins_mut(&mut self) -> Vec<&mut FactOrigin> { + let Self { + fabric, + adapter, + harness, + models, + mcp, + .. + } = self; + let mut origins = vec![ + fabric.core_version.origin_mut(), + fabric.host_core_version.origin_mut(), + adapter.package_version.origin_mut(), + harness.name.origin_mut(), + harness.version.origin_mut(), + ]; + for model in models.values_mut() { + origins.extend([model.requested.origin_mut(), model.served.origin_mut()]); + } + for server in mcp.servers.values_mut() { + let runtime = &mut server.runtime; + origins.extend([ + server.endpoint.origin_mut(), + runtime.connected.origin_mut(), + runtime.protocol_version.origin_mut(), + runtime.server_name.origin_mut(), + runtime.server_version.origin_mut(), + runtime.tool_names.origin_mut(), + ]); + } + origins + } + + /// Whether every `plan` MCP entry names its transport. Only a `harness` + /// entry may leave it out. + fn plan_servers_have_transports(&self) -> bool { + self.mcp + .servers + .values() + .all(|server| server.origin != ORIGIN_PLAN || server.transport.is_some()) + } +} + +/// Collect the facts the core that runs the adapter knows before it sends +/// the request: its own version and the plan's adapter ID, models, MCP +/// servers and environment provider. +/// +/// Core reads no adapter report key yet, so every adapter-reported fact +/// reads `unsupported` with reason `key_not_declared`. The collecting core +/// runs the adapter in a local environment, so `host_core_version` reads +/// `not_applicable`. A host that relays the result from a task core calls +/// [`attribute_to_task_core`]. +pub(crate) fn collect_before_request( + plan: &RunPlan, + adapter_id: Option, + environment_provider: &str, +) -> WhatRan { + WhatRan { + schema: WHAT_RAN_SCHEMA.to_string(), + version: WHAT_RAN_VERSION.to_string(), + fabric: FabricFacts { + core_version: core_version_fact(), + host_core_version: Fact::missing( + FactState::NotApplicable, + REASON_LOCAL_RUN, + FactSource::Observed, + HOST_CORE, + METHOD_CRATE_VERSION, + ), + }, + adapter: AdapterFacts { + id: adapter_id, + package_version: Fact::report_key_not_declared(), + }, + harness: HarnessFacts { + name: Fact::report_key_not_declared(), + version: Fact::report_key_not_declared(), + }, + models: plan + .config + .models + .iter() + .map(|(role, model)| (role.clone(), model_facts(model))) + .collect(), + mcp: McpFacts { + servers: plan + .capability_plan + .mcp_servers + .iter() + .map(|(name, server)| (name.clone(), mcp_server_facts(server))) + .collect(), + }, + environment: EnvironmentFacts { + provider: environment_provider.to_string(), + }, + } +} + +fn core_version_fact() -> Fact { + Fact::text( + crate::version(), + FactSource::Observed, + HOST_CORE, + METHOD_CRATE_VERSION, + ) +} + +fn model_facts(model: &ModelConfig) -> ModelFacts { + ModelFacts { + provider: model.provider.clone(), + requested: Fact::text( + &model.model, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + served: Fact::report_key_not_declared(), + } +} + +fn mcp_server_facts(server: &McpServerPlan) -> McpServerFacts { + McpServerFacts { + origin: ORIGIN_PLAN.to_string(), + transport: Some(server.transport.as_str().to_string()), + endpoint: configured_endpoint(server.transport, &server.url), + runtime: McpRuntimeFacts { + connected: Fact::report_key_not_declared(), + protocol_version: Fact::report_key_not_declared(), + server_name: Fact::report_key_not_declared(), + server_version: Fact::report_key_not_declared(), + tool_names: Fact::report_key_not_declared(), + }, + } +} + +/// The configured endpoint of one MCP server, without secrets. `url` holds +/// the server URL, or the command for a `stdio` server. +/// +/// An empty `url` reads `unavailable` with reason `not_configured`. A value +/// core cannot reduce to its safe form reads `collection_failed`, and the +/// raw value never enters the fact. +fn configured_endpoint(transport: McpTransport, url: &str) -> Fact { + let missing = |state, reason| { + Fact::missing( + state, + reason, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ) + }; + if url.trim().is_empty() { + return missing(FactState::Unavailable, REASON_NOT_CONFIGURED); + } + let endpoint = match transport { + McpTransport::Stdio => executable_name(url).ok_or(REASON_EXECUTABLE_UNPARSED), + McpTransport::Sse | McpTransport::StreamableHttp => { + endpoint_identity(url).ok_or(REASON_ENDPOINT_UNPARSED) + } + }; + match endpoint { + Ok(endpoint) => Fact::text( + &endpoint, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + Err(reason) => missing(FactState::CollectionFailed, reason), + } +} + +/// Attribute a task core's `what_ran` on the host that relayed its result. +/// +/// The task core runs in a projected `local` environment and labels its +/// own facts `host_core`. The host relabels them `task_core`, records its +/// own version as `host_core_version`, and replaces the environment +/// provider with its own, such as `openshell`. +pub(crate) fn attribute_to_task_core(what_ran: &mut WhatRan, environment_provider: &str) { + for origin in what_ran.origins_mut() { + if origin.collected_by == HOST_CORE { + origin.collected_by = TASK_CORE.to_string(); + } + } + what_ran.fabric.host_core_version = core_version_fact(); + what_ran.environment.provider = environment_provider.to_string(); +} + +/// `text` whole when it has at most 256 characters. A longer string keeps +/// its first 240 characters, then `~`, then the first 15 hex digits of the +/// whole string's SHA-256, so two long values collide only on a 60-bit +/// hash collision. Characters are Unicode scalar values, and the digest +/// covers the UTF-8 bytes. +fn bounded_text(text: &str) -> String { + if text.chars().count() <= MAX_VALUE_CHARS { + return text.to_string(); + } + let mut bounded: String = text.chars().take(TRUNCATED_PREFIX_CHARS).collect(); + bounded.push('~'); + let digest_hex: String = Sha256::digest(text.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + bounded.push_str(&digest_hex[..TRUNCATED_DIGEST_HEX_DIGITS]); + bounded +} + +/// Decode `RunResult.what_ran`, reading it as absent when it breaks the +/// schema or names another schema or version, so the rest of the result +/// still decodes. +pub(crate) fn deserialize_lenient<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + Ok(value + .and_then(|value| serde_json::from_value::(value).ok()) + .filter(|what_ran| { + what_ran.schema == WHAT_RAN_SCHEMA + && what_ran.version == WHAT_RAN_VERSION + && what_ran.plan_servers_have_transports() + })) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use serde_json::json; + + use super::*; + use crate::config::{ + FabricConfig, ResolveContext, resolve_run_plan_from_config_with_adapter_directories, + }; + + fn repository_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..") + } + + fn resolve_plan(config: Value) -> RunPlan { + let root = repository_root(); + let config: FabricConfig = serde_json::from_value(config).expect("typed config"); + resolve_run_plan_from_config_with_adapter_directories( + config, + ResolveContext::new(&root), + &[root.join("adapters")], + ) + .expect("resolve plan") + } + + fn config_for(adapter_id: &str) -> Value { + json!({ + "schema_version": "fabric.agent/v1alpha1", + "metadata": {"name": "what-ran-agent"}, + "harness": {"adapter_id": adapter_id}, + "runtime": {} + }) + } + + /// A Claude Code plan whose MCP servers hold a secret in every place a + /// configured server can: URL user info, path, query and fragment, + /// headers, and a `stdio` command's folder, arguments and environment. + fn plan_with_secrets() -> RunPlan { + let mut config = config_for("nvidia.fabric.claude"); + config["models"] = json!({ + "default": {"provider": "anthropic", "model": "claude-sonnet-4-5"}, + }); + config["mcp"] = json!({"servers": { + "github": { + "transport": "streamable-http", + "url": "https://user:SECRET_PASSWORD@API.GitHubCopilot.com/mcp/SECRET_PATH/?token=SECRET_QUERY#SECRET_FRAGMENT", + "custom_headers": {"Authorization": "Bearer SECRET_HEADER"}, + "exposure": "harness_native" + }, + "events": { + "transport": "sse", + "url": "http://localhost:8931/sse", + "exposure": "harness_native" + }, + "files": { + "transport": "stdio", + "url": "/home/SECRET_USER/.venv/bin/files-mcp", + "args": ["--api-key", "SECRET_ARGUMENT"], + "env": {"FILES_TOKEN": "SECRET_ENVIRONMENT"}, + "exposure": "harness_native" + } + }}); + resolve_plan(config) + } + + fn collect(plan: &RunPlan) -> Value { + let adapter_id = plan + .adapter_descriptor + .as_ref() + .map(|adapter| adapter.descriptor.adapter_id.clone()); + serde_json::to_value(collect_before_request(plan, adapter_id, "local")) + .expect("serialize what_ran") + } + + fn not_declared() -> Value { + json!({"state": "unsupported", "reason": "key_not_declared", "source": "reported", + "collected_by": "adapter", "method": "adapter_report"}) + } + + fn configured(value: &str) -> Value { + json!({"value": value, "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + } + + fn mcp_server(transport: &str, endpoint: Value) -> Value { + let missing = not_declared(); + json!({ + "origin": "plan", + "transport": transport, + "endpoint": endpoint, + "runtime": { + "connected": missing, + "protocol_version": missing, + "server_name": missing, + "server_version": missing, + "tool_names": missing + } + }) + } + + /// Every JSON object that carries `source` is a fact. + fn facts(value: &Value) -> Vec<&serde_json::Map> { + match value { + Value::Object(map) if map.contains_key("source") => vec![map], + Value::Object(map) => map.values().flat_map(facts).collect(), + _ => Vec::new(), + } + } + + #[test] + fn collected_facts_read_the_plan_and_mark_adapter_facts_key_not_declared() { + let what_ran = collect(&plan_with_secrets()); + let not_declared = not_declared(); + + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": {"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}, + "host_core_version": {"state": "not_applicable", "reason": "local_run", + "source": "observed", "collected_by": "host_core", + "method": "crate_version"} + }, + "adapter": {"id": "nvidia.fabric.claude", "package_version": not_declared}, + "harness": {"name": not_declared, "version": not_declared}, + "models": {"default": { + "provider": "anthropic", + "requested": configured("claude-sonnet-4-5"), + "served": not_declared + }}, + "mcp": {"servers": { + "events": mcp_server("sse", configured("http://localhost:8931")), + "files": mcp_server("stdio", configured("files-mcp")), + "github": mcp_server( + "streamable-http", + configured("https://api.githubcopilot.com:443"), + ) + }}, + "environment": {"provider": "local"} + }) + ); + } + + #[test] + fn collected_facts_hold_no_configured_secret() { + let text = serde_json::to_string(&collect(&plan_with_secrets())).expect("JSON"); + + assert!(!text.contains("SECRET"), "{text}"); + } + + #[test] + fn every_fact_has_a_value_or_a_state_with_a_reason_never_both() { + let what_ran = collect(&plan_with_secrets()); + let facts = facts(&what_ran); + + // 2 fabric + 1 adapter + 2 harness + 2 model + 3 servers x (endpoint + 5 runtime). + assert_eq!(facts.len(), 25); + for fact in facts { + assert_ne!( + fact.contains_key("value"), + fact.contains_key("state"), + "{fact:?}" + ); + assert_eq!( + fact.contains_key("state"), + fact.contains_key("reason"), + "{fact:?}" + ); + for key in ["collected_by", "method"] { + assert!(fact.get(key).is_some_and(Value::is_string), "{fact:?}"); + } + } + } + + #[test] + fn plan_without_models_or_mcp_servers_reads_empty_maps() { + let what_ran = collect(&resolve_plan(config_for("nvidia.fabric.claude"))); + + assert_eq!(what_ran["models"], json!({})); + assert_eq!(what_ran["mcp"], json!({"servers": {}})); + } + + #[test] + fn stdio_executable_with_whitespace_reads_unparsed_and_keeps_no_secret() { + let mut config = config_for("nvidia.fabric.claude"); + config["mcp"] = json!({"servers": { + "arguments": { + "transport": "stdio", + "url": "npx --token a/SECRET", + "exposure": "harness_native" + }, + "spaced": { + "transport": "stdio", + "url": "/opt/My Tools/db-server", + "exposure": "harness_native" + } + }}); + + let what_ran = collect(&resolve_plan(config)); + + let unparsed = json!({"state": "collection_failed", "reason": "executable_unparsed", + "source": "configured", "collected_by": "host_core", "method": "resolved_plan"}); + for name in ["arguments", "spaced"] { + assert_eq!( + what_ran["mcp"]["servers"][name]["endpoint"], unparsed, + "{name}" + ); + } + let text = serde_json::to_string(&what_ran).expect("JSON"); + assert!(!text.contains("SECRET"), "{text}"); + } + + #[test] + fn unparsed_or_empty_endpoint_reads_a_state_without_the_raw_value() { + for (transport, url, state, reason) in [ + ( + McpTransport::StreamableHttp, + "https://u:p/SECRET@host/x", + "collection_failed", + "endpoint_unparsed", + ), + ( + McpTransport::Stdio, + "TOKEN=prefix/SECRET", + "collection_failed", + "executable_unparsed", + ), + (McpTransport::Sse, " ", "unavailable", "not_configured"), + (McpTransport::Stdio, "", "unavailable", "not_configured"), + ] { + let endpoint = serde_json::to_value(configured_endpoint(transport, url)) + .expect("serialize endpoint"); + + assert_eq!( + endpoint, + json!({"state": state, "reason": reason, "source": "configured", + "collected_by": "host_core", "method": "resolved_plan"}), + "{url}" + ); + } + } + + #[test] + fn string_values_longer_than_256_characters_end_with_a_digest_prefix() { + let exact = "é".repeat(256); + assert_eq!(bounded_text(&exact), exact); + + // Expected digests: the first 15 hex digits of `sha256sum` over the + // whole value's UTF-8 bytes. + for (long, prefix, digest) in [ + (format!("{}a", "m".repeat(300)), "m", "96858eedeb3c8e6"), + ("é".repeat(300), "é", "7250b66610f8b7d"), + ] { + let bounded = bounded_text(&long); + + assert_eq!(bounded, format!("{}~{digest}", prefix.repeat(240))); + assert_eq!(bounded.chars().count(), 256); + } + } + + fn run_result_with(what_ran: Value) -> crate::RunResult { + serde_json::from_value(json!({ + "agent_name": "agent", "harness": "nvidia.fabric.codex", "adapter_kind": "python", + "runtime_id": "runtime-1", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "succeeded", "what_ran": what_ran + })) + .expect("the result decodes whatever what_ran holds") + } + + /// The reader vectors the Python SDK tests share, with each case's + /// patch applied to the base object. + fn reader_vectors(group: &str) -> Vec<(String, Value)> { + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/reader-vectors.json" + )) + .expect("reader vectors JSON"); + vectors[group] + .as_array() + .expect("vector group") + .iter() + .map(|case| { + let mut what_ran = vectors["base"].clone(); + for operation in case["patch"].as_array().expect("patch") { + apply_patch(&mut what_ran, operation); + } + (case["name"].as_str().expect("name").to_string(), what_ran) + }) + .collect() + } + + fn apply_patch(target: &mut Value, operation: &Value) { + let path: Vec<&str> = operation["path"] + .as_array() + .expect("path") + .iter() + .map(|key| key.as_str().expect("key")) + .collect(); + let Some((last, parents)) = path.split_last() else { + *target = operation["value"].clone(); + return; + }; + let parent = parents + .iter() + .fold(target, |value, key| &mut value[*key]) + .as_object_mut() + .expect("patch parent is an object"); + if operation.get("remove").is_some() { + parent.remove(*last); + } else { + parent.insert((*last).to_string(), operation["value"].clone()); + } + } + + fn what_ran_validator() -> jsonschema::Validator { + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunResult) + .expect("run result schema"); + let mut what_ran_schema = schema["properties"]["what_ran"].clone(); + what_ran_schema["$defs"] = schema["$defs"].clone(); + jsonschema::validator_for(&what_ran_schema).expect("valid what_ran schema") + } + + #[test] + fn reader_and_schema_keep_every_valid_vector() { + let validator = what_ran_validator(); + for (name, what_ran) in reader_vectors("valid") { + assert!(validator.is_valid(&what_ran), "{name}"); + let decoded = run_result_with(what_ran.clone()).what_ran; + assert!(decoded.is_some(), "{name}"); + } + } + + #[test] + fn reader_reads_every_invalid_vector_as_absent_and_schema_rejects_it() { + let validator = what_ran_validator(); + for (name, what_ran) in reader_vectors("invalid") { + assert!(!validator.is_valid(&what_ran), "{name}"); + assert_eq!(run_result_with(what_ran).what_ran, None, "{name}"); + } + assert_eq!(run_result_with(Value::Null).what_ran, None); + } + + #[test] + fn collected_facts_match_the_schema() { + let what_ran = collect(&plan_with_secrets()); + + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn run_result_accepts_unknown_properties_and_open_codes() { + let (_, newer) = reader_vectors("valid") + .into_iter() + .find(|(name, _)| name == "unknown properties and vendor codes") + .expect("vector"); + + let what_ran = run_result_with(newer) + .what_ran + .expect("unknown properties never fail what_ran"); + + assert_eq!( + what_ran.fabric.core_version.origin().collected_by, + "acme.collector" + ); + } +} diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 492ef7136..72a10b9bf 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -1,5 +1,25 @@ { "$defs": { + "AdapterFacts": { + "description": "Adapter facts.", + "properties": { + "id": { + "description": "Adapter ID from the plan, when an adapter was resolved.", + "type": [ + "string", + "null" + ] + }, + "package_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the adapter package, as the adapter reports it." + } + }, + "required": [ + "package_version" + ], + "type": "object" + }, "AdapterKind": { "description": "Adapter implementation kind.", "oneOf": [ @@ -82,6 +102,19 @@ ], "type": "object" }, + "EnvironmentFacts": { + "description": "Environment facts.", + "properties": { + "provider": { + "description": "Environment provider, such as `local`.", + "type": "string" + } + }, + "required": [ + "provider" + ], + "type": "object" + }, "ErrorInfo": { "description": "Normalized error metadata.", "properties": { @@ -195,6 +228,421 @@ ], "type": "object" }, + "FabricFacts": { + "description": "NeMo Fabric core facts.", + "properties": { + "core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the core that ran the adapter." + }, + "host_core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the host core that relayed the result from a task core\nin another environment. `not_applicable` for a local run." + } + }, + "required": [ + "core_version", + "host_core_version" + ], + "type": "object" + }, + "FactSource": { + "description": "How NeMo Fabric learned a fact.", + "oneOf": [ + { + "const": "observed", + "description": "NeMo Fabric core saw it itself.", + "type": "string" + }, + { + "const": "reported", + "description": "The adapter, the harness, an MCP server or a remote service said so.", + "type": "string" + }, + { + "const": "configured", + "description": "It comes from the resolved run plan.", + "type": "string" + } + ] + }, + "FactState": { + "description": "Why a fact has no value. A collector picks the first state that matches,\nin declaration order.", + "oneOf": [ + { + "const": "not_applicable", + "description": "The fact cannot exist for this run.", + "type": "string" + }, + { + "const": "unsupported", + "description": "The adapter or environment provider has no source for the fact.", + "type": "string" + }, + { + "const": "collection_failed", + "description": "A source exists, but it failed, timed out, returned an invalid report\nor never returned.", + "type": "string" + }, + { + "const": "unavailable", + "description": "NeMo Fabric holds no value and no evidence for the other states.", + "type": "string" + } + ] + }, + "Fact_for_Array_of_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_Array_of_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_boolean": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_boolean", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "HarnessFacts": { + "description": "Harness facts.", + "properties": { + "name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name of the harness program that ran, as the adapter reports it.\nCore never derives it from the adapter ID." + }, + "version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the harness program that ran, as the adapter reports it." + } + }, + "required": [ + "name", + "version" + ], + "type": "object" + }, + "KnownFact_for_Array_of_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_boolean": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "boolean" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "string" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "McpFacts": { + "description": "MCP facts.", + "properties": { + "servers": { + "additionalProperties": { + "$ref": "#/$defs/McpServerFacts" + }, + "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none.", + "type": "object" + } + }, + "required": [ + "servers" + ], + "type": "object" + }, + "McpRuntimeFacts": { + "description": "What the harness learned about one MCP server when it connected.", + "properties": { + "connected": { + "$ref": "#/$defs/Fact_for_boolean", + "description": "Whether the harness connected to the server." + }, + "protocol_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "MCP protocol version negotiated with the server." + }, + "server_name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name the server reports for itself." + }, + "server_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version the server reports for itself." + }, + "tool_names": { + "$ref": "#/$defs/Fact_for_Array_of_string", + "description": "Raw tool names the server lists, each bounded by the string length\nrule." + } + }, + "required": [ + "connected", + "protocol_version", + "server_name", + "server_version", + "tool_names" + ], + "type": "object" + }, + "McpServerFacts": { + "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, its `transport` may be\nabsent, and runtime facts the adapter did not collect read\n`unavailable` with reason `not_collected`.", + "if": { + "properties": { + "origin": { + "const": "plan" + } + } + }, + "properties": { + "endpoint": { + "$ref": "#/$defs/Fact_for_string", + "description": "Configured endpoint without secrets: the lowercase scheme and host\nand the port for a network server, or the executable's file name for\na `stdio` server." + }, + "origin": { + "description": "Where the entry comes from: `plan` for a server the plan configures,\n`harness` for one only the harness reports.", + "type": "string" + }, + "runtime": { + "$ref": "#/$defs/McpRuntimeFacts", + "description": "What the harness learned when it connected, as the adapter reports it." + }, + "transport": { + "description": "Transport, such as `stdio` or `streamable-http`. Always present for\na `plan` entry.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "origin", + "endpoint", + "runtime" + ], + "then": { + "properties": { + "transport": { + "type": "string" + } + }, + "required": [ + "transport" + ] + }, + "type": "object" + }, + "MissingFact": { + "description": "A fact without a value.", + "not": { + "required": [ + "value" + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "reason": { + "description": "Short code that explains the state, such as `key_not_declared`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "state": { + "$ref": "#/$defs/FactState", + "description": "Why the fact has no value." + } + }, + "required": [ + "state", + "reason", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "ModelFacts": { + "description": "Facts about one model role.", + "properties": { + "provider": { + "description": "Provider from the plan.", + "type": "string" + }, + "requested": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model the plan requested." + }, + "served": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model that served, as the harness or provider reports it." + } + }, + "required": [ + "provider", + "requested", + "served" + ], + "type": "object" + }, "RunStatus": { "description": "Runtime completion status.", "oneOf": [ @@ -322,6 +770,59 @@ } }, "type": "object" + }, + "WhatRan": { + "description": "What executed for one invocation.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact. `RunResult` also checks the schema name and version, and that\nevery `plan` MCP entry names its transport.", + "properties": { + "adapter": { + "$ref": "#/$defs/AdapterFacts", + "description": "Adapter that ran." + }, + "environment": { + "$ref": "#/$defs/EnvironmentFacts", + "description": "Environment the runtime ran in." + }, + "fabric": { + "$ref": "#/$defs/FabricFacts", + "description": "NeMo Fabric core that ran the adapter." + }, + "harness": { + "$ref": "#/$defs/HarnessFacts", + "description": "Harness behind the adapter." + }, + "mcp": { + "$ref": "#/$defs/McpFacts", + "description": "MCP server facts." + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelFacts" + }, + "description": "Model facts keyed by the role names of the plan's `models` settings.", + "type": "object" + }, + "schema": { + "const": "nemo.fabric.what_ran", + "description": "Schema name, `nemo.fabric.what_ran`.", + "type": "string" + }, + "version": { + "const": "v1alpha1", + "description": "Contract version, `v1alpha1`.", + "type": "string" + } + }, + "required": [ + "schema", + "version", + "fabric", + "adapter", + "harness", + "models", + "mcp", + "environment" + ], + "type": "object" } }, "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -411,6 +912,17 @@ } ], "description": "Normalized invocation usage when reported by the adapter target." + }, + "what_ran": { + "anyOf": [ + { + "$ref": "#/$defs/WhatRan" + }, + { + "type": "null" + } + ], + "description": "What executed for this invocation, and how NeMo Fabric learned each\nfact. Absent when the producing core predates it, or when this core\nfinds it malformed or of another schema or version." } }, "required": [ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 7ef06a715..0f23621a6 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -6,6 +6,8 @@ from __future__ import annotations import math +import re +from collections.abc import Callable from collections.abc import Iterator from collections.abc import Mapping from collections.abc import Sequence @@ -1963,6 +1965,133 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: return data +_WHAT_RAN_SCHEMA = "nemo.fabric.what_ran" +_WHAT_RAN_VERSION = "v1alpha1" +_FACT_SOURCES = frozenset({"observed", "reported", "configured"}) +_FACT_STATES = frozenset( + {"not_applicable", "unsupported", "collection_failed", "unavailable"} +) +_FACT_CODE = re.compile(r"[a-z0-9_.:#+/-]{1,128}") + + +def _is_text(value: Any) -> bool: + return isinstance(value, str) + + +def _is_flag(value: Any) -> bool: + return isinstance(value, bool) + + +def _is_text_list(value: Any) -> bool: + return isinstance(value, list) and all(isinstance(item, str) for item in value) + + +def _is_code(value: Any) -> bool: + return isinstance(value, str) and _FACT_CODE.fullmatch(value) is not None + + +def _is_fact(fact: Any, is_value: Callable[[Any], bool]) -> bool: + """Whether ``fact`` holds ``value``, or ``state`` and ``reason``, never both.""" + + if not isinstance(fact, Mapping): + return False + source = fact.get("source") + if not (isinstance(source, str) and source in _FACT_SOURCES): + return False + if not (_is_code(fact.get("collected_by")) and _is_code(fact.get("method"))): + return False + if "value" in fact: + return "state" not in fact and "reason" not in fact and is_value(fact["value"]) + state = fact.get("state") + return ( + isinstance(state, str) + and state in _FACT_STATES + and _is_code(fact.get("reason")) + ) + + +def _has_facts(section: Any, facts: Mapping[str, Callable[[Any], bool]]) -> bool: + return isinstance(section, Mapping) and all( + _is_fact(section.get(name), is_value) for name, is_value in facts.items() + ) + + +def _is_model_entry(model: Any) -> bool: + return _has_facts( + model, {"requested": _is_text, "served": _is_text} + ) and isinstance(model.get("provider"), str) + + +def _is_mcp_server_entry(server: Any) -> bool: + if not _has_facts(server, {"endpoint": _is_text}): + return False + origin = server.get("origin") + transport = server.get("transport") + # Only a server the harness reports may leave out its transport. + transport_ok = isinstance(transport, str) or ( + transport is None and origin != "plan" + ) + runtime_facts = { + "connected": _is_flag, + "protocol_version": _is_text, + "server_name": _is_text, + "server_version": _is_text, + "tool_names": _is_text_list, + } + return ( + isinstance(origin, str) + and transport_ok + and _has_facts(server.get("runtime"), runtime_facts) + ) + + +def _is_readable_what_ran(value: Any) -> bool: + """Whether ``value`` is a ``what_ran`` object this SDK reads. + + Mirrors the Rust core's reader: the schema and version match, every + section and fact is well formed, and unknown properties are ignored. + """ + + if not isinstance(value, Mapping): + return False + if ( + value.get("schema") != _WHAT_RAN_SCHEMA + or value.get("version") != _WHAT_RAN_VERSION + ): + return False + adapter = value.get("adapter") + models = value.get("models") + mcp = value.get("mcp") + servers = mcp.get("servers") if isinstance(mcp, Mapping) else None + environment = value.get("environment") + return ( + _has_facts( + value.get("fabric"), + {"core_version": _is_text, "host_core_version": _is_text}, + ) + and _has_facts(adapter, {"package_version": _is_text}) + and (adapter.get("id") is None or isinstance(adapter.get("id"), str)) + and _has_facts(value.get("harness"), {"name": _is_text, "version": _is_text}) + and isinstance(models, Mapping) + and all(_is_model_entry(model) for model in models.values()) + and isinstance(servers, Mapping) + and all(_is_mcp_server_entry(server) for server in servers.values()) + and isinstance(environment, Mapping) + and isinstance(environment.get("provider"), str) + ) + + +def _readable_what_ran(value: Any) -> Mapping[str, Any] | None: + """Return ``what_ran``, or ``None`` when this SDK cannot read it. + + As in the Rust core, a value that is absent, not a JSON object, names + another schema or version, or breaks the section's structure reads as + absent instead of failing the result. + """ + + return value if _is_readable_what_ran(value) else None + + class RunResult(FabricMapping): """Normalized terminal result from one NeMo Fabric invocation. @@ -1987,6 +2116,13 @@ class RunResult(FabricMapping): telemetry: Ordered telemetry references. events: Ordered lifecycle and invocation events. metadata: Result-specific structured details. + what_ran: What executed for this invocation, schema + ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. + Each fact holds ``value``, or ``state`` and ``reason``, plus + ``source``, ``collected_by`` and ``method``. ``None`` when the + producing core predates it, or when the value is not a + ``what_ran`` object of that schema and version with every section + and fact well formed, as the Rust core reads it. """ agent_name: str @@ -2004,6 +2140,7 @@ class RunResult(FabricMapping): telemetry: Sequence[TelemetryRef] events: Sequence[FabricEvent] metadata: Mapping[str, Any] + what_ran: Mapping[str, Any] | None _fields = frozenset( { "agent_name", @@ -2021,9 +2158,10 @@ class RunResult(FabricMapping): "telemetry", "events", "metadata", + "what_ran", } ) - _json_fields = frozenset({"metadata"}) + _json_fields = frozenset({"metadata", "what_ran"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: @@ -2059,6 +2197,7 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: FabricEvent.from_mapping(event) for event in data.get("events", []) ) data["metadata"] = _mapping(data.get("metadata", {}), "result metadata") + data["what_ran"] = _readable_what_ran(data.get("what_ran")) raw_output = data.get("output") if isinstance(raw_output, RunOutput): data["output"] = raw_output diff --git a/tests/fixtures/what_ran/reader-vectors.json b/tests/fixtures/what_ran/reader-vectors.json new file mode 100644 index 000000000..eb21c95e6 --- /dev/null +++ b/tests/fixtures/what_ran/reader-vectors.json @@ -0,0 +1,809 @@ +{ + "description": "Shared what_ran reader vectors. Each case applies its patch to base: a path with value sets that member (an empty path replaces the whole object), and a path with remove deletes it. Rust and Python readers keep every valid case and read every invalid case as absent; the generated run-result schema accepts and rejects the same cases.", + "base": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": { + "value": "0.6.0", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + }, + "host_core_version": { + "state": "not_applicable", + "reason": "local_run", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + } + }, + "adapter": { + "id": "nvidia.fabric.claude", + "package_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + }, + "harness": { + "name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + }, + "models": { + "default": { + "provider": "anthropic", + "requested": { + "value": "claude-sonnet-4-5", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "served": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + }, + "mcp": { + "servers": { + "docs": { + "origin": "plan", + "transport": "streamable-http", + "endpoint": { + "value": "https://docs.example.com:443", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "protocol_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "tool_names": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + }, + "files": { + "origin": "plan", + "transport": "stdio", + "endpoint": { + "value": "files-mcp", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "protocol_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_name": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "server_version": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + }, + "tool_names": { + "state": "unsupported", + "reason": "key_not_declared", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report" + } + } + } + } + }, + "environment": { + "provider": "local" + } + }, + "valid": [ + { + "name": "local run as core collects it", + "patch": [] + }, + { + "name": "known values of every value type", + "patch": [ + { + "path": [ + "harness", + "version" + ], + "value": { + "value": "2.1.0", + "source": "reported", + "collected_by": "adapter", + "method": "harness_init_message" + } + }, + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": true, + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + }, + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names" + ], + "value": { + "value": [ + "search", + "fetch" + ], + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "task core result relayed by an OpenShell host", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "value": "task_core" + }, + { + "path": [ + "fabric", + "host_core_version" + ], + "value": { + "value": "0.6.0", + "source": "observed", + "collected_by": "host_core", + "method": "crate_version" + } + }, + { + "path": [ + "environment", + "provider" + ], + "value": "openshell" + } + ] + }, + { + "name": "harness-origin MCP entry without a transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "plugin" + ], + "value": { + "origin": "harness", + "endpoint": { + "state": "not_applicable", + "reason": "not_configured", + "source": "configured", + "collected_by": "host_core", + "method": "resolved_plan" + }, + "runtime": { + "connected": { + "value": true, + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "protocol_version": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "server_name": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "server_version": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + }, + "tool_names": { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_status" + } + } + } + } + ] + }, + { + "name": "unknown properties and vendor codes", + "patch": [ + { + "path": [ + "workspace" + ], + "value": { + "base_commit": { + "value": "abc" + } + } + }, + { + "path": [ + "fabric", + "core_version", + "confidence" + ], + "value": "high" + }, + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "value": "acme.collector" + }, + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "acme:withheld#1" + }, + { + "path": [ + "harness", + "version", + "method" + ], + "value": "acme.lookup/v2#names" + } + ] + }, + { + "name": "no adapter ID, models or MCP servers", + "patch": [ + { + "path": [ + "adapter", + "id" + ], + "remove": true + }, + { + "path": [ + "models" + ], + "value": {} + }, + { + "path": [ + "mcp", + "servers" + ], + "value": {} + } + ] + }, + { + "name": "128-character code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + } + ], + "invalid": [ + { + "name": "value with state", + "patch": [ + { + "path": [ + "harness", + "version", + "value" + ], + "value": "2.1.0" + } + ] + }, + { + "name": "value with reason only", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "reason" + ], + "value": "local_run" + } + ] + }, + { + "name": "neither value nor state", + "patch": [ + { + "path": [ + "harness", + "version", + "state" + ], + "remove": true + }, + { + "path": [ + "harness", + "version", + "reason" + ], + "remove": true + } + ] + }, + { + "name": "state without reason", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "remove": true + } + ] + }, + { + "name": "unknown state", + "patch": [ + { + "path": [ + "harness", + "version", + "state" + ], + "value": "withheld" + } + ] + }, + { + "name": "unknown source", + "patch": [ + { + "path": [ + "harness", + "version", + "source" + ], + "value": "guessed" + } + ] + }, + { + "name": "number for a string value", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "value" + ], + "value": 6 + } + ] + }, + { + "name": "null value", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "value" + ], + "value": null + } + ] + }, + { + "name": "string for a boolean value", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": "true", + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "non-string tool name", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names" + ], + "value": { + "value": [ + "search", + 1 + ], + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "method outside the code pattern", + "patch": [ + { + "path": [ + "models", + "default", + "requested", + "method" + ], + "value": "models.Default.model" + } + ] + }, + { + "name": "empty reason", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "" + } + ] + }, + { + "name": "129-character code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + }, + { + "name": "non-ASCII code", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "réason" + } + ] + }, + { + "name": "code with a trailing newline", + "patch": [ + { + "path": [ + "harness", + "version", + "reason" + ], + "value": "key_not_declared\n" + } + ] + }, + { + "name": "number for a boolean value", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "connected" + ], + "value": { + "value": 1, + "source": "reported", + "collected_by": "adapter", + "method": "mcp_initialize" + } + } + ] + }, + { + "name": "missing collected_by", + "patch": [ + { + "path": [ + "fabric", + "core_version", + "collected_by" + ], + "remove": true + } + ] + }, + { + "name": "missing section", + "patch": [ + { + "path": [ + "harness" + ], + "remove": true + } + ] + }, + { + "name": "missing harness name", + "patch": [ + { + "path": [ + "harness", + "name" + ], + "remove": true + } + ] + }, + { + "name": "fact that is not an object", + "patch": [ + { + "path": [ + "adapter", + "package_version" + ], + "value": "1.0.0" + } + ] + }, + { + "name": "model without provider", + "patch": [ + { + "path": [ + "models", + "default", + "provider" + ], + "remove": true + } + ] + }, + { + "name": "plan MCP entry without a transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "transport" + ], + "remove": true + } + ] + }, + { + "name": "plan MCP entry with a null transport", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "transport" + ], + "value": null + } + ] + }, + { + "name": "MCP entry without runtime facts", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime" + ], + "remove": true + } + ] + }, + { + "name": "non-string environment provider", + "patch": [ + { + "path": [ + "environment", + "provider" + ], + "value": null + } + ] + }, + { + "name": "another schema", + "patch": [ + { + "path": [ + "schema" + ], + "value": "acme.what_ran" + } + ] + }, + { + "name": "another version", + "patch": [ + { + "path": [ + "version" + ], + "value": "v1beta1" + } + ] + }, + { + "name": "missing version", + "patch": [ + { + "path": [ + "version" + ], + "remove": true + } + ] + }, + { + "name": "string instead of an object", + "patch": [ + { + "path": [], + "value": "not an object" + } + ] + }, + { + "name": "array instead of an object", + "patch": [ + { + "path": [], + "value": [ + "fact" + ] + } + ] + } + ] +} diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index f571159b7..d58d1b5bd 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -418,6 +418,10 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: assert result["adapter_kind"] == "python" assert result["metadata"]["adapter_runner"] == "persistent_local_host" assert result["output"]["received"] == "hello native" + assert result.what_ran["fabric"]["core_version"]["value"] == native.version() + assert result.what_ran["adapter"]["id"] == "test.fabric.hermes_shim" + assert result.what_ran["harness"]["version"]["reason"] == "key_not_declared" + assert result.what_ran["environment"] == {"provider": "local"} assert result.output["native_mcp_servers"] == ["github"] assert any(artifact.name == "stdout" for artifact in result.artifacts.artifacts) assert first["status"] == "succeeded" diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index b71bea32d..403fb9fa8 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -8,6 +8,7 @@ import asyncio import json import threading +from copy import deepcopy from inspect import signature from pathlib import Path from typing import Any @@ -2001,6 +2002,79 @@ def test_run_result_wraps_normalized_usage(): assert result.usage.metadata == {"provider": "test"} +_WHAT_RAN_VECTORS = json.loads( + ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "what_ran" + / "reader-vectors.json" + ).read_text(encoding="utf-8") +) + + +def _what_ran_vector(case: dict[str, Any]) -> Any: + """Apply one shared reader vector's patch to the base ``what_ran``.""" + + what_ran = deepcopy(_WHAT_RAN_VECTORS["base"]) + for operation in case["patch"]: + path = operation["path"] + if not path: + what_ran = deepcopy(operation["value"]) + continue + *parents, last = path + parent = what_ran + for key in parents: + parent = parent[key] + if operation.get("remove"): + del parent[last] + else: + parent[last] = deepcopy(operation["value"]) + return what_ran + + +def test_run_result_exposes_what_ran_as_a_detached_json_mapping(): + what_ran = _what_ran_vector(_WHAT_RAN_VECTORS["valid"][0]) + result = RunResult.from_mapping(_run_result(what_ran=what_ran)) + + detached = result.what_ran + detached["fabric"]["core_version"]["value"] = "mutated" + + assert result.what_ran == what_ran + assert result["what_ran"] == what_ran + assert result.to_mapping()["what_ran"] == what_ran + assert "what_ran" not in result.extra_fields + + +@pytest.mark.parametrize( + "case", _WHAT_RAN_VECTORS["valid"], ids=lambda case: case["name"] +) +def test_run_result_keeps_valid_what_ran_vector(case): + what_ran = _what_ran_vector(case) + + result = RunResult.from_mapping(_run_result(what_ran=what_ran)) + + assert result.what_ran == what_ran + + +@pytest.mark.parametrize( + "case", _WHAT_RAN_VECTORS["invalid"], ids=lambda case: case["name"] +) +def test_run_result_reads_invalid_what_ran_vector_as_none(case): + result = RunResult.from_mapping(_run_result(what_ran=_what_ran_vector(case))) + + assert result.what_ran is None + assert result.to_mapping()["what_ran"] is None + assert result.status == "succeeded" + + +@pytest.mark.parametrize("mapping", [_run_result(), _run_result(what_ran=None)]) +def test_run_result_without_what_ran_reads_none(mapping): + result = RunResult.from_mapping(mapping) + + assert result.what_ran is None + assert result.to_mapping()["what_ran"] is None + + @pytest.mark.parametrize( "field", ["input_tokens", "cached_input_tokens", "output_tokens", "total_tokens"] ) From 4f16bfff9fbd64ff3389c6b0e00cb79411612070 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sat, 17 Oct 2026 22:22:18 -0700 Subject: [PATCH 060/159] docs: describe RunResult.what_ran in the Python SDK guide Explain the fact envelope, the four no-value states, the endpoint rule, the OpenShell attribution and the structural read, and keep the integration skill's RunResult table in step with the guide. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 74 +++++++++++++++++++ .../references/results-and-errors.md | 10 +++ 2 files changed, 84 insertions(+) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index adafb55f8..fa23c2a6c 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -967,6 +967,7 @@ Important fields: | `telemetry` | References to NeMo Relay or other telemetry streams produced by the run. | | `events` | Ordered normalized lifecycle and invocation events. | | `metadata` | Result-specific structured metadata. | +| `what_ran` | What executed for this invocation, and how NeMo Fabric learned each fact. | | `runtime_id`, `invocation_id`, `request_id` | IDs for correlation across runtimes, logs, telemetry, and artifacts. | These are structured correlation fields, not interchangeable metadata: @@ -996,6 +997,79 @@ If NeMo Fabric cannot resolve config, start a runtime, or obtain a normalized re the SDK raises a `FabricError` subclass instead of returning a partial `RunResult`. +### What Ran + +`what_ran` records what executed for the invocation, as schema +`nemo.fabric.what_ran` version `v1alpha1`. Each fact is an object with a +`value`, or with a `state` and a `reason` when it has no value. Both forms +carry `source`, `collected_by`, and `method`: + +- `source` is `observed` when NeMo Fabric core saw the fact itself, `reported` + when the adapter, harness, an MCP server, or a remote service said so, and + `configured` when it comes from the resolved plan. +- `state` is `not_applicable` when the fact cannot exist for this run, + `unsupported` when there is no source for it, `collection_failed` when the + source failed, and `unavailable` when NeMo Fabric holds no value and no + evidence for the other states. +- `collected_by`, `method`, and `reason` are open codes that match + `^[a-z0-9_.:#+/-]{1,128}$`. Accept values you do not recognize. + `collected_by` is `task_core` for facts a core inside an OpenShell sandbox + collected. `method` names the technique, such as `crate_version`, + `resolved_plan`, or `adapter_report`. + +| Part | Facts | +| --- | --- | +| `fabric` | `core_version` of the core that ran the adapter. `host_core_version` is `not_applicable` with reason `local_run` in a local run, and the host core's version in an OpenShell run. | +| `adapter` | `id` from the plan, and the reported `package_version`. | +| `harness` | The reported `name` and `version` of the harness program. | +| `models.` | `provider` and the `requested` model from the plan, and the reported `served` model. | +| `mcp.servers.` | `origin`, `transport`, the configured `endpoint`, and a `runtime` block with the reported `connected`, `protocol_version`, `server_name`, `server_version`, and `tool_names`. | +| `environment` | The environment `provider`, such as `local` or `openshell`. | + +NeMo Fabric reads no adapter-reported facts yet, so every reported fact has +state `unsupported` with reason `key_not_declared` and method +`adapter_report`. + +Today every MCP entry has `origin` `plan`. An entry with `origin` `harness` +describes a server the harness reports but the plan does not configure. It has +the same shape, except that `transport` can be absent and `endpoint` has state +`not_applicable` with reason `not_configured`. + +`endpoint` keeps no secret. A network server's endpoint is its lowercase scheme +and host and its port. A URL without a port gets 80 for `http` and `ws` and 443 +for `https` and `wss`, so `https://example.com` and `https://example.com:443` +give the same endpoint. An IPv6 host keeps its brackets, in compressed lowercase +form such as `[2001:db8::1]`. A `stdio` server's endpoint is the file name of +its executable, such as `server.exe` for `C:\MCP\server.exe`. +User info, paths, queries, fragments, headers, arguments, and environment values +are never copied. + +An endpoint NeMo Fabric cannot reduce safely has state `collection_failed` with +reason `endpoint_unparsed` or `executable_unparsed`. Examples are a URL with an +`@` after the host, a bracketed host that is not an IPv6 address, and an +executable that contains `=` or any whitespace, even in a folder name. An empty +URL or executable has state `unavailable` with reason `not_configured`. + +A string value longer than 256 characters keeps its first 240 characters, then +`~`, then the first 15 hex digits of its SHA-256. The rule applies to each +element of a list value too. + +`what_ran` is `None` when the core that produced the result predates it. The +Rust core and the Python SDK apply the same structural checks. A `what_ran` +that is not an object, names another schema or version, or has a malformed +section or fact reads as `None`, and the rest of the result is unchanged. A +fact with both `value` and `state` is malformed. Unknown properties are +ignored. Treat `what_ran` as evidence, not as a key for grouping runs. + +```python +what_ran = result.what_ran +if what_ran is not None: + core = what_ran["fabric"]["core_version"] + print(core.get("value"), core["source"]) + for role, model in what_ran["models"].items(): + print(role, model["requested"].get("value"), model["served"].get("state")) +``` + ## Events Each `RunResult` includes the normalized events collected for that invocation. diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index f2660d5e3..b9b51edfc 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -21,6 +21,7 @@ Every invocation that reaches the adapter boundary returns a normalized | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | | `metadata` | Result-specific structured metadata. | +| `what_ran` | What executed for this invocation: core version, adapter ID, requested models, configured MCP servers, and environment provider, each fact with its source, or `None` when the core predates it. | | `runtime_id`, `invocation_id`, `request_id` | Correlation IDs across runtimes, logs, telemetry, and artifacts. | Handle a result with this status-first pattern: @@ -46,6 +47,15 @@ if result.usage is not None: Treat missing counters and cost as unknown, not zero. `cached_input_tokens` reports cached prompt tokens; `input_tokens_include_cache` declares whether `input_tokens` includes them (`True`), excludes them (`False`), or has unknown semantics (absent). Do not add cache to an inclusive input count. Adapters can put cache-write, reasoning, peak request, and per-model detail in the descriptor-declared `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` object. NeMo Fabric preserves that object at `result.usage["metadata"]["nemo_fabric.detailed_token_usage.v1"]`. Treat detailed input and output independently: any detailed accounting field for a dimension makes that dimension authoritative, with no fallback to its top-level counters. The views can diverge. Calculate an inclusive total only when the selected detailed counters provide enough information; otherwise leave the total unknown. Keep estimated costs separate in provider metadata; `cost_usd` is reported cost, not an estimate. +Read `what_ran` as evidence of what executed, never as a grouping key. Each +fact holds a `value`, or a `state` and `reason` when it has no value, plus +`source`, `collected_by`, and `method`. Accept `collected_by`, `method`, and +`reason` codes you do not recognize. Adapter-reported facts, such as the +harness version and the served model, currently read state `unsupported` with +reason `key_not_declared`. A malformed `what_ran` reads as `None` without +failing the result. Refer to +[What Ran](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/sdk/python.mdx#what-ran). + ## Correlation IDs `runtime_id` identifies the runtime lifecycle, `invocation_id` identifies one From 26f06b06464132e1ec72ced8db0b32d308b8f6d5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sat, 17 Oct 2026 22:22:19 -0700 Subject: [PATCH 061/159] docs: regenerate API references for what_ran Regenerate the Rust library reference with Rust 1.94.0 for the new what_ran module and the RunResult field, and the Python types reference for RunResult.what_ran. Only the pages this change affects are included. Signed-off-by: mnajafian-nv --- .../nemo_fabric.types.md | 2 + .../nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/index.mdx | 1 + .../runtime/struct-runresult.mdx | 6 +- .../what-ran/constant-what-ran-schema.mdx | 15 ++ .../what-ran/constant-what-ran-version.mdx | 15 ++ .../nemo-fabric-core/what-ran/enum-fact.mdx | 109 +++++++++++++++ .../what-ran/enum-factsource.mdx | 123 +++++++++++++++++ .../what-ran/enum-factstate.mdx | 130 ++++++++++++++++++ .../nemo-fabric-core/what-ran/index.mdx | 52 +++++++ .../what-ran/struct-adapterfacts.mdx | 103 ++++++++++++++ .../what-ran/struct-environmentfacts.mdx | 99 +++++++++++++ .../what-ran/struct-fabricfacts.mdx | 103 ++++++++++++++ .../what-ran/struct-factorigin.mdx | 107 ++++++++++++++ .../what-ran/struct-harnessfacts.mdx | 103 ++++++++++++++ .../what-ran/struct-knownfact.mdx | 103 ++++++++++++++ .../what-ran/struct-mcpfacts.mdx | 99 +++++++++++++ .../what-ran/struct-mcpruntimefacts.mdx | 115 ++++++++++++++++ .../what-ran/struct-mcpserverfacts.mdx | 113 +++++++++++++++ .../what-ran/struct-missingfact.mdx | 107 ++++++++++++++ .../what-ran/struct-modelfacts.mdx | 107 ++++++++++++++ .../what-ran/struct-whatran.mdx | 129 +++++++++++++++++ 22 files changed, 1741 insertions(+), 2 deletions(-) create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index ea149fc81..7664714b5 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1443,6 +1443,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `telemetry`: Ordered telemetry references. - `events`: Ordered lifecycle and invocation events. - `metadata`: Result-specific structured details. + - `what_ran`: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus ``source``, ``collected_by`` and ``method``. ``None`` when the producing core predates it, or when the value is not a ``what_ran`` object of that schema and version with every section and fact well formed, as the Rust core reads it. @@ -1467,6 +1468,7 @@ The mapping exposes the following typed fields: | `telemetry` | `Sequence[TelemetryRef]` | | `events` | `Sequence[FabricEvent]` | | `metadata` | `Mapping[str, Any]` | +| `what_ran` | `Mapping[str, Any] \| None` | ### method `__init__` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 8af97f312..3a81be390 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 133 +position: 138 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 08ae7ee42..6a951bfe9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -152,6 +152,7 @@ Core config and runtime contract for NeMo Fabric. - [error](error/index.mdx): Error types for NeMo Fabric core. - [runtime](runtime/index.mdx): Runtime invocation helpers. - [schema](schema/index.mdx): JSON Schema generation for the public NeMo Fabric contract. +- [what_ran](what-ran/index.mdx): What executed for one invocation, and how NeMo Fabric learned each fact. ## Functions diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 1cb5bd052..9dfc3f1be 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
Result from a NeMo Fabric-managed harness invocation. @@ -75,6 +75,10 @@ NeMo Fabric lifecycle/progress events emitted during the run. Adapter-specific metadata. +### `what_ran: Option` + +What executed for this invocation, and how NeMo Fabric learned each fact. Absent when the producing core predates it, or when this core finds it malformed or of another schema or version. + ## Trait Implementations ### `impl Clone for RunResult` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx new file mode 100644 index 000000000..06068c300 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant WHAT_RAN_SCHEMA" +sidebar-title: "WHAT_RAN_SCHEMA" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-schema" +description: "Schema name written into every `WhatRan`." +position: 16 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo.fabric.what_ran\";"}} />
+ +Schema name written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx new file mode 100644 index 000000000..01dc53369 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant WHAT_RAN_VERSION" +sidebar-title: "WHAT_RAN_VERSION" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-version" +description: "Contract version written into every `WhatRan`." +position: 17 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"v1alpha1\";"}} />
+ +Contract version written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx new file mode 100644 index 000000000..55c94c912 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx @@ -0,0 +1,109 @@ +--- +title: "Enum Fact" +sidebar-title: "Fact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-fact" +description: "One fact: a `value`, or a `state` with a `reason`, never both." +position: 13 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
KnownFact<T>),\n    Missing(MissingFact),\n}"}} />
+ +One fact: a `value`, or a `state` with a `reason`, never both. + +## Variants + +### `Known(KnownFact)` + +
KnownFact<T>)"}} />
+ +The fact has a value. + +### `Missing(MissingFact)` + +
MissingFact)"}} />
+ +The fact has no value. + +## Trait Implementations + +### `impl Clone for Fact` + +
Clone> Clone for Fact<T>"}} />
+ +#### `clone` + +
clone(&self) -> Fact<T>"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for Fact` + +
Debug> Debug for Fact<T>"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de, T: DeserializeOwned> Deserialize<'de> for Fact` + +
DeserializeOwned> Deserialize<'de> for Fact<T>"}} />
+ +#### `deserialize` + +
deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error>"}} />
+ +Decode a fact, rejecting one that holds `value` together with `state` or `reason`, or an open code outside its pattern. Unknown properties are ignored. + +### `impl JsonSchema for Factwhere T: JsonSchema,` + +
Fact<T>where\n    T: JsonSchema,"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for Fact` + +
PartialEq> PartialEq for Fact<T>"}} />
+ +#### `eq` + +
eq(&self, other: &Fact<T>) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for Factwhere T: Serialize,` + +
Serialize for Fact<T>where\n    T: Serialize,"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for Fact` + +
Eq> Eq for Fact<T>"}} />
+ +### `impl StructuralPartialEq for Fact` + +
StructuralPartialEq for Fact<T>"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx new file mode 100644 index 000000000..66a4e30c5 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx @@ -0,0 +1,123 @@ +--- +title: "Enum Fact Source" +sidebar-title: "FactSource" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factsource" +description: "How NeMo Fabric learned a fact." +position: 14 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum FactSource { + Observed, + Reported, + Configured, +} +``` + +How NeMo Fabric learned a fact. + +## Variants + +### `Observed` + +
+ +NeMo Fabric core saw it itself. + +### `Reported` + +
+ +The adapter, the harness, an MCP server or a remote service said so. + +### `Configured` + +
+ +It comes from the resolved run plan. + +## Trait Implementations + +### `impl Clone for FactSource` + +
Clone for FactSource"}} />
+ +#### `clone` + +
clone(&self) -> FactSource"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactSource` + +
Debug for FactSource"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactSource` + +
Deserialize<'de> for FactSource"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactSource` + +
FactSource"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactSource` + +
PartialEq for FactSource"}} />
+ +#### `eq` + +
eq(&self, other: &FactSource) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactSource` + +
Serialize for FactSource"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for FactSource` + +
Copy for FactSource"}} />
+ +### `impl Eq for FactSource` + +
Eq for FactSource"}} />
+ +### `impl StructuralPartialEq for FactSource` + +
StructuralPartialEq for FactSource"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx new file mode 100644 index 000000000..9fd1a13cf --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx @@ -0,0 +1,130 @@ +--- +title: "Enum Fact State" +sidebar-title: "FactState" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factstate" +description: "Why a fact has no value. A collector picks the first state that matches, in declaration order." +position: 15 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum FactState { + NotApplicable, + Unsupported, + CollectionFailed, + Unavailable, +} +``` + +Why a fact has no value. A collector picks the first state that matches, in declaration order. + +## Variants + +### `NotApplicable` + +
+ +The fact cannot exist for this run. + +### `Unsupported` + +
+ +The adapter or environment provider has no source for the fact. + +### `CollectionFailed` + +
+ +A source exists, but it failed, timed out, returned an invalid report or never returned. + +### `Unavailable` + +
+ +NeMo Fabric holds no value and no evidence for the other states. + +## Trait Implementations + +### `impl Clone for FactState` + +
Clone for FactState"}} />
+ +#### `clone` + +
clone(&self) -> FactState"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactState` + +
Debug for FactState"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactState` + +
Deserialize<'de> for FactState"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactState` + +
FactState"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactState` + +
PartialEq for FactState"}} />
+ +#### `eq` + +
eq(&self, other: &FactState) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactState` + +
Serialize for FactState"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for FactState` + +
Copy for FactState"}} />
+ +### `impl Eq for FactState` + +
Eq for FactState"}} />
+ +### `impl StructuralPartialEq for FactState` + +
StructuralPartialEq for FactState"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx new file mode 100644 index 000000000..18ba5a872 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -0,0 +1,52 @@ +--- +title: "Module what_ran" +sidebar-title: "what_ran" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran" +description: "What executed for one invocation, and how NeMo Fabric learned each fact." +position: 137 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +What executed for one invocation, and how NeMo Fabric learned each fact. + +[`WhatRan`](struct-whatran.mdx) rides on [`RunResult`](../runtime/struct-runresult.mdx). Each fact is a [`Fact`](enum-fact.mdx): a value, or a [`FactState`](enum-factstate.mdx) with a reason, and in both cases the [`FactSource`](enum-factsource.mdx), the component that collected it and the method it used. A few names fixed by the run plan stay plain strings: the adapter ID, each model role's provider, the environment provider, and each MCP server's origin and transport. + +A string value longer than 256 characters, in a string fact or as an element of a string-list fact, keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. Map keys and the plain names above are never cut. + +`what_ran` is evidence. It never decides which runs are compared. + +Compatibility rules for readers: + +- `source` and `state` are closed sets. A new value changes [`WHAT_RAN_VERSION`](constant-what-ran-version.mdx). +- `collected_by`, `method` and `reason` are open. Readers accept values they do not know. +- Readers accept unknown properties. Adding an optional property keeps [`WHAT_RAN_VERSION`](constant-what-ran-version.mdx); removing one or changing its meaning changes it. The Rust types drop unknown properties when they decode, so a host that relays a task core's result drops them too. +- A fact holds `value`, or `state` and `reason`, never both. A `RunResult` reader that finds a fact breaking this, any other value the schema rejects, or another schema or version reads `what_ran` as absent and still decodes the result. + +## Structs + +- [AdapterFacts](struct-adapterfacts.mdx): Adapter facts. +- [EnvironmentFacts](struct-environmentfacts.mdx): Environment facts. +- [FabricFacts](struct-fabricfacts.mdx): NeMo Fabric core facts. +- [FactOrigin](struct-factorigin.mdx): Who collected a fact and how. +- [HarnessFacts](struct-harnessfacts.mdx): Harness facts. +- [KnownFact](struct-knownfact.mdx): A fact with a value. +- [McpFacts](struct-mcpfacts.mdx): MCP facts. +- [McpRuntimeFacts](struct-mcpruntimefacts.mdx): What the harness learned about one MCP server when it connected. +- [McpServerFacts](struct-mcpserverfacts.mdx): Facts about one MCP server. +- [MissingFact](struct-missingfact.mdx): A fact without a value. +- [ModelFacts](struct-modelfacts.mdx): Facts about one model role. +- [WhatRan](struct-whatran.mdx): What executed for one invocation. + +## Enums + +- [Fact](enum-fact.mdx): One fact: a `value`, or a `state` with a `reason`, never both. +- [FactSource](enum-factsource.mdx): How NeMo Fabric learned a fact. +- [FactState](enum-factstate.mdx): Why a fact has no value. A collector picks the first state that matches, in declaration order. + +## Constants + +- [WHAT_RAN_SCHEMA](constant-what-ran-schema.mdx): Schema name written into every [`WhatRan`](struct-whatran.mdx). +- [WHAT_RAN_VERSION](constant-what-ran-version.mdx): Contract version written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx new file mode 100644 index 000000000..dcd1f6145 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-adapterfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Adapter Facts" +sidebar-title: "AdapterFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-adapterfacts" +description: "Adapter facts." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<String>,\n    pub package_version: Fact<String>,\n}"}} />
+ +Adapter facts. + +## Fields + +### `id: Option` + +Adapter ID from the plan, when an adapter was resolved. + +### `package_version: Fact` + +Version of the adapter package, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for AdapterFacts` + +
Clone for AdapterFacts"}} />
+ +#### `clone` + +
clone(&self) -> AdapterFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for AdapterFacts` + +
Debug for AdapterFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for AdapterFacts` + +
Deserialize<'de> for AdapterFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for AdapterFacts` + +
AdapterFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for AdapterFacts` + +
PartialEq for AdapterFacts"}} />
+ +#### `eq` + +
eq(&self, other: &AdapterFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for AdapterFacts` + +
Serialize for AdapterFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for AdapterFacts` + +
Eq for AdapterFacts"}} />
+ +### `impl StructuralPartialEq for AdapterFacts` + +
StructuralPartialEq for AdapterFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx new file mode 100644 index 000000000..f4b0a06b0 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-environmentfacts.mdx @@ -0,0 +1,99 @@ +--- +title: "Struct Environment Facts" +sidebar-title: "EnvironmentFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-environmentfacts" +description: "Environment facts." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n}"}} />
+ +Environment facts. + +## Fields + +### `provider: String` + +Environment provider, such as `local`. + +## Trait Implementations + +### `impl Clone for EnvironmentFacts` + +
Clone for EnvironmentFacts"}} />
+ +#### `clone` + +
clone(&self) -> EnvironmentFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for EnvironmentFacts` + +
Debug for EnvironmentFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for EnvironmentFacts` + +
Deserialize<'de> for EnvironmentFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for EnvironmentFacts` + +
EnvironmentFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for EnvironmentFacts` + +
PartialEq for EnvironmentFacts"}} />
+ +#### `eq` + +
eq(&self, other: &EnvironmentFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for EnvironmentFacts` + +
Serialize for EnvironmentFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for EnvironmentFacts` + +
Eq for EnvironmentFacts"}} />
+ +### `impl StructuralPartialEq for EnvironmentFacts` + +
StructuralPartialEq for EnvironmentFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx new file mode 100644 index 000000000..eea91521c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-fabricfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Fabric Facts" +sidebar-title: "FabricFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-fabricfacts" +description: "NeMo Fabric core facts." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<String>,\n    pub host_core_version: Fact<String>,\n}"}} />
+ +NeMo Fabric core facts. + +## Fields + +### `core_version: Fact` + +Version of the core that ran the adapter. + +### `host_core_version: Fact` + +Version of the host core that relayed the result from a task core in another environment. `not_applicable` for a local run. + +## Trait Implementations + +### `impl Clone for FabricFacts` + +
Clone for FabricFacts"}} />
+ +#### `clone` + +
clone(&self) -> FabricFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FabricFacts` + +
Debug for FabricFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FabricFacts` + +
Deserialize<'de> for FabricFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FabricFacts` + +
FabricFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FabricFacts` + +
PartialEq for FabricFacts"}} />
+ +#### `eq` + +
eq(&self, other: &FabricFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FabricFacts` + +
Serialize for FabricFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for FabricFacts` + +
Eq for FabricFacts"}} />
+ +### `impl StructuralPartialEq for FabricFacts` + +
StructuralPartialEq for FabricFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx new file mode 100644 index 000000000..513cb6af9 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-factorigin.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Fact Origin" +sidebar-title: "FactOrigin" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-factorigin" +description: "Who collected a fact and how." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactSource,\n    pub collected_by: String,\n    pub method: String,\n}"}} />
+ +Who collected a fact and how. + +## Fields + +### `source: FactSource` + +How NeMo Fabric learned the fact. + +### `collected_by: String` + +Component that collected the fact, such as `host_core`, `task_core` or `adapter`. + +### `method: String` + +How the collector got the fact, such as `crate_version`, `resolved_plan` or `adapter_report`. + +## Trait Implementations + +### `impl Clone for FactOrigin` + +
Clone for FactOrigin"}} />
+ +#### `clone` + +
clone(&self) -> FactOrigin"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for FactOrigin` + +
Debug for FactOrigin"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for FactOrigin` + +
Deserialize<'de> for FactOrigin"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for FactOrigin` + +
FactOrigin"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for FactOrigin` + +
PartialEq for FactOrigin"}} />
+ +#### `eq` + +
eq(&self, other: &FactOrigin) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for FactOrigin` + +
Serialize for FactOrigin"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for FactOrigin` + +
Eq for FactOrigin"}} />
+ +### `impl StructuralPartialEq for FactOrigin` + +
StructuralPartialEq for FactOrigin"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx new file mode 100644 index 000000000..a683a0d5f --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-harnessfacts.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Harness Facts" +sidebar-title: "HarnessFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-harnessfacts" +description: "Harness facts." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<String>,\n    pub version: Fact<String>,\n}"}} />
+ +Harness facts. + +## Fields + +### `name: Fact` + +Name of the harness program that ran, as the adapter reports it. Core never derives it from the adapter ID. + +### `version: Fact` + +Version of the harness program that ran, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for HarnessFacts` + +
Clone for HarnessFacts"}} />
+ +#### `clone` + +
clone(&self) -> HarnessFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for HarnessFacts` + +
Debug for HarnessFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for HarnessFacts` + +
Deserialize<'de> for HarnessFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for HarnessFacts` + +
HarnessFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for HarnessFacts` + +
PartialEq for HarnessFacts"}} />
+ +#### `eq` + +
eq(&self, other: &HarnessFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for HarnessFacts` + +
Serialize for HarnessFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for HarnessFacts` + +
Eq for HarnessFacts"}} />
+ +### `impl StructuralPartialEq for HarnessFacts` + +
StructuralPartialEq for HarnessFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx new file mode 100644 index 000000000..8e559977a --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-knownfact.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Known Fact" +sidebar-title: "KnownFact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-knownfact" +description: "A fact with a value." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactOrigin,\n}"}} />
+ +A fact with a value. + +## Fields + +### `value: T` + +The fact itself: a string, a boolean or a list of strings. + +### `origin: FactOrigin` + +Who collected the fact and how. + +## Trait Implementations + +### `impl Clone for KnownFact` + +
Clone> Clone for KnownFact<T>"}} />
+ +#### `clone` + +
clone(&self) -> KnownFact<T>"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for KnownFact` + +
Debug> Debug for KnownFact<T>"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de, T> Deserialize<'de> for KnownFactwhere T: Deserialize<'de>,` + +
Deserialize<'de> for KnownFact<T>where\n    T: Deserialize<'de>,"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for KnownFactwhere T: JsonSchema,` + +
KnownFact<T>where\n    T: JsonSchema,"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for KnownFact` + +
PartialEq> PartialEq for KnownFact<T>"}} />
+ +#### `eq` + +
eq(&self, other: &KnownFact<T>) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for KnownFactwhere T: Serialize,` + +
Serialize for KnownFact<T>where\n    T: Serialize,"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for KnownFact` + +
Eq> Eq for KnownFact<T>"}} />
+ +### `impl StructuralPartialEq for KnownFact` + +
StructuralPartialEq for KnownFact<T>"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx new file mode 100644 index 000000000..c5ddca739 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx @@ -0,0 +1,99 @@ +--- +title: "Struct McpFacts" +sidebar-title: "McpFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpfacts" +description: "MCP facts." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
BTreeMap<String, McpServerFacts>,\n}"}} />
+ +MCP facts. + +## Fields + +### `servers: BTreeMap` + +Servers keyed by name. Empty when the plan configures no MCP server and the harness reports none. + +## Trait Implementations + +### `impl Clone for McpFacts` + +
Clone for McpFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpFacts` + +
Debug for McpFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpFacts` + +
Deserialize<'de> for McpFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpFacts` + +
McpFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpFacts` + +
PartialEq for McpFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpFacts` + +
Serialize for McpFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpFacts` + +
Eq for McpFacts"}} />
+ +### `impl StructuralPartialEq for McpFacts` + +
StructuralPartialEq for McpFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx new file mode 100644 index 000000000..afa1bece5 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx @@ -0,0 +1,115 @@ +--- +title: "Struct McpRuntime Facts" +sidebar-title: "McpRuntimeFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpruntimefacts" +description: "What the harness learned about one MCP server when it connected." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Fact<bool>,\n    pub protocol_version: Fact<String>,\n    pub server_name: Fact<String>,\n    pub server_version: Fact<String>,\n    pub tool_names: Fact<Vec<String>>,\n}"}} />
+ +What the harness learned about one MCP server when it connected. + +## Fields + +### `connected: Fact` + +Whether the harness connected to the server. + +### `protocol_version: Fact` + +MCP protocol version negotiated with the server. + +### `server_name: Fact` + +Name the server reports for itself. + +### `server_version: Fact` + +Version the server reports for itself. + +### `tool_names: Fact>` + +Raw tool names the server lists, each bounded by the string length rule. + +## Trait Implementations + +### `impl Clone for McpRuntimeFacts` + +
Clone for McpRuntimeFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpRuntimeFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpRuntimeFacts` + +
Debug for McpRuntimeFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpRuntimeFacts` + +
Deserialize<'de> for McpRuntimeFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpRuntimeFacts` + +
McpRuntimeFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpRuntimeFacts` + +
PartialEq for McpRuntimeFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpRuntimeFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpRuntimeFacts` + +
Serialize for McpRuntimeFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpRuntimeFacts` + +
Eq for McpRuntimeFacts"}} />
+ +### `impl StructuralPartialEq for McpRuntimeFacts` + +
StructuralPartialEq for McpRuntimeFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx new file mode 100644 index 000000000..19730218c --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx @@ -0,0 +1,113 @@ +--- +title: "Struct McpServer Facts" +sidebar-title: "McpServerFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-mcpserverfacts" +description: "Facts about one MCP server." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub transport: Option<String>,\n    pub endpoint: Fact<String>,\n    pub runtime: McpRuntimeFacts,\n}"}} />
+ +Facts about one MCP server. + +An entry with origin `harness` is a server the harness reports and the plan does not configure. It has the same shape: its `endpoint` reads `not_applicable` with reason `not_configured`, its `transport` may be absent, and runtime facts the adapter did not collect read `unavailable` with reason `not_collected`. + +## Fields + +### `origin: String` + +Where the entry comes from: `plan` for a server the plan configures, `harness` for one only the harness reports. + +### `transport: Option` + +Transport, such as `stdio` or `streamable-http`. Always present for a `plan` entry. + +### `endpoint: Fact` + +Configured endpoint without secrets: the lowercase scheme and host and the port for a network server, or the executable's file name for a `stdio` server. + +### `runtime: McpRuntimeFacts` + +What the harness learned when it connected, as the adapter reports it. + +## Trait Implementations + +### `impl Clone for McpServerFacts` + +
Clone for McpServerFacts"}} />
+ +#### `clone` + +
clone(&self) -> McpServerFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for McpServerFacts` + +
Debug for McpServerFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for McpServerFacts` + +
Deserialize<'de> for McpServerFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for McpServerFacts` + +
McpServerFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for McpServerFacts` + +
PartialEq for McpServerFacts"}} />
+ +#### `eq` + +
eq(&self, other: &McpServerFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for McpServerFacts` + +
Serialize for McpServerFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for McpServerFacts` + +
Eq for McpServerFacts"}} />
+ +### `impl StructuralPartialEq for McpServerFacts` + +
StructuralPartialEq for McpServerFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx new file mode 100644 index 000000000..04a2c23d3 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-missingfact.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Missing Fact" +sidebar-title: "MissingFact" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-missingfact" +description: "A fact without a value." +position: 10 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
FactState,\n    pub reason: String,\n    pub origin: FactOrigin,\n}"}} />
+ +A fact without a value. + +## Fields + +### `state: FactState` + +Why the fact has no value. + +### `reason: String` + +Short code that explains the state, such as `key_not_declared`. + +### `origin: FactOrigin` + +Who would have collected the fact and how. + +## Trait Implementations + +### `impl Clone for MissingFact` + +
Clone for MissingFact"}} />
+ +#### `clone` + +
clone(&self) -> MissingFact"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for MissingFact` + +
Debug for MissingFact"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for MissingFact` + +
Deserialize<'de> for MissingFact"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for MissingFact` + +
MissingFact"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for MissingFact` + +
PartialEq for MissingFact"}} />
+ +#### `eq` + +
eq(&self, other: &MissingFact) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for MissingFact` + +
Serialize for MissingFact"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for MissingFact` + +
Eq for MissingFact"}} />
+ +### `impl StructuralPartialEq for MissingFact` + +
StructuralPartialEq for MissingFact"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx new file mode 100644 index 000000000..e42753677 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-modelfacts.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Model Facts" +sidebar-title: "ModelFacts" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-modelfacts" +description: "Facts about one model role." +position: 11 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub requested: Fact<String>,\n    pub served: Fact<String>,\n}"}} />
+ +Facts about one model role. + +## Fields + +### `provider: String` + +Provider from the plan. + +### `requested: Fact` + +Model the plan requested. + +### `served: Fact` + +Model that served, as the harness or provider reports it. + +## Trait Implementations + +### `impl Clone for ModelFacts` + +
Clone for ModelFacts"}} />
+ +#### `clone` + +
clone(&self) -> ModelFacts"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ModelFacts` + +
Debug for ModelFacts"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for ModelFacts` + +
Deserialize<'de> for ModelFacts"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ModelFacts` + +
ModelFacts"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ModelFacts` + +
PartialEq for ModelFacts"}} />
+ +#### `eq` + +
eq(&self, other: &ModelFacts) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ModelFacts` + +
Serialize for ModelFacts"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for ModelFacts` + +
Eq for ModelFacts"}} />
+ +### `impl StructuralPartialEq for ModelFacts` + +
StructuralPartialEq for ModelFacts"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx new file mode 100644 index 000000000..d25aaa76f --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx @@ -0,0 +1,129 @@ +--- +title: "Struct WhatRan" +sidebar-title: "WhatRan" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-whatran" +description: "What executed for one invocation." +position: 12 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub version: String,\n    pub fabric: FabricFacts,\n    pub adapter: AdapterFacts,\n    pub harness: HarnessFacts,\n    pub models: BTreeMap<String, ModelFacts>,\n    pub mcp: McpFacts,\n    pub environment: EnvironmentFacts,\n}"}} />
+ +What executed for one invocation. + +Decoding a `WhatRan` directly checks the shape of every section and fact. `RunResult` also checks the schema name and version, and that every `plan` MCP entry names its transport. + +## Fields + +### `schema: String` + +Schema name, `nemo.fabric.what_ran`. + +### `version: String` + +Contract version, `v1alpha1`. + +### `fabric: FabricFacts` + +NeMo Fabric core that ran the adapter. + +### `adapter: AdapterFacts` + +Adapter that ran. + +### `harness: HarnessFacts` + +Harness behind the adapter. + +### `models: BTreeMap` + +Model facts keyed by the role names of the plan's `models` settings. + +### `mcp: McpFacts` + +MCP server facts. + +### `environment: EnvironmentFacts` + +Environment the runtime ran in. + +## Trait Implementations + +### `impl Clone for WhatRan` + +
Clone for WhatRan"}} />
+ +#### `clone` + +
clone(&self) -> WhatRan"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for WhatRan` + +
Debug for WhatRan"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for WhatRan` + +
Deserialize<'de> for WhatRan"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for WhatRan` + +
WhatRan"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for WhatRan` + +
PartialEq for WhatRan"}} />
+ +#### `eq` + +
eq(&self, other: &WhatRan) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for WhatRan` + +
Serialize for WhatRan"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for WhatRan` + +
Eq for WhatRan"}} />
+ +### `impl StructuralPartialEq for WhatRan` + +
StructuralPartialEq for WhatRan"}} />
From 638cb2eef7e70d0c20f1f53df51af2cf5b458f47 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 16:22:16 -0700 Subject: [PATCH 062/159] feat: bound what_ran size and entry counts A plan with many model roles or MCP servers, or very long names, could make what_ran arbitrarily large, and every RunResult carries it. Keep it bounded without ever failing a run. Core keeps the first 16 model roles and 32 MCP servers in sorted key order, and drops entries whose key is longer than 128 characters. While the compact JSON of what_ran is larger than 128 KiB, it drops MCP servers from the end of the sorted order, then model roles. If plan identifiers alone still exceed the budget, what_ran becomes a minimal object with only the schema name, the version and what_ran_truncated. A marker next to each map counts the dropped entries and names the last rule applied. The OpenShell host applies the same bounds again after it attributes the task core's facts, so they hold for a task core of any version. The six what_ran sections become optional in the Rust type so the minimal object can be represented. Every section is still required unless what_ran_truncated is present, and no section may be null. The Rust reader, the generated schema and the Python reader check this against the shared vectors. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 77 ++ crates/fabric-core/src/runtime/openshell.rs | 47 ++ crates/fabric-core/src/what_ran.rs | 657 ++++++++++++++++-- schemas/sdk/run-result.schema.json | 69 +- .../src/nemo_fabric/types.py | 117 +++- tests/fixtures/what_ran/reader-vectors.json | 214 ++++++ 6 files changed, 1089 insertions(+), 92 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 893914ff9..1b5493545 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -5501,6 +5501,83 @@ for line in sys.stdin: } } + #[test] + fn local_host_what_ran_bounds_keep_the_status_and_error() { + let model = |provider: &str| -> crate::config::ModelConfig { + serde_json::from_value(serde_json::json!({"provider": provider, "model": "m"})) + .expect("model config") + }; + // One `default` role keeps the adapter's single-role check satisfied. + let roles = || { + std::iter::once("default".to_string()) + .chain((0..16).map(|index| format!("role-{index:02}"))) + }; + for mode in ["success", "adapter_reported_failure"] { + let (root, plan) = local_host_plan(mode); + let baseline = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + let mut accepting_models = plan.clone(); + accepting_models + .adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .config + .accepts + .push(crate::config::AdapterConfigField::Models); + + let mut past_the_cap = accepting_models.clone(); + past_the_cap.config.models = roles().map(|role| (role, model("openai"))).collect(); + let mut past_the_budget = accepting_models.clone(); + past_the_budget.config.models = roles() + .map(|role| (role, model(&"p".repeat(9_000)))) + .collect(); + let mut minimal = plan.clone(); + minimal + .adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .adapter_id = "a".repeat(140_000); + + for (case, bounded_plan) in [ + ("entry cap", past_the_cap), + ("byte budget", past_the_budget), + ("minimal", minimal), + ] { + let result = + run_plan(&bounded_plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, baseline.status, "{case}"); + assert_eq!( + serde_json::to_value(&result.error).expect("error JSON"), + serde_json::to_value(&baseline.error).expect("error JSON"), + "{case}" + ); + let what_ran = result.what_ran.expect("what_ran"); + let marker = |marker: Option| { + marker.map(|marker| (marker.reason, marker.dropped)) + }; + match case { + "entry cap" => assert_eq!( + marker(what_ran.models_truncated), + Some(("entry_cap".to_string(), 1)) + ), + "byte budget" => { + let (reason, dropped) = + marker(what_ran.models_truncated).expect("models marker"); + assert_eq!(reason, "byte_budget"); + assert!(dropped > 1, "{dropped}"); + } + _ => assert_eq!( + marker(what_ran.what_ran_truncated), + Some(("byte_budget".to_string(), 1)) + ), + } + } + let _ = fs::remove_dir_all(root); + } + } + #[test] fn local_host_preserves_detailed_usage_in_metadata() { let (root, plan) = local_host_plan("detailed_usage_result"); diff --git a/crates/fabric-core/src/runtime/openshell.rs b/crates/fabric-core/src/runtime/openshell.rs index b8e7f5e80..9d3bf6fb9 100644 --- a/crates/fabric-core/src/runtime/openshell.rs +++ b/crates/fabric-core/src/runtime/openshell.rs @@ -934,6 +934,53 @@ mod tests { ); } + #[test] + fn task_what_ran_over_the_byte_budget_becomes_minimal_on_this_host() { + let minimal = serde_json::json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": {"reason": "byte_budget", "dropped": 1} + }); + let mut oversized = task_what_ran(); + oversized["adapter"]["id"] = Value::from("a".repeat(140_000)); + for what_ran in [oversized, minimal.clone()] { + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + assert_eq!( + serde_json::to_value(result.what_ran).expect("what_ran JSON"), + minimal + ); + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.output, Value::from("done")); + } + } + + #[test] + fn task_what_ran_past_the_entry_caps_is_bounded_on_this_host() { + let mut what_ran = task_what_ran(); + let server = what_ran["mcp"]["servers"]["docs"].clone(); + what_ran["mcp"]["servers"] = (0..35) + .map(|index| (format!("server-{index:02}"), server.clone())) + .collect::>() + .into(); + what_ran["mcp"]["servers_truncated"] = + serde_json::json!({"reason": "key_too_long", "dropped": 2}); + let mut result = task_result(Some(what_ran)); + + adopt_task_what_ran(&mut result, &environment()); + + let what_ran = serde_json::to_value(result.what_ran).expect("what_ran JSON"); + let servers = what_ran["mcp"]["servers"].as_object().expect("servers"); + let expected: Vec = (0..32).map(|index| format!("server-{index:02}")).collect(); + assert_eq!(servers.keys().cloned().collect::>(), expected); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + serde_json::json!({"reason": "entry_cap", "dropped": 5}) + ); + } + #[test] fn task_result_without_readable_what_ran_is_kept_without_it() { let mut undecodable = task_what_ran(); diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index ef5686ee4..27dbda17f 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -15,6 +15,16 @@ //! then the first 15 hex digits of its SHA-256. Map keys and the plain names //! above are never cut. //! +//! `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in +//! sorted key order. An entry whose key is longer than 128 characters is +//! dropped. While the compact JSON of the whole object is larger than +//! 128 KiB, MCP server entries are dropped from the end of the sorted order, +//! then model roles. Dropped entries are counted in a [`TruncatedEntries`] +//! marker next to the map: `models_truncated` and `mcp.servers_truncated`. +//! If the object is still too large with no MCP server and no model role, +//! it is replaced by a minimal one that holds only the schema name, the +//! version and the `what_ran_truncated` marker. +//! //! `what_ran` is evidence. It never decides which runs are compared. //! //! Compatibility rules for readers: @@ -180,12 +190,66 @@ fn is_code(code: &str) -> bool { }) } +/// Entries core dropped to keep `what_ran` within its bounds: next to a +/// map for the entries dropped from it, or as `what_ran_truncated` for the +/// whole object. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub struct TruncatedEntries { + /// Why the entries were dropped: `key_too_long` for a key longer than + /// 128 characters, `entry_cap` for an entry past the map's cap, or + /// `byte_budget` for an entry dropped to keep the whole object within + /// 128 KiB. One marker covers a map: when more than one rule dropped + /// entries from it, `reason` names the last rule applied, in that + /// order, and `dropped` counts them all. An open code like a fact's + /// `reason`. + #[schemars(extend("pattern" = CODE_PATTERN))] + pub reason: String, + /// How many entries were dropped. A JSON integer: the Rust and Python + /// readers reject `1.0`, although a JSON Schema validator may accept it. + #[schemars(range(min = 1, max = u64::MAX))] + pub dropped: u64, +} + +impl<'de> Deserialize<'de> for TruncatedEntries { + /// Decode a marker, rejecting a `reason` outside the code pattern or a + /// `dropped` count of zero. Unknown properties are ignored. + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + struct Marker { + reason: String, + dropped: u64, + } + let Marker { reason, dropped } = Marker::deserialize(deserializer)?; + if !is_code(&reason) { + return Err(D::Error::custom(format!( + "a truncation `reason` must match {CODE_PATTERN}" + ))); + } + if dropped == 0 { + return Err(D::Error::custom( + "a truncation `dropped` count is at least 1", + )); + } + Ok(Self { reason, dropped }) + } +} + /// What executed for one invocation. /// +/// Every section is present, unless `what_ran_truncated` is present: then +/// the object is a minimal one that the byte budget left with only its +/// schema name and version, and every section may be absent. +/// /// Decoding a `WhatRan` directly checks the shape of every section and -/// fact. `RunResult` also checks the schema name and version, and that -/// every `plan` MCP entry names its transport. +/// fact, and that no section is `null`. `RunResult` also checks the schema +/// name and version, that every section is present unless +/// `what_ran_truncated` is, and that every `plan` MCP entry names its +/// transport. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[schemars(extend("anyOf" = [ + {"required": ["what_ran_truncated"]}, + {"required": ["fabric", "adapter", "harness", "models", "mcp", "environment"]} +]))] pub struct WhatRan { /// Schema name, `nemo.fabric.what_ran`. #[schemars(extend("const" = WHAT_RAN_SCHEMA))] @@ -194,17 +258,83 @@ pub struct WhatRan { #[schemars(extend("const" = WHAT_RAN_VERSION))] pub version: String, /// NeMo Fabric core that ran the adapter. - pub fabric: FabricFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "FabricFacts")] + pub fabric: Option, /// Adapter that ran. - pub adapter: AdapterFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "AdapterFacts")] + pub adapter: Option, /// Harness behind the adapter. - pub harness: HarnessFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "HarnessFacts")] + pub harness: Option, /// Model facts keyed by the role names of the plan's `models` settings. - pub models: BTreeMap, + /// At most 16 roles, in sorted key order, each key at most 128 + /// characters. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "BTreeMap")] + pub models: Option>, + /// Model roles dropped from `models`. Absent when none was dropped, + /// never `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub models_truncated: Option, /// MCP server facts. - pub mcp: McpFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "McpFacts")] + pub mcp: Option, /// Environment the runtime ran in. - pub environment: EnvironmentFacts, + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "EnvironmentFacts")] + pub environment: Option, + /// Present only on a minimal object: every section was dropped to keep + /// the object within 128 KiB, with reason `byte_budget` and a dropped + /// count of 1. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub what_ran_truncated: Option, +} + +/// Decode a property that may be absent but is never `null`. +fn present<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + T::deserialize(deserializer).map(Some) } /// NeMo Fabric core facts. @@ -252,17 +382,28 @@ pub struct ModelFacts { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct McpFacts { /// Servers keyed by name. Empty when the plan configures no MCP server - /// and the harness reports none. + /// and the harness reports none. At most 32 servers, in sorted key + /// order, each key at most 128 characters. pub servers: BTreeMap, + /// Servers dropped from `servers`. Absent when none was dropped, never + /// `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub servers_truncated: Option, } /// Facts about one MCP server. /// /// An entry with origin `harness` is a server the harness reports and the /// plan does not configure. It has the same shape: its `endpoint` reads -/// `not_applicable` with reason `not_configured`, its `transport` may be -/// absent, and runtime facts the adapter did not collect read -/// `unavailable` with reason `not_collected`. +/// `not_applicable` with reason `not_configured`, and runtime facts the +/// adapter did not collect read `unavailable` with reason `not_collected`. +/// Only a `plan` entry must name its transport. An entry of any other +/// origin may leave it out. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[schemars(extend( "if" = {"properties": {"origin": {"const": ORIGIN_PLAN}}}, @@ -272,8 +413,8 @@ pub struct McpServerFacts { /// Where the entry comes from: `plan` for a server the plan configures, /// `harness` for one only the harness reports. pub origin: String, - /// Transport, such as `stdio` or `streamable-http`. Always present for - /// a `plan` entry. + /// Transport, such as `stdio` or `streamable-http`. Required for a + /// `plan` entry, and optional for any other origin. #[serde(default, skip_serializing_if = "Option::is_none")] pub transport: Option, /// Configured endpoint without secrets: the lowercase scheme and host @@ -319,6 +460,18 @@ const REASON_LOCAL_RUN: &str = "local_run"; const REASON_NOT_CONFIGURED: &str = "not_configured"; const REASON_ENDPOINT_UNPARSED: &str = "endpoint_unparsed"; const REASON_EXECUTABLE_UNPARSED: &str = "executable_unparsed"; +const REASON_ENTRY_CAP: &str = "entry_cap"; +const REASON_KEY_TOO_LONG: &str = "key_too_long"; +const REASON_BYTE_BUDGET: &str = "byte_budget"; + +/// Most model roles `what_ran` keeps. +const MAX_MODEL_ROLES: usize = 16; +/// Most MCP servers `what_ran` keeps. +const MAX_MCP_SERVERS: usize = 32; +/// Longest map key `what_ran` keeps, in characters. +const MAX_KEY_CHARS: usize = 128; +/// Largest compact JSON of a whole `what_ran`, in bytes. +const MAX_WHAT_RAN_BYTES: usize = 128 * 1024; /// Longest string value kept whole, in characters. const MAX_VALUE_CHARS: usize = 256; @@ -399,17 +552,23 @@ impl WhatRan { mcp, .. } = self; - let mut origins = vec![ - fabric.core_version.origin_mut(), - fabric.host_core_version.origin_mut(), - adapter.package_version.origin_mut(), - harness.name.origin_mut(), - harness.version.origin_mut(), - ]; - for model in models.values_mut() { + let mut origins = Vec::new(); + if let Some(fabric) = fabric { + origins.extend([ + fabric.core_version.origin_mut(), + fabric.host_core_version.origin_mut(), + ]); + } + if let Some(adapter) = adapter { + origins.push(adapter.package_version.origin_mut()); + } + if let Some(harness) = harness { + origins.extend([harness.name.origin_mut(), harness.version.origin_mut()]); + } + for model in models.iter_mut().flat_map(BTreeMap::values_mut) { origins.extend([model.requested.origin_mut(), model.served.origin_mut()]); } - for server in mcp.servers.values_mut() { + for server in mcp.iter_mut().flat_map(|mcp| mcp.servers.values_mut()) { let runtime = &mut server.runtime; origins.extend([ server.endpoint.origin_mut(), @@ -423,19 +582,95 @@ impl WhatRan { origins } - /// Whether every `plan` MCP entry names its transport. Only a `harness` - /// entry may leave it out. + /// Keep the compact JSON of the whole object within 128 KiB. While it + /// is larger, drop the last MCP server in sorted key order, then the + /// last model role, counting each in its map's marker with reason + /// `byte_budget`. With neither left and the object still too large, + /// replace it with [`WhatRan::minimal`]. + fn apply_byte_budget(&mut self) { + while self.serialized_len() > MAX_WHAT_RAN_BYTES { + if let Some(mcp) = self.mcp.as_mut() + && mcp.servers.pop_last().is_some() + { + mcp.servers_truncated = Some(one_more_byte_budget_drop(&mcp.servers_truncated)); + } else if let Some(models) = self.models.as_mut() + && models.pop_last().is_some() + { + self.models_truncated = Some(one_more_byte_budget_drop(&self.models_truncated)); + } else { + *self = Self::minimal(); + return; + } + } + } + + /// Apply the entry caps and the key rule to an object another core + /// collected, adding what they drop to the existing markers. + fn apply_entry_caps(&mut self) { + if let Some(models) = self.models.take() { + let (kept, dropped) = bounded_entries(&models, MAX_MODEL_ROLES, ModelFacts::clone); + self.models = Some(kept); + self.models_truncated = merged_marker(self.models_truncated.take(), dropped); + } + if let Some(mcp) = self.mcp.as_mut() { + let (kept, dropped) = + bounded_entries(&mcp.servers, MAX_MCP_SERVERS, McpServerFacts::clone); + mcp.servers = kept; + mcp.servers_truncated = merged_marker(mcp.servers_truncated.take(), dropped); + } + } + + /// The object left when every section had to go: the schema name, the + /// version and a `what_ran_truncated` marker. + fn minimal() -> Self { + Self { + schema: WHAT_RAN_SCHEMA.to_string(), + version: WHAT_RAN_VERSION.to_string(), + fabric: None, + adapter: None, + harness: None, + models: None, + models_truncated: None, + mcp: None, + environment: None, + what_ran_truncated: Some(TruncatedEntries { + reason: REASON_BYTE_BUDGET.to_string(), + dropped: 1, + }), + } + } + + fn serialized_len(&self) -> usize { + serde_json::to_vec(self) + .expect("what_ran has only string keys and is always JSON serializable") + .len() + } + + /// Whether every `plan` MCP entry names its transport. An entry of any + /// other origin may leave it out. fn plan_servers_have_transports(&self) -> bool { self.mcp - .servers - .values() + .iter() + .flat_map(|mcp| mcp.servers.values()) .all(|server| server.origin != ORIGIN_PLAN || server.transport.is_some()) } + + /// Whether every section is present, or the object is a minimal one. + fn has_every_section_unless_truncated(&self) -> bool { + self.what_ran_truncated.is_some() + || (self.fabric.is_some() + && self.adapter.is_some() + && self.harness.is_some() + && self.models.is_some() + && self.mcp.is_some() + && self.environment.is_some()) + } } /// Collect the facts the core that runs the adapter knows before it sends /// the request: its own version and the plan's adapter ID, models, MCP -/// servers and environment provider. +/// servers and environment provider. The model and MCP server maps follow +/// the entry caps and the byte budget, and collection never fails. /// /// Core reads no adapter report key yet, so every adapter-reported fact /// reads `unsupported` with reason `key_not_declared`. The collecting core @@ -447,10 +682,17 @@ pub(crate) fn collect_before_request( adapter_id: Option, environment_provider: &str, ) -> WhatRan { - WhatRan { + let (models, models_truncated) = + bounded_entries(&plan.config.models, MAX_MODEL_ROLES, model_facts); + let (servers, servers_truncated) = bounded_entries( + &plan.capability_plan.mcp_servers, + MAX_MCP_SERVERS, + mcp_server_facts, + ); + let mut what_ran = WhatRan { schema: WHAT_RAN_SCHEMA.to_string(), version: WHAT_RAN_VERSION.to_string(), - fabric: FabricFacts { + fabric: Some(FabricFacts { core_version: core_version_fact(), host_core_version: Fact::missing( FactState::NotApplicable, @@ -459,32 +701,90 @@ pub(crate) fn collect_before_request( HOST_CORE, METHOD_CRATE_VERSION, ), - }, - adapter: AdapterFacts { + }), + adapter: Some(AdapterFacts { id: adapter_id, package_version: Fact::report_key_not_declared(), - }, - harness: HarnessFacts { + }), + harness: Some(HarnessFacts { name: Fact::report_key_not_declared(), version: Fact::report_key_not_declared(), - }, - models: plan - .config - .models - .iter() - .map(|(role, model)| (role.clone(), model_facts(model))) - .collect(), - mcp: McpFacts { - servers: plan - .capability_plan - .mcp_servers - .iter() - .map(|(name, server)| (name.clone(), mcp_server_facts(server))) - .collect(), - }, - environment: EnvironmentFacts { + }), + models: Some(models), + models_truncated, + mcp: Some(McpFacts { + servers, + servers_truncated, + }), + environment: Some(EnvironmentFacts { provider: environment_provider.to_string(), - }, + }), + what_ran_truncated: None, + }; + what_ran.apply_byte_budget(); + what_ran +} + +/// The facts of at most `cap` entries, taken in sorted key order, and a +/// marker that counts the dropped entries. +/// +/// An entry whose key is longer than 128 characters is dropped and takes no +/// place under the cap. The marker reason is `entry_cap` when the cap +/// dropped an entry, and `key_too_long` when only long keys were dropped. +/// [`BTreeMap`] iteration gives the sorted key order, whatever order the +/// plan's entries were written in. +fn bounded_entries( + entries: &BTreeMap, + cap: usize, + facts: impl Fn(&P) -> F, +) -> (BTreeMap, Option) { + let mut kept = BTreeMap::new(); + let (mut key_too_long, mut over_cap) = (0_u64, 0_u64); + for (key, entry) in entries { + if key.chars().count() > MAX_KEY_CHARS { + key_too_long += 1; + } else if kept.len() == cap { + over_cap += 1; + } else { + kept.insert(key.clone(), facts(entry)); + } + } + let reason = if over_cap > 0 { + REASON_ENTRY_CAP + } else { + REASON_KEY_TOO_LONG + }; + let dropped = key_too_long + over_cap; + let truncated = (dropped > 0).then(|| TruncatedEntries { + reason: reason.to_string(), + dropped, + }); + (kept, truncated) +} + +/// `marker` with one more dropped entry and reason `byte_budget`. +fn one_more_byte_budget_drop(marker: &Option) -> TruncatedEntries { + TruncatedEntries { + reason: REASON_BYTE_BUDGET.to_string(), + dropped: marker + .as_ref() + .map_or(0, |marker| marker.dropped) + .saturating_add(1), + } +} + +/// An `earlier` marker followed by a `later` one: the later reason, and the +/// two counts added. A task core can send any count, so the sum saturates. +fn merged_marker( + earlier: Option, + later: Option, +) -> Option { + match (earlier, later) { + (Some(earlier), Some(later)) => Some(TruncatedEntries { + reason: later.reason, + dropped: earlier.dropped.saturating_add(later.dropped), + }), + (earlier, later) => later.or(earlier), } } @@ -566,15 +866,24 @@ fn configured_endpoint(transport: McpTransport, url: &str) -> Fact { /// The task core runs in a projected `local` environment and labels its /// own facts `host_core`. The host relabels them `task_core`, records its /// own version as `host_core_version`, and replaces the environment -/// provider with its own, such as `openshell`. +/// provider with its own, such as `openshell`. A minimal object stays +/// minimal. The task core may run another Fabric version, and the host's +/// own facts can grow the object, so the entry caps, the key rule and the +/// byte budget apply again. pub(crate) fn attribute_to_task_core(what_ran: &mut WhatRan, environment_provider: &str) { for origin in what_ran.origins_mut() { if origin.collected_by == HOST_CORE { origin.collected_by = TASK_CORE.to_string(); } } - what_ran.fabric.host_core_version = core_version_fact(); - what_ran.environment.provider = environment_provider.to_string(); + if let Some(fabric) = what_ran.fabric.as_mut() { + fabric.host_core_version = core_version_fact(); + } + if let Some(environment) = what_ran.environment.as_mut() { + environment.provider = environment_provider.to_string(); + } + what_ran.apply_entry_caps(); + what_ran.apply_byte_budget(); } /// `text` whole when it has at most 256 characters. A longer string keeps @@ -609,6 +918,7 @@ where .filter(|what_ran| { what_ran.schema == WHAT_RAN_SCHEMA && what_ran.version == WHAT_RAN_VERSION + && what_ran.has_every_section_unless_truncated() && what_ran.plan_servers_have_transports() })) } @@ -791,6 +1101,238 @@ mod tests { } } + /// `plan_with_secrets` with `models` and MCP servers under the given + /// keys, each a copy of its `default` model or its `files` server. + fn plan_with_entries(model_roles: &[String], server_names: &[String]) -> RunPlan { + let mut plan = plan_with_secrets(); + let model = plan.config.models["default"].clone(); + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.config.models = model_roles + .iter() + .map(|role| (role.clone(), model.clone())) + .collect(); + plan.capability_plan.mcp_servers = server_names + .iter() + .map(|name| (name.clone(), server.clone())) + .collect(); + plan + } + + fn numbered_keys(prefix: &str, count: usize) -> Vec { + (0..count) + .map(|index| format!("{prefix}-{index:02}")) + .collect() + } + + fn map_keys(map: &Value) -> Vec { + map.as_object().expect("map").keys().cloned().collect() + } + + #[test] + fn models_past_16_roles_keep_the_first_in_sorted_order_and_count_the_rest() { + let mut roles = numbered_keys("role", 17); + roles.reverse(); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), numbered_keys("role", 16)); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + assert!(what_ran["mcp"].get("servers_truncated").is_none()); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn mcp_servers_past_32_keep_the_first_in_sorted_order_and_count_the_rest() { + let what_ran = collect(&plan_with_entries(&[], &numbered_keys("server", 35))); + + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + numbered_keys("server", 32) + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 3}) + ); + assert!(what_ran.get("models_truncated").is_none()); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn keys_longer_than_128_characters_are_dropped_without_taking_a_place() { + let longest = "é".repeat(128); + let too_long = "é".repeat(129); + let mut roles = numbered_keys("role", 15); + roles.extend([longest.clone(), too_long.clone()]); + let mut expected_roles = numbered_keys("role", 15); + expected_roles.push(longest.clone()); + + let what_ran = collect(&plan_with_entries(&roles, &[longest, too_long])); + + assert_eq!(map_keys(&what_ran["models"]), expected_roles); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + assert_eq!(map_keys(&what_ran["mcp"]["servers"]), ["é".repeat(128)]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + } + + #[test] + fn a_long_key_and_the_cap_together_count_every_dropped_entry_as_entry_cap() { + let mut roles = numbered_keys("role", 17); + roles.push("r".repeat(129)); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), numbered_keys("role", 16)); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 2}) + ); + } + + #[test] + fn retained_entries_do_not_depend_on_the_order_the_plan_lists_them() { + let roles = numbered_keys("role", 20); + let servers = numbered_keys("server", 40); + let mut reversed_roles = roles.clone(); + reversed_roles.reverse(); + let mut reversed_servers = servers.clone(); + reversed_servers.reverse(); + + assert_eq!( + collect(&plan_with_entries(&roles, &servers)), + collect(&plan_with_entries(&reversed_roles, &reversed_servers)) + ); + } + + /// A plan whose 16 model providers take most of the 128 KiB budget. + fn plan_near_the_byte_budget(server_names: &[String]) -> RunPlan { + let mut plan = plan_with_entries(&numbered_keys("role", 16), server_names); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(7_000); + } + plan + } + + fn json_len(value: &Value) -> usize { + serde_json::to_vec(value).expect("JSON").len() + } + + #[test] + fn mcp_servers_past_the_byte_budget_are_dropped_from_the_end_of_sorted_order() { + let servers = numbered_keys("server", 32); + + let what_ran = collect(&plan_near_the_byte_budget(&servers)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + let kept = map_keys(&what_ran["mcp"]["servers"]); + assert!(!kept.is_empty() && kept.len() < servers.len(), "{kept:?}"); + assert_eq!(kept, servers[..kept.len()]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": servers.len() - kept.len()}) + ); + assert_eq!(map_keys(&what_ran["models"]).len(), 16); + assert!(what_ran_validator().is_valid(&what_ran)); + + // Keeping the next server in sorted order would exceed the budget. + let full = collect(&plan_with_entries(&[], &servers)); + let mut one_more = what_ran.clone(); + let next = &servers[kept.len()]; + one_more["mcp"]["servers"][next] = full["mcp"]["servers"][next].clone(); + assert!(json_len(&one_more) > 128 * 1024); + } + + #[test] + fn the_byte_budget_counts_servers_the_entry_cap_already_dropped() { + let within_cap = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let budget_dropped = within_cap["mcp"]["servers_truncated"]["dropped"] + .as_u64() + .expect("dropped count"); + + let what_ran = collect(&plan_near_the_byte_budget(&numbered_keys("server", 35))); + + assert_eq!(what_ran["mcp"]["servers"], within_cap["mcp"]["servers"]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": budget_dropped + 3}) + ); + } + + /// A plan with the given model roles and MCP servers whose providers + /// together exceed the 128 KiB budget. + fn plan_over_the_byte_budget(roles: &[String], server_names: &[String]) -> RunPlan { + let mut plan = plan_with_entries(roles, server_names); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(9_000); + } + plan + } + + #[test] + fn model_roles_are_dropped_for_the_byte_budget_after_every_mcp_server() { + let roles = numbered_keys("role", 18); + let servers = numbered_keys("server", 3); + + let what_ran = collect(&plan_over_the_byte_budget(&roles, &servers)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + assert_eq!(what_ran["mcp"]["servers"], json!({})); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + let kept = map_keys(&what_ran["models"]); + assert!(!kept.is_empty() && kept.len() < 16, "{kept:?}"); + assert_eq!(kept, roles[..kept.len()]); + // Two roles past the cap, then every role the budget dropped. + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "byte_budget", "dropped": roles.len() - kept.len()}) + ); + assert!(what_ran_validator().is_valid(&what_ran)); + + // Keeping the next role in sorted order would exceed the budget. + let full = collect(&plan_with_entries(&roles, &[])); + let mut one_more = what_ran.clone(); + let next = &roles[kept.len()]; + one_more["models"][next] = full["models"][next].clone(); + one_more["models"][next]["provider"] = json!("p".repeat(9_000)); + assert!(json_len(&one_more) > 128 * 1024); + } + + #[test] + fn an_object_over_the_byte_budget_without_maps_to_drop_becomes_minimal() { + let plan = plan_with_entries(&numbered_keys("role", 2), &numbered_keys("server", 2)); + let adapter_id = "a".repeat(140_000); + + let what_ran = + serde_json::to_value(collect_before_request(&plan, Some(adapter_id), "local")) + .expect("serialize what_ran"); + + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": {"reason": "byte_budget", "dropped": 1} + }) + ); + assert!(what_ran_validator().is_valid(&what_ran)); + let decoded = run_result_with(what_ran) + .what_ran + .expect("a minimal what_ran reads as present"); + assert_eq!(decoded.fabric, None); + } + #[test] fn plan_without_models_or_mcp_servers_reads_empty_maps() { let what_ran = collect(&resolve_plan(config_for("nvidia.fabric.claude"))); @@ -977,7 +1519,12 @@ mod tests { .expect("unknown properties never fail what_ran"); assert_eq!( - what_ran.fabric.core_version.origin().collected_by, + what_ran + .fabric + .expect("fabric section") + .core_version + .origin() + .collected_by, "acme.collector" ); } diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 72a10b9bf..008750fb9 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -490,8 +490,12 @@ "additionalProperties": { "$ref": "#/$defs/McpServerFacts" }, - "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none.", + "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none. At most 32 servers, in sorted key\norder, each key at most 128 characters.", "type": "object" + }, + "servers_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Servers dropped from `servers`. Absent when none was dropped, never\n`null`." } }, "required": [ @@ -533,7 +537,7 @@ "type": "object" }, "McpServerFacts": { - "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, its `transport` may be\nabsent, and runtime facts the adapter did not collect read\n`unavailable` with reason `not_collected`.", + "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, and runtime facts the\nadapter did not collect read `unavailable` with reason `not_collected`.\nOnly a `plan` entry must name its transport. An entry of any other\norigin may leave it out.", "if": { "properties": { "origin": { @@ -555,7 +559,7 @@ "description": "What the harness learned when it connected, as the adapter reports it." }, "transport": { - "description": "Transport, such as `stdio` or `streamable-http`. Always present for\na `plan` entry.", + "description": "Transport, such as `stdio` or `streamable-http`. Required for a\n`plan` entry, and optional for any other origin.", "type": [ "string", "null" @@ -771,8 +775,47 @@ }, "type": "object" }, + "TruncatedEntries": { + "description": "Entries core dropped to keep `what_ran` within its bounds: next to a\nmap for the entries dropped from it, or as `what_ran_truncated` for the\nwhole object.", + "properties": { + "dropped": { + "description": "How many entries were dropped. A JSON integer: the Rust and Python\nreaders reject `1.0`, although a JSON Schema validator may accept it.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 1, + "type": "integer" + }, + "reason": { + "description": "Why the entries were dropped: `key_too_long` for a key longer than\n128 characters, `entry_cap` for an entry past the map's cap, or\n`byte_budget` for an entry dropped to keep the whole object within\n128 KiB. One marker covers a map: when more than one rule dropped\nentries from it, `reason` names the last rule applied, in that\norder, and `dropped` counts them all. An open code like a fact's\n`reason`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + } + }, + "required": [ + "reason", + "dropped" + ], + "type": "object" + }, "WhatRan": { - "description": "What executed for one invocation.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact. `RunResult` also checks the schema name and version, and that\nevery `plan` MCP entry names its transport.", + "anyOf": [ + { + "required": [ + "what_ran_truncated" + ] + }, + { + "required": [ + "fabric", + "adapter", + "harness", + "models", + "mcp", + "environment" + ] + } + ], + "description": "What executed for one invocation.\n\nEvery section is present, unless `what_ran_truncated` is present: then\nthe object is a minimal one that the byte budget left with only its\nschema name and version, and every section may be absent.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact, and that no section is `null`. `RunResult` also checks the schema\nname and version, that every section is present unless\n`what_ran_truncated` is, and that every `plan` MCP entry names its\ntransport.", "properties": { "adapter": { "$ref": "#/$defs/AdapterFacts", @@ -798,9 +841,13 @@ "additionalProperties": { "$ref": "#/$defs/ModelFacts" }, - "description": "Model facts keyed by the role names of the plan's `models` settings.", + "description": "Model facts keyed by the role names of the plan's `models` settings.\nAt most 16 roles, in sorted key order, each key at most 128\ncharacters.", "type": "object" }, + "models_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Model roles dropped from `models`. Absent when none was dropped,\nnever `null`." + }, "schema": { "const": "nemo.fabric.what_ran", "description": "Schema name, `nemo.fabric.what_ran`.", @@ -810,17 +857,15 @@ "const": "v1alpha1", "description": "Contract version, `v1alpha1`.", "type": "string" + }, + "what_ran_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Present only on a minimal object: every section was dropped to keep\nthe object within 128 KiB, with reason `byte_budget` and a dropped\ncount of 1." } }, "required": [ "schema", - "version", - "fabric", - "adapter", - "harness", - "models", - "mcp", - "environment" + "version" ], "type": "object" } diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 0f23621a6..ebaf128e9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -1972,6 +1972,7 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: {"not_applicable", "unsupported", "collection_failed", "unavailable"} ) _FACT_CODE = re.compile(r"[a-z0-9_.:#+/-]{1,128}") +_MAX_U64 = 2**64 - 1 def _is_text(value: Any) -> bool: @@ -2027,7 +2028,8 @@ def _is_mcp_server_entry(server: Any) -> bool: return False origin = server.get("origin") transport = server.get("transport") - # Only a server the harness reports may leave out its transport. + # A `plan` entry requires `transport`. An entry of any other origin, + # including an origin this SDK does not know, may leave it out. transport_ok = isinstance(transport, str) or ( transport is None and origin != "plan" ) @@ -2045,11 +2047,84 @@ def _is_mcp_server_entry(server: Any) -> bool: ) +def _is_truncation_marker(marker: Any) -> bool: + """Whether ``marker`` counts the entries dropped from a map or section.""" + + if not isinstance(marker, Mapping): + return False + dropped = marker.get("dropped") + return ( + _is_code(marker.get("reason")) + and isinstance(dropped, int) + and not isinstance(dropped, bool) + and 1 <= dropped <= _MAX_U64 + ) + + +def _has_valid_marker(section: Mapping[str, Any], name: str) -> bool: + """Whether marker ``name`` is absent from ``section`` or well formed. + + A marker is never ``null``. + """ + + return name not in section or _is_truncation_marker(section[name]) + + +def _is_fabric_section(fabric: Any) -> bool: + return _has_facts(fabric, {"core_version": _is_text, "host_core_version": _is_text}) + + +def _is_adapter_section(adapter: Any) -> bool: + return _has_facts(adapter, {"package_version": _is_text}) and ( + adapter.get("id") is None or isinstance(adapter.get("id"), str) + ) + + +def _is_harness_section(harness: Any) -> bool: + return _has_facts(harness, {"name": _is_text, "version": _is_text}) + + +def _is_models_section(models: Any) -> bool: + return isinstance(models, Mapping) and all( + _is_model_entry(model) for model in models.values() + ) + + +def _is_mcp_section(mcp: Any) -> bool: + if not isinstance(mcp, Mapping): + return False + servers = mcp.get("servers") + return ( + isinstance(servers, Mapping) + and all(_is_mcp_server_entry(server) for server in servers.values()) + and _has_valid_marker(mcp, "servers_truncated") + ) + + +def _is_environment_section(environment: Any) -> bool: + return isinstance(environment, Mapping) and isinstance( + environment.get("provider"), str + ) + + +_WHAT_RAN_SECTIONS: Mapping[str, Callable[[Any], bool]] = { + "fabric": _is_fabric_section, + "adapter": _is_adapter_section, + "harness": _is_harness_section, + "models": _is_models_section, + "mcp": _is_mcp_section, + "environment": _is_environment_section, +} + + def _is_readable_what_ran(value: Any) -> bool: """Whether ``value`` is a ``what_ran`` object this SDK reads. Mirrors the Rust core's reader: the schema and version match, every - section and fact is well formed, and unknown properties are ignored. + present section and fact is well formed, no section is ``null``, and + unknown properties are ignored. Every section is required unless + ``what_ran_truncated`` is present, which marks a minimal object that + the byte budget left with only its schema name and version. """ if not isinstance(value, Mapping): @@ -2059,26 +2134,16 @@ def _is_readable_what_ran(value: Any) -> bool: or value.get("version") != _WHAT_RAN_VERSION ): return False - adapter = value.get("adapter") - models = value.get("models") - mcp = value.get("mcp") - servers = mcp.get("servers") if isinstance(mcp, Mapping) else None - environment = value.get("environment") - return ( - _has_facts( - value.get("fabric"), - {"core_version": _is_text, "host_core_version": _is_text}, - ) - and _has_facts(adapter, {"package_version": _is_text}) - and (adapter.get("id") is None or isinstance(adapter.get("id"), str)) - and _has_facts(value.get("harness"), {"name": _is_text, "version": _is_text}) - and isinstance(models, Mapping) - and all(_is_model_entry(model) for model in models.values()) - and isinstance(servers, Mapping) - and all(_is_mcp_server_entry(server) for server in servers.values()) - and isinstance(environment, Mapping) - and isinstance(environment.get("provider"), str) - ) + minimal = "what_ran_truncated" in value + if minimal and not _is_truncation_marker(value["what_ran_truncated"]): + return False + for name, is_section in _WHAT_RAN_SECTIONS.items(): + if name in value: + if not is_section(value[name]): + return False + elif not minimal: + return False + return _has_valid_marker(value, "models_truncated") def _readable_what_ran(value: Any) -> Mapping[str, Any] | None: @@ -2119,10 +2184,12 @@ class RunResult(FabricMapping): what_ran: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus - ``source``, ``collected_by`` and ``method``. ``None`` when the + ``source``, ``collected_by`` and ``method``. A minimal object + that the byte budget left holds only ``schema``, ``version`` and + ``what_ran_truncated``, and reads as present. ``None`` when the producing core predates it, or when the value is not a - ``what_ran`` object of that schema and version with every section - and fact well formed, as the Rust core reads it. + ``what_ran`` object of that schema and version with every present + section and fact well formed, as the Rust core reads it. """ agent_name: str diff --git a/tests/fixtures/what_ran/reader-vectors.json b/tests/fixtures/what_ran/reader-vectors.json index eb21c95e6..8866d76fe 100644 --- a/tests/fixtures/what_ran/reader-vectors.json +++ b/tests/fixtures/what_ran/reader-vectors.json @@ -393,6 +393,60 @@ "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } ] + }, + { + "name": "truncation markers next to models and MCP servers", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 1 + } + }, + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "byte_budget", + "dropped": 3 + } + } + ] + }, + { + "name": "vendor truncation reason", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "acme.quota", + "dropped": 2 + } + } + ] + }, + { + "name": "minimal section left by the byte budget", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 1 + } + } + } + ] } ], "invalid": [ @@ -804,6 +858,166 @@ ] } ] + }, + { + "name": "truncation marker without a dropped count", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap" + } + } + ] + }, + { + "name": "truncation marker with a zero dropped count", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "byte_budget", + "dropped": 0 + } + } + ] + }, + { + "name": "truncation marker with a string dropped count", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": "1" + } + } + ] + }, + { + "name": "truncation reason outside the code pattern", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": { + "reason": "Entry Cap", + "dropped": 1 + } + } + ] + }, + { + "name": "truncation marker that is not an object", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": "entry_cap" + } + ] + }, + { + "name": "schema and version only, without the minimal marker", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1" + } + } + ] + }, + { + "name": "minimal section with a null marker", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": null + } + } + ] + }, + { + "name": "minimal section with a zero dropped count", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 0 + } + } + } + ] + }, + { + "name": "minimal section with a malformed section", + "patch": [ + { + "path": [], + "value": { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": { + "reason": "byte_budget", + "dropped": 1 + }, + "harness": "codex" + } + } + ] + }, + { + "name": "null section", + "patch": [ + { + "path": [ + "harness" + ], + "value": null + } + ] + }, + { + "name": "null models_truncated", + "patch": [ + { + "path": [ + "models_truncated" + ], + "value": null + } + ] + }, + { + "name": "null servers_truncated", + "patch": [ + { + "path": [ + "mcp", + "servers_truncated" + ], + "value": null + } + ] } ] } From 6dea00aad281a751bacd554064a427873dda5f2b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 16:22:17 -0700 Subject: [PATCH 063/159] docs: describe what_ran bounds in the Python SDK guide Explain the entry caps, the key length rule, the byte budget, the truncation markers and the minimal object, and tell readers to check what_ran_truncated before reading a section. Keep the integration skill's RunResult notes in step. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 22 ++++++++++++++++--- .../references/results-and-errors.md | 4 +++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index fa23c2a6c..cfb12c1ef 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1032,8 +1032,9 @@ state `unsupported` with reason `key_not_declared` and method Today every MCP entry has `origin` `plan`. An entry with `origin` `harness` describes a server the harness reports but the plan does not configure. It has -the same shape, except that `transport` can be absent and `endpoint` has state -`not_applicable` with reason `not_configured`. +the same shape, except that `endpoint` has state `not_applicable` with reason +`not_configured`. A `plan` entry always has `transport`. An entry of any other +origin can leave it out. `endpoint` keeps no secret. A network server's endpoint is its lowercase scheme and host and its port. A URL without a port gets 80 for `http` and `ws` and 443 @@ -1054,6 +1055,21 @@ A string value longer than 256 characters keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. The rule applies to each element of a list value too. +`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, taken in +sorted key order. An entry whose key is longer than 128 characters is dropped. +While the whole `what_ran` is larger than 128 KiB of compact JSON, MCP servers +are dropped from the end of the sorted order, and then model roles. A map that +lost entries has a marker next to it, `models_truncated` or +`mcp.servers_truncated`, with the `reason` (`key_too_long`, `entry_cap`, or +`byte_budget`) and the `dropped` count. Each map has one marker: when more than +one rule dropped entries, `reason` names the last rule applied, in that order, +and `dropped` counts every dropped entry. `dropped` is a JSON integer. The Rust +core and the Python SDK reject `1.0`, although a JSON Schema validator may +accept it. If `what_ran` is still too large with no MCP server and no model +role left, it holds only `schema`, `version`, and `what_ran_truncated` with +reason `byte_budget` and `dropped` 1. Check for `what_ran_truncated` before you +read a section. Dropping entries never changes the run's status or error. + `what_ran` is `None` when the core that produced the result predates it. The Rust core and the Python SDK apply the same structural checks. A `what_ran` that is not an object, names another schema or version, or has a malformed @@ -1063,7 +1079,7 @@ ignored. Treat `what_ran` as evidence, not as a key for grouping runs. ```python what_ran = result.what_ran -if what_ran is not None: +if what_ran is not None and "what_ran_truncated" not in what_ran: core = what_ran["fabric"]["core_version"] print(core.get("value"), core["source"]) for role, model in what_ran["models"].items(): diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index b9b51edfc..3793b558a 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -53,7 +53,9 @@ fact holds a `value`, or a `state` and `reason` when it has no value, plus `reason` codes you do not recognize. Adapter-reported facts, such as the harness version and the served model, currently read state `unsupported` with reason `key_not_declared`. A malformed `what_ran` reads as `None` without -failing the result. Refer to +failing the result. A `what_ran` over its 128 KiB budget can hold only +`schema`, `version`, and `what_ran_truncated`, so check for that marker before +reading a section. Refer to [What Ran](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/sdk/python.mdx#what-ran). ## Correlation IDs From e3a287652b04b8fd23fe75712bc753294bc11a91 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 16:22:18 -0700 Subject: [PATCH 064/159] docs: regenerate API references for what_ran bounds Regenerate the Rust library reference pages of the what_ran module for TruncatedEntries and the optional sections, and update the Python types reference for RunResult.what_ran. Only the pages this change affects are included. Signed-off-by: mnajafian-nv --- .../nemo_fabric.types.md | 2 +- .../what-ran/constant-what-ran-schema.mdx | 2 +- .../what-ran/constant-what-ran-version.mdx | 2 +- .../nemo-fabric-core/what-ran/enum-fact.mdx | 2 +- .../what-ran/enum-factsource.mdx | 2 +- .../what-ran/enum-factstate.mdx | 2 +- .../nemo-fabric-core/what-ran/index.mdx | 3 + .../what-ran/struct-mcpfacts.mdx | 8 +- .../what-ran/struct-mcpserverfacts.mdx | 4 +- .../what-ran/struct-truncatedentries.mdx | 105 ++++++++++++++++++ .../what-ran/struct-whatran.mdx | 30 +++-- 11 files changed, 142 insertions(+), 20 deletions(-) create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 7664714b5..159aa7afd 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -1443,7 +1443,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `telemetry`: Ordered telemetry references. - `events`: Ordered lifecycle and invocation events. - `metadata`: Result-specific structured details. - - `what_ran`: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus ``source``, ``collected_by`` and ``method``. ``None`` when the producing core predates it, or when the value is not a ``what_ran`` object of that schema and version with every section and fact well formed, as the Rust core reads it. + - `what_ran`: What executed for this invocation, schema ``nemo.fabric.what_ran`` version ``v1alpha1``, as a JSON mapping. Each fact holds ``value``, or ``state`` and ``reason``, plus ``source``, ``collected_by`` and ``method``. A minimal object that the byte budget left holds only ``schema``, ``version`` and ``what_ran_truncated``, and reads as present. ``None`` when the producing core predates it, or when the value is not a ``what_ran`` object of that schema and version with every present section and fact well formed, as the Rust core reads it. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx index 06068c300..3b95258d3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx @@ -3,7 +3,7 @@ title: "Constant WHAT_RAN_SCHEMA" sidebar-title: "WHAT_RAN_SCHEMA" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-schema" description: "Schema name written into every `WhatRan`." -position: 16 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx index 01dc53369..f517163e1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx @@ -3,7 +3,7 @@ title: "Constant WHAT_RAN_VERSION" sidebar-title: "WHAT_RAN_VERSION" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-version" description: "Contract version written into every `WhatRan`." -position: 17 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx index 55c94c912..027e467d6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-fact.mdx @@ -3,7 +3,7 @@ title: "Enum Fact" sidebar-title: "Fact" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-fact" description: "One fact: a `value`, or a `state` with a `reason`, never both." -position: 13 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx index 66a4e30c5..fde659009 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factsource.mdx @@ -3,7 +3,7 @@ title: "Enum Fact Source" sidebar-title: "FactSource" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factsource" description: "How NeMo Fabric learned a fact." -position: 14 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx index 9fd1a13cf..b91461c16 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/enum-factstate.mdx @@ -3,7 +3,7 @@ title: "Enum Fact State" sidebar-title: "FactState" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/enum-factstate" description: "Why a fact has no value. A collector picks the first state that matches, in declaration order." -position: 15 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx index 18ba5a872..b99298410 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -16,6 +16,8 @@ What executed for one invocation, and how NeMo Fabric learned each fact. A string value longer than 256 characters, in a string fact or as an element of a string-list fact, keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. Map keys and the plain names above are never cut. +`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in sorted key order. An entry whose key is longer than 128 characters is dropped. While the compact JSON of the whole object is larger than 128 KiB, MCP server entries are dropped from the end of the sorted order, then model roles. Dropped entries are counted in a [`TruncatedEntries`](struct-truncatedentries.mdx) marker next to the map: `models_truncated` and `mcp.servers_truncated`. If the object is still too large with no MCP server and no model role, it is replaced by a minimal one that holds only the schema name, the version and the `what_ran_truncated` marker. + `what_ran` is evidence. It never decides which runs are compared. Compatibility rules for readers: @@ -38,6 +40,7 @@ Compatibility rules for readers: - [McpServerFacts](struct-mcpserverfacts.mdx): Facts about one MCP server. - [MissingFact](struct-missingfact.mdx): A fact without a value. - [ModelFacts](struct-modelfacts.mdx): Facts about one model role. +- [TruncatedEntries](struct-truncatedentries.mdx): Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object. - [WhatRan](struct-whatran.mdx): What executed for one invocation. ## Enums diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx index c5ddca739..a5628947d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpfacts.mdx @@ -10,7 +10,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<String, McpServerFacts>,\n}"}} />
+
BTreeMap<String, McpServerFacts>,\n    pub servers_truncated: Option<TruncatedEntries>,\n}"}} />
MCP facts. @@ -18,7 +18,11 @@ MCP facts. ### `servers: BTreeMap` -Servers keyed by name. Empty when the plan configures no MCP server and the harness reports none. +Servers keyed by name. Empty when the plan configures no MCP server and the harness reports none. At most 32 servers, in sorted key order, each key at most 128 characters. + +### `servers_truncated: Option` + +Servers dropped from `servers`. Absent when none was dropped, never `null`. ## Trait Implementations diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx index 19730218c..ad6c67371 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpserverfacts.mdx @@ -14,7 +14,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Facts about one MCP server. -An entry with origin `harness` is a server the harness reports and the plan does not configure. It has the same shape: its `endpoint` reads `not_applicable` with reason `not_configured`, its `transport` may be absent, and runtime facts the adapter did not collect read `unavailable` with reason `not_collected`. +An entry with origin `harness` is a server the harness reports and the plan does not configure. It has the same shape: its `endpoint` reads `not_applicable` with reason `not_configured`, and runtime facts the adapter did not collect read `unavailable` with reason `not_collected`. Only a `plan` entry must name its transport. An entry of any other origin may leave it out. ## Fields @@ -24,7 +24,7 @@ Where the entry comes from: `plan` for a server the plan configures, `harness` f ### `transport: Option` -Transport, such as `stdio` or `streamable-http`. Always present for a `plan` entry. +Transport, such as `stdio` or `streamable-http`. Required for a `plan` entry, and optional for any other origin. ### `endpoint: Fact` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx new file mode 100644 index 000000000..062b10bd0 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-truncatedentries.mdx @@ -0,0 +1,105 @@ +--- +title: "Struct Truncated Entries" +sidebar-title: "TruncatedEntries" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-truncatedentries" +description: "Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object." +position: 12 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub dropped: u64,\n}"}} />
+ +Entries core dropped to keep `what_ran` within its bounds: next to a map for the entries dropped from it, or as `what_ran_truncated` for the whole object. + +## Fields + +### `reason: String` + +Why the entries were dropped: `key_too_long` for a key longer than 128 characters, `entry_cap` for an entry past the map's cap, or `byte_budget` for an entry dropped to keep the whole object within 128 KiB. One marker covers a map: when more than one rule dropped entries from it, `reason` names the last rule applied, in that order, and `dropped` counts them all. An open code like a fact's `reason`. + +### `dropped: u64` + +How many entries were dropped. A JSON integer: the Rust and Python readers reject `1.0`, although a JSON Schema validator may accept it. + +## Trait Implementations + +### `impl Clone for TruncatedEntries` + +
Clone for TruncatedEntries"}} />
+ +#### `clone` + +
clone(&self) -> TruncatedEntries"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TruncatedEntries` + +
Debug for TruncatedEntries"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TruncatedEntries` + +
Deserialize<'de> for TruncatedEntries"}} />
+ +#### `deserialize` + +
deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error>"}} />
+ +Decode a marker, rejecting a `reason` outside the code pattern or a `dropped` count of zero. Unknown properties are ignored. + +### `impl JsonSchema for TruncatedEntries` + +
TruncatedEntries"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TruncatedEntries` + +
PartialEq for TruncatedEntries"}} />
+ +#### `eq` + +
eq(&self, other: &TruncatedEntries) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TruncatedEntries` + +
Serialize for TruncatedEntries"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for TruncatedEntries` + +
Eq for TruncatedEntries"}} />
+ +### `impl StructuralPartialEq for TruncatedEntries` + +
StructuralPartialEq for TruncatedEntries"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx index d25aaa76f..2873628a8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx @@ -3,18 +3,20 @@ title: "Struct WhatRan" sidebar-title: "WhatRan" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/struct-whatran" description: "What executed for one invocation." -position: 12 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub version: String,\n    pub fabric: FabricFacts,\n    pub adapter: AdapterFacts,\n    pub harness: HarnessFacts,\n    pub models: BTreeMap<String, ModelFacts>,\n    pub mcp: McpFacts,\n    pub environment: EnvironmentFacts,\n}"}} />
+
String,\n    pub version: String,\n    pub fabric: Option<FabricFacts>,\n    pub adapter: Option<AdapterFacts>,\n    pub harness: Option<HarnessFacts>,\n    pub models: Option<BTreeMap<String, ModelFacts>>,\n    pub models_truncated: Option<TruncatedEntries>,\n    pub mcp: Option<McpFacts>,\n    pub environment: Option<EnvironmentFacts>,\n    pub what_ran_truncated: Option<TruncatedEntries>,\n}"}} />
What executed for one invocation. -Decoding a `WhatRan` directly checks the shape of every section and fact. `RunResult` also checks the schema name and version, and that every `plan` MCP entry names its transport. +Every section is present, unless `what_ran_truncated` is present: then the object is a minimal one that the byte budget left with only its schema name and version, and every section may be absent. + +Decoding a `WhatRan` directly checks the shape of every section and fact, and that no section is `null`. `RunResult` also checks the schema name and version, that every section is present unless `what_ran_truncated` is, and that every `plan` MCP entry names its transport. ## Fields @@ -26,30 +28,38 @@ Schema name, `nemo.fabric.what_ran`. Contract version, `v1alpha1`. -### `fabric: FabricFacts` +### `fabric: Option` NeMo Fabric core that ran the adapter. -### `adapter: AdapterFacts` +### `adapter: Option` Adapter that ran. -### `harness: HarnessFacts` +### `harness: Option` Harness behind the adapter. -### `models: BTreeMap` +### `models: Option>` + +Model facts keyed by the role names of the plan's `models` settings. At most 16 roles, in sorted key order, each key at most 128 characters. + +### `models_truncated: Option` -Model facts keyed by the role names of the plan's `models` settings. +Model roles dropped from `models`. Absent when none was dropped, never `null`. -### `mcp: McpFacts` +### `mcp: Option` MCP server facts. -### `environment: EnvironmentFacts` +### `environment: Option` Environment the runtime ran in. +### `what_ran_truncated: Option` + +Present only on a minimal object: every section was dropped to keep the object within 128 KiB, with reason `byte_budget` and a dropped count of 1. + ## Trait Implementations ### `impl Clone for WhatRan` From cdc0636a647d562a02a72110f9921b8943d3bd04 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 17:00:16 -0700 Subject: [PATCH 065/159] docs: name NeMo Fabric in the Deep Agents usage limits Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 78e94ce51..66a44eece 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -243,7 +243,7 @@ unknown, because the provider may have done work without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the adapter, and only the final response's reported usage counts. Usage replayed from a configured LangChain model cache -would be counted as reported; Fabric configures no such cache. +would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming From 566003925848ac3b00393af70e26bdf49155b613 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 17:22:03 -0700 Subject: [PATCH 066/159] docs: clarify when run records get sample lines Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 069f9b30d..faf9a193d 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1088,11 +1088,11 @@ result lists it as the `record` artifact with kind `run_record`: record = next(a for a in result.artifacts.artifacts if a.kind == "run_record") ``` -The file holds a start line, a sample line about every 10 seconds until the -adapter host answers or the timeout fires when the start line has -`sample_interval_ms`, and an end line. Every line carries `schema` -(`nemo.fabric.run_record`), `version` (`v1alpha1`), `kind`, `runtime_id`, -`invocation_id`, `request_id`, and `time`. +The file holds a start line, sample lines, and an end line. Sample lines are +written only when the start line has `sample_interval_ms`; then one is written +about every 10 seconds until the adapter host answers or the timeout fires. +Every line carries `schema` (`nemo.fabric.run_record`), `version` (`v1alpha1`), +`kind`, `runtime_id`, `invocation_id`, `request_id`, and `time`. - The `start` line is written before the request is sent. It holds the agent name, harness, adapter ID, each configured model's provider and model name, From f076c101323b4c424bfe8af0dbbc07241a11bdde Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 22:51:03 -0700 Subject: [PATCH 067/159] test: cover what_ran on a realistic config Run a local host invocation with two model roles and three MCP servers (npx with arguments, streamable-http and sse) and no environment section, and assert the exact what_ran a user gets, including the origin of every unsupported adapter-reported fact and that no configured secret appears. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 97 +++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 1b5493545..d05ddfc1e 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -5501,6 +5501,103 @@ for line in sys.stdin: } } + /// The `what_ran` a user gets from a normal benchmark config: two model + /// roles and three MCP servers on an adapter that accepts both, and no + /// `environment` section, so the provider defaults to `local`. + #[test] + fn local_host_what_ran_reports_a_realistic_config() { + use serde_json::json; + let (root, plan) = local_host_plan("success"); + let descriptor_path = root.join("adapters/local-host/local-host.fabric-adapter.json"); + let mut descriptor: Value = + serde_json::from_str(&fs::read_to_string(&descriptor_path).expect("read descriptor")) + .expect("descriptor JSON"); + descriptor["config"] = json!({"accepts": ["models", "mcp"]}); + fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); + let mut config = serde_json::to_value(&plan.config).expect("config JSON"); + config + .as_object_mut() + .expect("config") + .remove("environment"); + config["models"] = json!({ + "default": {"provider": "anthropic", "model": "anthropic/claude-sonnet-4-5"}, + "summarizer": {"provider": "nvidia", "model": "nvidia/nemotron-nano-9b-v2"}, + }); + config["mcp"] = json!({"servers": { + "filesystem": { + "transport": "stdio", + "url": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/SECRET_ARGUMENT"], + "env": {"FS_TOKEN": "SECRET_ENVIRONMENT"}, + "exposure": "harness_native" + }, + "search": { + "transport": "streamable-http", + "url": "https://mcp.example.com/mcp?api_key=SECRET_QUERY", + "custom_headers": {"Authorization": "Bearer SECRET_HEADER"}, + "exposure": "harness_native" + }, + "events": { + "transport": "sse", + "url": "http://localhost:8931/sse", + "exposure": "harness_native" + } + }}); + let config: crate::config::FabricConfig = + serde_json::from_value(config).expect("typed config"); + let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)) + .expect("resolve realistic plan"); + + let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let configured = |value: &str| { + json!({"value": value, "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + }; + let not_declared = json!({"state": "unsupported", "reason": "key_not_declared", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + let model = |provider: &str, requested: &str| { + json!({"provider": provider, "requested": configured(requested), + "served": not_declared}) + }; + let server = |transport: &str, endpoint: &str| { + json!({"origin": "plan", "transport": transport, "endpoint": configured(endpoint), + "runtime": {"connected": not_declared, "protocol_version": not_declared, + "server_name": not_declared, "server_version": not_declared, + "tool_names": not_declared}}) + }; + assert_eq!( + what_ran, + json!({ + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "fabric": { + "core_version": {"value": crate::version(), "source": "observed", + "collected_by": "host_core", "method": "crate_version"}, + "host_core_version": {"state": "not_applicable", "reason": "local_run", + "source": "observed", "collected_by": "host_core", + "method": "crate_version"} + }, + "adapter": {"id": "acme.fabric.local-host", "package_version": not_declared}, + "harness": {"name": not_declared, "version": not_declared}, + "models": { + "default": model("anthropic", "anthropic/claude-sonnet-4-5"), + "summarizer": model("nvidia", "nvidia/nemotron-nano-9b-v2") + }, + "mcp": {"servers": { + "events": server("sse", "http://localhost:8931"), + "filesystem": server("stdio", "npx"), + "search": server("streamable-http", "https://mcp.example.com:443") + }}, + "environment": {"provider": "local"} + }) + ); + assert!(!what_ran.to_string().contains("SECRET"), "{what_ran}"); + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_what_ran_bounds_keep_the_status_and_error() { let model = |provider: &str| -> crate::config::ModelConfig { From d64832dfaa38e3cc80030736d8134a77beb6e2fe Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 19 Oct 2026 22:51:09 -0700 Subject: [PATCH 068/159] test: cover what_ran for each bundled descriptor and at the entry caps Resolve a plan from each of the 16 bundled descriptors with a model and, where the adapter accepts MCP, three servers, and check that what_ran passes the schema, carries no truncation marker, and reads every adapter-reported fact as unsupported/key_not_declared. Also show that 16 roles and 32 servers with 128-character keys fit the byte budget, and that stdio placeholders read executable_unparsed. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 150 +++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 27dbda17f..9773a9756 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1341,6 +1341,143 @@ mod tests { assert_eq!(what_ran["mcp"], json!({"servers": {}})); } + /// Every adapter-reported fact in `what_ran`, by its JSON path. + fn reported_facts(value: &Value, path: &str) -> Vec<(String, Value)> { + match value { + Value::Object(map) if map.get("source") == Some(&json!("reported")) => { + vec![(path.to_string(), value.clone())] + } + Value::Object(map) => map + .iter() + .flat_map(|(key, value)| reported_facts(value, &format!("{path}.{key}"))) + .collect(), + _ => Vec::new(), + } + } + + #[test] + fn a_plan_from_each_bundled_descriptor_collects_plan_facts_and_undeclared_adapter_facts() { + let catalog_path = repository_root().join( + "sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json", + ); + let catalog: Value = + serde_json::from_str(&std::fs::read_to_string(catalog_path).expect("read catalog")) + .expect("catalog JSON"); + let adapters = catalog["adapters"].as_object().expect("catalog adapters"); + assert_eq!(adapters.len(), 16); + let validator = what_ran_validator(); + for (adapter_id, entry) in adapters { + let descriptor = &entry["descriptor"]; + let mut config = config_for(adapter_id); + for setting in descriptor["settings_schema"]["required"] + .as_array() + .into_iter() + .flatten() + { + let setting = setting.as_str().expect("setting name"); + config["harness"]["settings"][setting] = json!("https://agent.example.com"); + } + let accepts = |field: &str| { + descriptor["config"]["accepts"] + .as_array() + .is_some_and(|accepts| accepts.contains(&json!(field))) + }; + // A model each adapter's model schema accepts: its fixed provider + // if it has one, and every connection field the schema names. + let model_fields = &descriptor["model_schema"]["properties"]; + let provider = model_fields["provider"]["const"] + .as_str() + .unwrap_or("nvidia"); + config["models"] = + json!({"default": {"provider": provider, "model": "nvidia/nemotron"}}); + for (field, value) in [ + ("base_url", "https://inference.example.com/v1"), + ("api_key_env", "MODEL_API_KEY"), + ] { + if model_fields.get(field).is_some() { + config["models"]["default"][field] = json!(value); + } + } + let accepts_mcp = accepts("mcp"); + if accepts_mcp { + config["mcp"] = json!({"servers": { + "events": {"transport": "sse", "url": "http://localhost:8931/sse", + "exposure": "harness_native"}, + "filesystem": {"transport": "stdio", "url": "npx", + "args": ["-y", "@scope/server-filesystem"], + "exposure": "harness_native"}, + "search": {"transport": "streamable-http", "url": "https://mcp.example.com/mcp", + "exposure": "harness_native"} + }}); + } + + // Core discovers the repository's adapters as its bundled set. + let config: FabricConfig = serde_json::from_value(config).expect("typed config"); + let plan = crate::config::resolve_run_plan_from_config( + config, + ResolveContext::new(repository_root()), + ) + .unwrap_or_else(|error| panic!("{adapter_id}: {error}")); + let what_ran = collect(&plan); + + assert!(validator.is_valid(&what_ran), "{adapter_id}: {what_ran}"); + assert_eq!(what_ran["adapter"]["id"], json!(adapter_id)); + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!( + what_ran["models"]["default"]["requested"], + configured("nvidia/nemotron") + ); + let servers = if accepts_mcp { 3 } else { 0 }; + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]).len(), + servers, + "{adapter_id}" + ); + // package version, harness name and version, served, 5 per server. + let reported = reported_facts(&what_ran, "what_ran"); + assert_eq!(reported.len(), 4 + 5 * servers, "{adapter_id}"); + for (path, fact) in reported { + assert_eq!(fact, not_declared(), "{adapter_id} {path}"); + } + for marker in [ + &what_ran["models_truncated"], + &what_ran["mcp"]["servers_truncated"], + &what_ran["what_ran_truncated"], + ] { + assert!(marker.is_null(), "{adapter_id}: {what_ran}"); + } + assert!(json_len(&what_ran) <= 128 * 1024, "{adapter_id}"); + } + } + + #[test] + fn both_entry_caps_with_the_longest_keys_and_values_fit_the_byte_budget() { + let longest_key = |prefix: &str, index: usize| { + let key = format!("{prefix}-{index:02}-"); + format!("{key}{}", "k".repeat(128 - key.len())) + }; + let roles: Vec = (0..16).map(|index| longest_key("role", index)).collect(); + let servers: Vec = (0..32).map(|index| longest_key("server", index)).collect(); + let mut plan = plan_with_entries(&roles, &servers); + for model in plan.config.models.values_mut() { + model.model = "m".repeat(256); + } + + let what_ran = collect(&plan); + + assert_eq!(map_keys(&what_ran["models"]), roles); + assert_eq!(map_keys(&what_ran["mcp"]["servers"]), servers); + assert!(what_ran.get("models_truncated").is_none(), "{what_ran}"); + assert!( + what_ran["mcp"].get("servers_truncated").is_none(), + "{what_ran}" + ); + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + } + #[test] fn stdio_executable_with_whitespace_reads_unparsed_and_keeps_no_secret() { let mut config = config_for("nvidia.fabric.claude"); @@ -1386,6 +1523,19 @@ mod tests { "collection_failed", "executable_unparsed", ), + // Placeholders a harness expands later, not a file name. + ( + McpTransport::Stdio, + "$MCP_COMMAND", + "collection_failed", + "executable_unparsed", + ), + ( + McpTransport::Stdio, + "/opt/mcp/server-{env:MCP_VARIANT}", + "collection_failed", + "executable_unparsed", + ), (McpTransport::Sse, " ", "unavailable", "not_configured"), (McpTransport::Stdio, "", "unavailable", "not_configured"), ] { From 8eba8b5c8b755544847a0ae30ec17b88a8736ca9 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:33:43 -0700 Subject: [PATCH 069/159] test: loosen the catalog count and rename the what_ran size test Loop over whatever the catalog lists without pinning its size, so adding or removing a bundled adapter doesn't break the test. Name the size test after what it builds: ASCII keys and model names at their maximum length. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 9773a9756..d21df1d3c 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1364,7 +1364,7 @@ mod tests { serde_json::from_str(&std::fs::read_to_string(catalog_path).expect("read catalog")) .expect("catalog JSON"); let adapters = catalog["adapters"].as_object().expect("catalog adapters"); - assert_eq!(adapters.len(), 16); + assert!(!adapters.is_empty()); let validator = what_ran_validator(); for (adapter_id, entry) in adapters { let descriptor = &entry["descriptor"]; @@ -1454,7 +1454,7 @@ mod tests { } #[test] - fn both_entry_caps_with_the_longest_keys_and_values_fit_the_byte_budget() { + fn both_entry_caps_with_max_length_ascii_keys_and_models_fit_the_byte_budget() { let longest_key = |prefix: &str, index: usize| { let key = format!("{prefix}-{index:02}-"); format!("{key}{}", "k".repeat(128 - key.len())) From 07ecc566cd3f8a4e3fc05057e21c2a31b84c7c6d Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:30:54 -0700 Subject: [PATCH 070/159] feat: read adapter report keys into what_ran Adapters had no way to report their own facts, so the adapter package version, the harness name and version, the served model and the MCP runtime facts always read unsupported with reason key_not_declared. Core now reads nemo_fabric.versions.v1 and nemo_fabric.mcp.v1 from a result's extensions after the descriptor's extension check, when the descriptor's run_result schema names the key under properties. Only adapter-owned facts with source reported and collector adapter are kept; a malformed or forged fact reads invalid_report, an omitted one fact_omitted, an omitted key key_omitted, and a key over 64 KiB too_large. A reported server the plan does not configure becomes a harness entry. tool_names keeps 64 sorted names and counts the rest in a new tool_names_truncated marker, added to the schema and both readers. Ingestion never changes the run's status or error. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 78 +- crates/fabric-core/src/what_ran.rs | 756 +++++++++++++++++- schemas/sdk/run-result.schema.json | 6 +- .../src/nemo_fabric/types.py | 4 +- tests/fixtures/what_ran/reader-vectors.json | 51 ++ 5 files changed, 873 insertions(+), 22 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d05ddfc1e..17c4fee03 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1923,7 +1923,7 @@ fn run_local_host_invocation_with_timeout( request_id: request.request_id.clone(), runtime_id: runtime.runtime_id.clone(), }; - let what_ran = crate::what_ran::collect_before_request( + let mut what_ran = crate::what_ran::collect_before_request( plan, adapter_id(plan), &runtime.environment.provider, @@ -2034,15 +2034,16 @@ fn run_local_host_invocation_with_timeout( fabric_invocation, ) = exchange_result?; - let agent_result: AgentRunResult = serde_json::from_value(adapter_output).map_err(|error| { - lifecycle_error( - operation, - &runtime.runtime_id, - "invalid_agent_run_result", - format!("adapter returned an invalid AgentRunResult: {error}"), - &stderr, - ) - })?; + let mut agent_result: AgentRunResult = + serde_json::from_value(adapter_output).map_err(|error| { + lifecycle_error( + operation, + &runtime.runtime_id, + "invalid_agent_run_result", + format!("adapter returned an invalid AgentRunResult: {error}"), + &stderr, + ) + })?; agent_result.validate().map_err(|error| { lifecycle_error( operation, @@ -2053,6 +2054,7 @@ fn run_local_host_invocation_with_timeout( ) })?; validate_agent_run_result_extensions(&agent_result, plan.adapter_descriptor.as_ref())?; + crate::what_ran::ingest_adapter_reports(&mut what_ran, plan, &mut agent_result.extensions); if !agent_result.artifacts.is_empty() && artifacts.root.is_none() { return Err(lifecycle_error( operation, @@ -4156,6 +4158,25 @@ for line in sys.stdin: "extensions": {"source": "fake-host"}, }, } + elif MODE == "versions_report": + fact = lambda value, method: { + "value": value, "source": "reported", "collected_by": "adapter", + "method": method, + } + result = { + "status": "succeeded", + "output": output, + "extensions": { + "trace_id": "trace-1", + "nemo_fabric.versions.v1": { + "adapter": {"package_version": fact("1.2.3", "package_metadata")}, + "harness": { + "name": fact("fake-harness", "harness_api"), + "version": fact("4.5.6", "harness_api"), + }, + }, + }, + } elif MODE == "typed_result": result = { "status": "succeeded", @@ -5598,6 +5619,43 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + /// An adapter that declares the versions key and reports it gets its + /// facts into `what_ran`, and the key leaves `metadata.adapter`. + #[test] + fn local_host_what_ran_ingests_a_declared_versions_report() { + use serde_json::json; + let (root, plan) = local_host_plan("versions_report"); + let descriptor_path = root.join("adapters/local-host/local-host.fabric-adapter.json"); + let mut descriptor: Value = + serde_json::from_str(&fs::read_to_string(&descriptor_path).expect("read descriptor")) + .expect("descriptor JSON"); + descriptor["extension_schemas"]["run_result"]["properties"]["nemo_fabric.versions.v1"] = + json!({"type": "object"}); + fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); + let plan = resolve_run_plan_from_config(plan.config, ResolveContext::new(&root)) + .expect("resolve plan"); + + let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.metadata["adapter"], json!({"trace_id": "trace-1"})); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let reported = |value: &str, method: &str| { + json!({"value": value, "source": "reported", "collected_by": "adapter", + "method": method}) + }; + assert_eq!( + what_ran["adapter"]["package_version"], + reported("1.2.3", "package_metadata") + ); + assert_eq!( + what_ran["harness"], + json!({"name": reported("fake-harness", "harness_api"), + "version": reported("4.5.6", "harness_api")}) + ); + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_what_ran_bounds_keep_the_status_and_error() { let model = |provider: &str| -> crate::config::ModelConfig { diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index d21df1d3c..f9be07a25 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -25,6 +25,12 @@ //! it is replaced by a minimal one that holds only the schema name, the //! version and the `what_ran_truncated` marker. //! +//! An adapter reports its own facts through the report keys +//! [`VERSIONS_REPORT_KEY`] and [`MCP_REPORT_KEY`] in its result's +//! extensions, once its descriptor declares them. A reported list keeps at +//! most 64 elements in sorted order, counted in a marker beside it, such as +//! `tool_names_truncated`. +//! //! `what_ran` is evidence. It never decides which runs are compared. //! //! Compatibility rules for readers: @@ -50,6 +56,7 @@ use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; use sha2::{Digest, Sha256}; +use crate::adapter_contract::AdapterExtensionPoint; use crate::config::{McpServerPlan, McpTransport, ModelConfig, RunPlan}; use crate::endpoint::{endpoint_identity, executable_name}; @@ -437,8 +444,17 @@ pub struct McpRuntimeFacts { /// Version the server reports for itself. pub server_version: Fact, /// Raw tool names the server lists, each bounded by the string length - /// rule. + /// rule. At most 64 names, in sorted order. pub tool_names: Fact>, + /// Tool names dropped from `tool_names`, by the adapter's helper or by + /// core. Absent when none was dropped, never `null`. + #[serde( + default, + deserialize_with = "present", + skip_serializing_if = "Option::is_none" + )] + #[schemars(with = "TruncatedEntries")] + pub tool_names_truncated: Option, } /// Environment facts. @@ -530,13 +546,7 @@ impl Fact { /// An adapter-reported fact while core reads no adapter report key. fn report_key_not_declared() -> Self { - Self::missing( - FactState::Unsupported, - REASON_KEY_NOT_DECLARED, - FactSource::Reported, - ADAPTER, - METHOD_ADAPTER_REPORT, - ) + reported_missing(FactState::Unsupported, REASON_KEY_NOT_DECLARED) } } @@ -672,8 +682,9 @@ impl WhatRan { /// servers and environment provider. The model and MCP server maps follow /// the entry caps and the byte budget, and collection never fails. /// -/// Core reads no adapter report key yet, so every adapter-reported fact -/// reads `unsupported` with reason `key_not_declared`. The collecting core +/// Every adapter-reported fact reads `unsupported` with reason +/// `key_not_declared` until [`ingest_adapter_reports`] reads the report keys +/// the result carries. The collecting core /// runs the adapter in a local environment, so `host_core_version` reads /// `not_applicable`. A host that relays the result from a task core calls /// [`attribute_to_task_core`]. @@ -821,6 +832,7 @@ fn mcp_server_facts(server: &McpServerPlan) -> McpServerFacts { server_name: Fact::report_key_not_declared(), server_version: Fact::report_key_not_declared(), tool_names: Fact::report_key_not_declared(), + tool_names_truncated: None, }, } } @@ -886,6 +898,280 @@ pub(crate) fn attribute_to_task_core(what_ran: &mut WhatRan, environment_provide what_ran.apply_byte_budget(); } +/// Report key for the adapter package version, the harness name and version, +/// and the served model of each role. +pub const VERSIONS_REPORT_KEY: &str = "nemo_fabric.versions.v1"; +/// Report key for the `runtime` block of each MCP server, and for servers +/// the harness reports that the plan does not configure. +pub const MCP_REPORT_KEY: &str = "nemo_fabric.mcp.v1"; + +/// Largest compact JSON of one report key, in bytes. +const MAX_REPORT_BYTES: usize = 64 * 1024; +/// Most elements a list fact keeps. +const MAX_LIST_ENTRIES: usize = 64; +const ORIGIN_HARNESS: &str = "harness"; +const REASON_KEY_OMITTED: &str = "key_omitted"; +const REASON_FACT_OMITTED: &str = "fact_omitted"; +const REASON_INVALID_REPORT: &str = "invalid_report"; +const REASON_TOO_LARGE: &str = "too_large"; + +/// Move the adapter's declared report keys from a result's extensions into +/// `what_ran`. +/// +/// Core calls this after the descriptor's extension schema accepted +/// `extensions`, so ingestion never rejects a result. A key is declared +/// only when the descriptor's `run_result` extension schema names it +/// literally under `properties`; the facts of an undeclared key keep +/// `unsupported` with reason `key_not_declared`, and the key stays in the +/// extensions. A declared key that is absent reads `unavailable` with +/// reason `key_omitted`, and one that is not an object reads +/// `collection_failed` with reason `invalid_report` and stays in the +/// extensions. A declared object is removed from the extensions. Past +/// 64 KiB of compact JSON, every fact of that key reads `collection_failed` +/// with reason `too_large`. +/// +/// Inside a report, a key may hold only adapter-owned facts, each with +/// source `reported` and collector `adapter`: the adapter package version, +/// the harness name and version, each plan role's served model, each MCP +/// server's `runtime` block with its `tool_names_truncated` marker, and a +/// `harness` server's transport. A malformed or forged fact reads +/// `collection_failed` with reason `invalid_report`, and an omitted one +/// reads `unavailable` with reason `fact_omitted`. Core ignores every other +/// property, so a report never changes a fact core owns. The string length +/// rule, the 64-entry cap on `tool_names`, the entry caps and the byte +/// budget then apply to the result. +pub(crate) fn ingest_adapter_reports( + what_ran: &mut WhatRan, + plan: &RunPlan, + extensions: &mut BTreeMap, +) { + let run_result_schema = plan.adapter_descriptor.as_ref().and_then(|adapter| { + adapter + .descriptor + .extension_schemas + .get(&AdapterExtensionPoint::RunResult) + }); + for key in [VERSIONS_REPORT_KEY, MCP_REPORT_KEY] { + if !declares_report_key(run_result_schema, key) { + continue; + } + let removed = match extensions.get(key) { + Some(Value::Object(_)) => extensions.remove(key), + _ => None, + }; + let report = match &removed { + Some(Value::Object(object)) if compact_json_len(object) > MAX_REPORT_BYTES => { + Report::missing(FactState::CollectionFailed, REASON_TOO_LARGE) + } + Some(Value::Object(object)) => Report::Object(object), + _ if extensions.contains_key(key) => { + Report::missing(FactState::CollectionFailed, REASON_INVALID_REPORT) + } + _ => Report::missing(FactState::Unavailable, REASON_KEY_OMITTED), + }; + if key == VERSIONS_REPORT_KEY { + apply_versions_report(what_ran, &report); + } else { + apply_mcp_report(what_ran, plan, &report); + } + } + what_ran.apply_entry_caps(); + what_ran.apply_byte_budget(); +} + +fn compact_json_len(object: &serde_json::Map) -> usize { + serde_json::to_vec(object) + .expect("a JSON object is always serializable") + .len() +} + +/// Whether a `run_result` extension schema names `key` literally under +/// `properties`. Core never resolves `$ref`, `allOf` or patterns. +fn declares_report_key(schema: Option<&serde_json::Map>, key: &str) -> bool { + schema + .and_then(|schema| schema.get("properties")) + .and_then(Value::as_object) + .is_some_and(|properties| properties.contains_key(key)) +} + +/// One object of an adapter report, or the state every fact under it reads +/// when the object is missing or malformed. +enum Report<'a> { + Object(&'a serde_json::Map), + Missing(MissingFact), +} + +impl<'a> Report<'a> { + fn missing(state: FactState, reason: &str) -> Self { + match reported_missing::<()>(state, reason) { + Fact::Missing(missing) => Self::Missing(missing), + Fact::Known(_) => unreachable!("reported_missing builds a missing fact"), + } + } + + /// The object under `key`: `fact_omitted` when absent, `invalid_report` + /// when not an object. + fn child(&self, key: &str) -> Report<'a> { + match self { + Self::Object(object) => match object.get(key) { + Some(Value::Object(child)) => Report::Object(child), + Some(_) => Report::missing(FactState::CollectionFailed, REASON_INVALID_REPORT), + None => Report::missing(FactState::Unavailable, REASON_FACT_OMITTED), + }, + Self::Missing(missing) => Report::Missing(missing.clone()), + } + } + + /// The reported fact under `key`, kept only when it decodes and names + /// source `reported` and collector `adapter`. + fn fact(&self, key: &str) -> Fact { + let value = match self { + Self::Missing(missing) => return Fact::Missing(missing.clone()), + Self::Object(object) => object.get(key), + }; + match value.map(Fact::::deserialize) { + None => reported_missing(FactState::Unavailable, REASON_FACT_OMITTED), + Some(Ok(fact)) + if fact.origin().source == FactSource::Reported + && fact.origin().collected_by == ADAPTER => + { + fact + } + Some(_) => reported_missing(FactState::CollectionFailed, REASON_INVALID_REPORT), + } + } +} + +/// A reported fact without a value, collected by the adapter's report. +fn reported_missing(state: FactState, reason: &str) -> Fact { + Fact::missing( + state, + reason, + FactSource::Reported, + ADAPTER, + METHOD_ADAPTER_REPORT, + ) +} + +fn apply_versions_report(what_ran: &mut WhatRan, report: &Report<'_>) { + if let Some(adapter) = what_ran.adapter.as_mut() { + adapter.package_version = bounded_string(report.child("adapter").fact("package_version")); + } + if let Some(harness) = what_ran.harness.as_mut() { + let reported = report.child("harness"); + harness.name = bounded_string(reported.fact("name")); + harness.version = bounded_string(reported.fact("version")); + } + let models = report.child("models"); + for (role, model) in what_ran.models.iter_mut().flatten() { + model.served = bounded_string(models.child(role).fact("served")); + } +} + +/// Fill each plan server's `runtime` block, and add a `harness` entry for +/// each reported server the plan does not configure. +fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) { + let Some(mcp) = what_ran.mcp.as_mut() else { + return; + }; + let servers = report.child("servers"); + for (name, server) in &mut mcp.servers { + server.runtime = runtime_facts(&servers.child(name).child("runtime")); + } + let Report::Object(reported) = servers else { + return; + }; + let plan_servers = &plan.capability_plan.mcp_servers; + for (name, entry) in reported { + if plan_servers.contains_key(name) { + continue; + } + let entry_report = servers.child(name); + mcp.servers.insert( + name.clone(), + McpServerFacts { + origin: ORIGIN_HARNESS.to_string(), + transport: entry + .get("transport") + .and_then(Value::as_str) + .map(str::to_lowercase), + endpoint: Fact::missing( + FactState::NotApplicable, + REASON_NOT_CONFIGURED, + FactSource::Configured, + HOST_CORE, + METHOD_RESOLVED_PLAN, + ), + runtime: runtime_facts(&entry_report.child("runtime")), + }, + ); + } +} + +/// One server's `runtime` block from its report. `tool_names` keeps at most +/// 64 names in sorted order, and its marker adds core's drops to the +/// helper's. +fn runtime_facts(report: &Report<'_>) -> McpRuntimeFacts { + let (mut tool_names, core_marker) = bounded_list(report.fact("tool_names")); + let mut helper_marker = None; + if let (Fact::Known(_), Report::Object(object)) = (&tool_names, report) + && let Some(marker) = object.get("tool_names_truncated") + { + match TruncatedEntries::deserialize(marker) { + Ok(marker) => helper_marker = Some(marker), + Err(_) => { + tool_names = reported_missing(FactState::CollectionFailed, REASON_INVALID_REPORT); + } + } + } + let tool_names_truncated = match tool_names { + Fact::Known(_) => merged_marker(helper_marker, core_marker), + Fact::Missing(_) => None, + }; + McpRuntimeFacts { + connected: report.fact("connected"), + protocol_version: bounded_string(report.fact("protocol_version")), + server_name: bounded_string(report.fact("server_name")), + server_version: bounded_string(report.fact("server_version")), + tool_names, + tool_names_truncated, + } +} + +/// `fact` with the string length rule applied to its value. +fn bounded_string(fact: Fact) -> Fact { + match fact { + Fact::Known(known) => Fact::Known(KnownFact { + value: bounded_text(&known.value), + origin: known.origin, + }), + missing => missing, + } +} + +/// `fact` with each element bounded by the string length rule, sorted by +/// code point, and cut to 64 elements, with a marker that counts the cut. +fn bounded_list(fact: Fact>) -> (Fact>, Option) { + let Fact::Known(known) = fact else { + return (fact, None); + }; + let mut value: Vec = known.value.iter().map(|item| bounded_text(item)).collect(); + value.sort(); + let dropped = value.len().saturating_sub(MAX_LIST_ENTRIES); + value.truncate(MAX_LIST_ENTRIES); + let marker = (dropped > 0).then(|| TruncatedEntries { + reason: REASON_ENTRY_CAP.to_string(), + dropped: dropped as u64, + }); + ( + Fact::Known(KnownFact { + value, + origin: known.origin, + }), + marker, + ) +} + /// `text` whole when it has at most 256 characters. A longer string keeps /// its first 240 characters, then `~`, then the first 15 hex digits of the /// whole string's SHA-256, so two long values collide only on a 60-bit @@ -1678,4 +1964,454 @@ mod tests { "acme.collector" ); } + + /// The secrets plan with a descriptor that declares `keys` in its + /// `run_result` extension schema. + fn plan_declaring(keys: &[&str]) -> RunPlan { + let mut plan = plan_with_secrets(); + let properties: serde_json::Map = keys + .iter() + .map(|key| (key.to_string(), json!({"type": "object"}))) + .collect(); + let schema = json!({"type": "object", "properties": properties}); + plan.adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .extension_schemas + .insert( + AdapterExtensionPoint::RunResult, + schema.as_object().expect("schema object").clone(), + ); + plan + } + + fn reported(value: Value, method: &str) -> Value { + json!({"value": value, "source": "reported", "collected_by": "adapter", + "method": method}) + } + + fn report_state(state: &str, reason: &str) -> Value { + json!({"state": state, "reason": reason, "source": "reported", + "collected_by": "adapter", "method": "adapter_report"}) + } + + /// Ingest `extensions` into the facts collected for `plan`, and return + /// `what_ran` with the extensions core leaves on the result. + fn ingest(plan: &RunPlan, extensions: Value) -> (Value, Value) { + let adapter_id = plan + .adapter_descriptor + .as_ref() + .map(|adapter| adapter.descriptor.adapter_id.clone()); + let mut what_ran = collect_before_request(plan, adapter_id, "local"); + let mut extensions: BTreeMap = + serde_json::from_value(extensions).expect("extensions map"); + ingest_adapter_reports(&mut what_ran, plan, &mut extensions); + let what_ran = serde_json::to_value(what_ran).expect("serialize what_ran"); + assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); + ( + what_ran, + serde_json::to_value(extensions).expect("extensions JSON"), + ) + } + + fn versions_report() -> Value { + json!({ + "adapter": {"package_version": reported(json!("0.6.0"), "package_metadata")}, + "harness": { + "name": reported(json!("claude-code"), "harness_init_message"), + "version": reported(json!("2.1.294"), "harness_init_message") + }, + "models": {"default": {"served": reported(json!("claude-sonnet-4-5-20250929"), + "harness_api")}} + }) + } + + #[test] + fn a_declared_versions_report_fills_the_adapter_facts_and_leaves_the_extensions() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + + let (what_ran, extensions) = ingest( + &plan, + json!({VERSIONS_REPORT_KEY: versions_report(), "trace_id": "trace-1"}), + ); + + assert_eq!( + what_ran["adapter"]["package_version"], + reported(json!("0.6.0"), "package_metadata") + ); + assert_eq!( + what_ran["harness"], + json!({"name": reported(json!("claude-code"), "harness_init_message"), + "version": reported(json!("2.1.294"), "harness_init_message")}) + ); + assert_eq!( + what_ran["models"]["default"]["served"], + reported(json!("claude-sonnet-4-5-20250929"), "harness_api") + ); + assert_eq!(extensions, json!({"trace_id": "trace-1"})); + // The MCP key is not declared, so its facts keep `key_not_declared`. + assert_eq!( + what_ran["mcp"]["servers"]["github"]["runtime"]["connected"], + not_declared() + ); + } + + #[test] + fn an_undeclared_report_key_stays_in_the_extensions_and_reads_key_not_declared() { + let plan = plan_with_secrets(); + let extensions = json!({VERSIONS_REPORT_KEY: versions_report()}); + + let (what_ran, kept) = ingest(&plan, extensions.clone()); + + assert_eq!(kept, extensions); + for fact in [ + &what_ran["adapter"]["package_version"], + &what_ran["harness"]["name"], + &what_ran["harness"]["version"], + &what_ran["models"]["default"]["served"], + ] { + assert_eq!(fact, ¬_declared()); + } + } + + #[test] + fn a_report_key_declared_only_through_a_reference_is_not_declared() { + let mut plan = plan_with_secrets(); + let schema = json!({"type": "object", "allOf": [{"properties": { + VERSIONS_REPORT_KEY: {"type": "object"}}}]}); + plan.adapter_descriptor + .as_mut() + .expect("adapter descriptor") + .descriptor + .extension_schemas + .insert( + AdapterExtensionPoint::RunResult, + schema.as_object().expect("schema").clone(), + ); + + let (what_ran, kept) = ingest(&plan, json!({VERSIONS_REPORT_KEY: versions_report()})); + + assert_eq!(what_ran["harness"]["version"], not_declared()); + assert_eq!(kept, json!({VERSIONS_REPORT_KEY: versions_report()})); + } + + #[test] + fn an_omitted_non_object_or_oversized_report_key_marks_every_fact_of_that_key() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY, MCP_REPORT_KEY]); + let oversized = json!({"harness": {"version": reported( + json!("v".repeat(MAX_REPORT_BYTES)), "harness_api")}}); + for (extensions, state, reason, kept) in [ + (json!({}), "unavailable", "key_omitted", json!({})), + ( + json!({VERSIONS_REPORT_KEY: "0.6.0", MCP_REPORT_KEY: []}), + "collection_failed", + "invalid_report", + json!({VERSIONS_REPORT_KEY: "0.6.0", MCP_REPORT_KEY: []}), + ), + ( + json!({VERSIONS_REPORT_KEY: oversized, MCP_REPORT_KEY: {"servers": { + "github": {"runtime": {"server_name": reported( + json!("s".repeat(MAX_REPORT_BYTES)), "harness_status")}}}}}), + "collection_failed", + "too_large", + json!({}), + ), + ] { + let (what_ran, extensions) = ingest(&plan, extensions); + + let expected = report_state(state, reason); + assert_eq!(extensions, kept, "{reason}"); + for fact in [ + &what_ran["adapter"]["package_version"], + &what_ran["harness"]["name"], + &what_ran["harness"]["version"], + &what_ran["models"]["default"]["served"], + ] { + assert_eq!(fact, &expected, "{reason}"); + } + for server in what_ran["mcp"]["servers"] + .as_object() + .expect("servers") + .values() + { + for fact in server["runtime"].as_object().expect("runtime").values() { + assert_eq!(fact, &expected, "{reason}"); + } + } + } + } + + #[test] + fn a_report_at_64_kib_is_read_and_one_byte_more_is_too_large() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let with_padding = |padding: usize| { + let mut report = versions_report(); + report["padding"] = json!(""); + let base = serde_json::to_vec(&report).expect("JSON").len(); + report["padding"] = json!("p".repeat(padding - base)); + report + }; + + let (at_limit, _) = ingest( + &plan, + json!({VERSIONS_REPORT_KEY: with_padding(MAX_REPORT_BYTES)}), + ); + let (past_limit, _) = ingest( + &plan, + json!({VERSIONS_REPORT_KEY: with_padding(MAX_REPORT_BYTES + 1)}), + ); + + assert_eq!( + at_limit["harness"]["version"], + reported(json!("2.1.294"), "harness_init_message") + ); + assert_eq!( + past_limit["harness"]["version"], + report_state("collection_failed", "too_large") + ); + } + + #[test] + fn the_origin_rule_rejects_forged_or_malformed_facts_and_keeps_the_rest_of_the_report() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let forged = json!({"value": "9.9.9", "source": "observed", "collected_by": "adapter", + "method": "harness_api"}); + let other_collector = json!({"value": "9.9.9", "source": "reported", + "collected_by": "task_core", "method": "harness_api"}); + let report = json!({ + "fabric": {"core_version": reported(json!("0.0.0-forged"), "crate_version")}, + "adapter": {"id": "forged.adapter", + "package_version": {"value": "0.6.0", "state": "unavailable", + "reason": "both", "source": "reported", "collected_by": "adapter", + "method": "package_metadata"}}, + "harness": {"name": forged, "version": other_collector}, + "models": { + "default": {"served": reported(json!(42), "harness_api"), + "requested": reported(json!("forged"), "harness_api"), + "provider": "forged"}, + "unplanned": {"served": reported(json!("other"), "harness_api")} + } + }); + + let (what_ran, _) = ingest(&plan, json!({VERSIONS_REPORT_KEY: report})); + + let invalid = report_state("collection_failed", "invalid_report"); + assert_eq!(what_ran["adapter"]["package_version"], invalid); + assert_eq!(what_ran["harness"]["name"], invalid); + assert_eq!(what_ran["harness"]["version"], invalid); + assert_eq!(what_ran["models"]["default"]["served"], invalid); + // Facts core owns keep core's values, and roles stay the plan's. + assert_eq!( + what_ran["fabric"]["core_version"]["value"], + crate::version() + ); + assert_eq!(what_ran["adapter"]["id"], "nvidia.fabric.claude"); + assert_eq!( + what_ran["models"]["default"]["requested"], + configured("claude-sonnet-4-5") + ); + assert_eq!(what_ran["models"]["default"]["provider"], "anthropic"); + assert_eq!(map_keys(&what_ran["models"]), ["default"]); + } + + #[test] + fn omitted_facts_read_fact_omitted_and_a_reported_state_is_kept() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let no_source = json!({"state": "unsupported", "reason": "no_source", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + let report = json!({ + "harness": {"name": no_source, "version": no_source}, + "models": "not an object" + }); + + let (what_ran, _) = ingest(&plan, json!({VERSIONS_REPORT_KEY: report})); + + assert_eq!( + what_ran["adapter"]["package_version"], + report_state("unavailable", "fact_omitted") + ); + assert_eq!(what_ran["harness"]["name"], no_source); + assert_eq!( + what_ran["models"]["default"]["served"], + report_state("collection_failed", "invalid_report") + ); + } + + #[test] + fn reported_strings_follow_the_length_rule() { + let plan = plan_declaring(&[VERSIONS_REPORT_KEY]); + let long = format!("{}a", "m".repeat(300)); + let mut report = versions_report(); + report["harness"]["version"] = reported(json!(long), "harness_api"); + + let (what_ran, _) = ingest(&plan, json!({VERSIONS_REPORT_KEY: report})); + + assert_eq!( + what_ran["harness"]["version"]["value"], + format!("{}~96858eedeb3c8e6", "m".repeat(240)) + ); + } + + fn tool_names(count: usize) -> Vec { + // Reverse order, so core's sort decides which names stay. + (0..count) + .rev() + .map(|index| format!("tool-{index:02}")) + .collect() + } + + #[test] + fn mcp_runtime_facts_fill_plan_servers_and_cap_tool_names_at_64() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let runtime = |names: Vec, marker: Option| { + let mut runtime = json!({ + "connected": reported(json!(true), "harness_status"), + "protocol_version": reported(json!("2025-06-18"), "mcp_initialize"), + "server_name": reported(json!("github-mcp"), "harness_status"), + "server_version": reported(json!("1.2.3"), "harness_status"), + "tool_names": reported(json!(names), "harness_status") + }); + if let Some(marker) = marker { + runtime["tool_names_truncated"] = marker; + } + runtime + }; + let report = json!({"servers": { + "github": {"transport": "stdio", "endpoint": configured("forged"), + "runtime": runtime(tool_names(65), None)}, + "events": {"runtime": runtime(tool_names(70), + Some(json!({"reason": "byte_budget", "dropped": 3})))}, + "files": {"runtime": runtime(tool_names(64), None)} + }}); + + let (what_ran, extensions) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!(extensions, json!({})); + let servers = &what_ran["mcp"]["servers"]; + let github = &servers["github"]; + let kept: Vec = tool_names(64).into_iter().rev().collect(); + assert_eq!(github["runtime"]["tool_names"]["value"], json!(kept)); + assert_eq!( + github["runtime"]["tool_names_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + assert_eq!( + servers["events"]["runtime"]["tool_names_truncated"], + json!({"reason": "entry_cap", "dropped": 9}) + ); + assert!(servers["files"]["runtime"]["tool_names_truncated"].is_null()); + // A plan entry keeps its own origin, transport and endpoint. + assert_eq!(github["origin"], "plan"); + assert_eq!(github["transport"], "streamable-http"); + assert_eq!( + github["endpoint"], + configured("https://api.githubcopilot.com:443") + ); + assert_eq!( + github["runtime"]["connected"], + reported(json!(true), "harness_status") + ); + } + + #[test] + fn mcp_report_adds_harness_entries_and_marks_omitted_plan_servers() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let not_collected = json!({"state": "unavailable", "reason": "not_collected", + "source": "reported", "collected_by": "adapter", + "method": "harness_startup_status"}); + let long_name = "h".repeat(129); + let report = json!({"servers": { + "plugin-docs": {"transport": "STDIO", "runtime": { + "connected": reported(json!(true), "harness_startup_status"), + "protocol_version": not_collected, + "server_name": not_collected, + "server_version": not_collected, + "tool_names": not_collected, + "tool_names_truncated": {"reason": "entry_cap", "dropped": 2} + }}, + "broken": "not an object", + long_name: {"runtime": {}} + }}); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + let servers = &what_ran["mcp"]["servers"]; + assert_eq!( + map_keys(servers), + ["broken", "events", "files", "github", "plugin-docs"] + ); + let harness_entry = &servers["plugin-docs"]; + assert_eq!(harness_entry["origin"], "harness"); + assert_eq!(harness_entry["transport"], "stdio"); + assert_eq!( + harness_entry["endpoint"], + json!({"state": "not_applicable", "reason": "not_configured", + "source": "configured", "collected_by": "host_core", + "method": "resolved_plan"}) + ); + assert_eq!(harness_entry["runtime"]["server_name"], not_collected); + // A marker beside a list without a value is dropped. + assert!(harness_entry["runtime"]["tool_names_truncated"].is_null()); + let invalid = report_state("collection_failed", "invalid_report"); + assert_eq!(servers["broken"]["runtime"]["connected"], invalid); + assert!(servers["broken"].get("transport").is_none()); + assert_eq!( + servers["github"]["runtime"]["connected"], + report_state("unavailable", "fact_omitted") + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "key_too_long", "dropped": 1}) + ); + } + + #[test] + fn a_malformed_helper_marker_invalidates_tool_names() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let report = json!({"servers": {"github": {"runtime": { + "tool_names": reported(json!(["a"]), "harness_status"), + "tool_names_truncated": {"reason": "entry_cap", "dropped": 0} + }}}}); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + let runtime = &what_ran["mcp"]["servers"]["github"]["runtime"]; + assert_eq!( + runtime["tool_names"], + report_state("collection_failed", "invalid_report") + ); + assert!(runtime["tool_names_truncated"].is_null()); + } + + #[test] + fn a_plan_server_dropped_by_the_entry_cap_never_returns_as_a_harness_entry() { + let names = numbered_keys("server", 33); + let mut plan = plan_declaring(&[MCP_REPORT_KEY]); + let template = plan.capability_plan.mcp_servers["events"].clone(); + plan.capability_plan.mcp_servers = names + .iter() + .map(|name| (name.clone(), template.clone())) + .collect(); + let servers: serde_json::Map = names + .iter() + .map(|name| { + ( + name.clone(), + json!({"runtime": { + "connected": reported(json!(true), "harness_status")}}), + ) + }) + .collect(); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: {"servers": servers}})); + + let servers = what_ran["mcp"]["servers"].as_object().expect("servers"); + assert_eq!(servers.len(), 32); + assert!(servers.values().all(|server| server["origin"] == "plan")); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + } } diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 008750fb9..867799101 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -524,7 +524,11 @@ }, "tool_names": { "$ref": "#/$defs/Fact_for_Array_of_string", - "description": "Raw tool names the server lists, each bounded by the string length\nrule." + "description": "Raw tool names the server lists, each bounded by the string length\nrule. At most 64 names, in sorted order." + }, + "tool_names_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Tool names dropped from `tool_names`, by the adapter's helper or by\ncore. Absent when none was dropped, never `null`." } }, "required": [ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index ebaf128e9..3e40fc608 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2040,10 +2040,12 @@ def _is_mcp_server_entry(server: Any) -> bool: "server_version": _is_text, "tool_names": _is_text_list, } + runtime = server.get("runtime") return ( isinstance(origin, str) and transport_ok - and _has_facts(server.get("runtime"), runtime_facts) + and _has_facts(runtime, runtime_facts) + and _has_valid_marker(runtime, "tool_names_truncated") ) diff --git a/tests/fixtures/what_ran/reader-vectors.json b/tests/fixtures/what_ran/reader-vectors.json index 8866d76fe..e3d931c17 100644 --- a/tests/fixtures/what_ran/reader-vectors.json +++ b/tests/fixtures/what_ran/reader-vectors.json @@ -447,6 +447,24 @@ } } ] + }, + { + "name": "tool names marker inside an MCP runtime block", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 1 + } + } + ] } ], "invalid": [ @@ -1018,6 +1036,39 @@ "value": null } ] + }, + { + "name": "tool names marker with a zero dropped count", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names_truncated" + ], + "value": { + "reason": "entry_cap", + "dropped": 0 + } + } + ] + }, + { + "name": "null tool_names_truncated", + "patch": [ + { + "path": [ + "mcp", + "servers", + "docs", + "runtime", + "tool_names_truncated" + ], + "value": null + } + ] } ] } From 4a7425b971779c921977c9aaf1a2a0e167182329 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:52:14 -0700 Subject: [PATCH 071/159] feat: report adapter and harness versions from six adapters A what_ran could not say which harness build ran. Six adapters now declare nemo_fabric.versions.v1 and report their package version and the harness name and version from a source that names the build that ran: - Claude Code: claude_code_version from the CLI's init message. - Codex: the thread's cli_version from one bounded thread/read request. - Hermes: get_version_info(), a blocking call that runs git, on a daemon thread with a 5 second limit. - OpenClaw: the --version check, or an attached gateway's /startupz version, both already read at startup. - Deep Agents and mini-swe-agent: package metadata, accepted only when the distribution installed the module the adapter loaded. A shared helper builds the report, applies core's string length rule, and runs blocking calls with at most one outstanding probe per runtime; stopping a runtime never waits for it. A source that times out, fails or answers nothing reads a state with a reason, never a substitute value. The adapter catalog is regenerated for the six changed descriptors. Signed-off-by: mnajafian-nv --- .../python/claude/claude.fabric-adapter.json | 8 + .../nemo_fabric_adapters/claude/adapter.py | 89 ++++++-- .../python/codex/codex.fabric-adapter.json | 8 + .../src/nemo_fabric_adapters/codex/adapter.py | 63 ++++- .../nemo_fabric_adapters/common/provenance.py | 215 ++++++++++++++++++ .../deepagents/deepagents.fabric-adapter.json | 10 + .../deepagents/adapter.py | 27 +++ .../python/hermes/hermes.fabric-adapter.json | 8 + .../nemo_fabric_adapters/hermes/adapter.py | 68 ++++++ .../mini-swe-agent.fabric-adapter.json | 10 + .../mini_swe_agent/adapter.py | 26 +++ .../openclaw/openclaw.fabric-adapter.json | 10 + .../nemo_fabric_adapters/openclaw/adapter.py | 34 ++- .../claude/claude.fabric-adapter.json | 8 + .../adapters/codex/codex.fabric-adapter.json | 8 + .../deepagents/deepagents.fabric-adapter.json | 10 + .../hermes/hermes.fabric-adapter.json | 8 + crates/fabric-core/src/config.rs | 4 + crates/fabric-core/src/what_ran.rs | 4 +- .../nemo_fabric_adapter_catalog/catalog.json | 78 ++++++- .../test_adapters_common_provenance.py | 168 ++++++++++++++ tests/adapters/test_claude_adapter.py | 104 +++++++++ tests/adapters/test_codex_adapter.py | 62 +++++ tests/adapters/test_hermes_adapter.py | 155 +++++++++++++ tests/adapters/test_openclaw.py | 21 ++ tests/e2e/test_deepagents.py | 20 ++ tests/e2e/test_mini_swe_agent.py | 27 +++ tests/fixtures/claude_streams/api-error.jsonl | 3 + tests/fixtures/claude_streams/normal.jsonl | 3 + 29 files changed, 1231 insertions(+), 28 deletions(-) create mode 100644 adapters/python/common/src/nemo_fabric_adapters/common/provenance.py create mode 100644 tests/adapters/test_adapters_common_provenance.py create mode 100644 tests/fixtures/claude_streams/api-error.jsonl create mode 100644 tests/fixtures/claude_streams/normal.jsonl diff --git a/adapters/python/claude/claude.fabric-adapter.json b/adapters/python/claude/claude.fabric-adapter.json index 5af1a312f..035cb01a1 100644 --- a/adapters/python/claude/claude.fabric-adapter.json +++ b/adapters/python/claude/claude.fabric-adapter.json @@ -99,6 +99,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py index 458a58cde..ec9238886 100644 --- a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py +++ b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py @@ -30,6 +30,7 @@ from claude_agent_sdk import Message from claude_agent_sdk import ProcessError from claude_agent_sdk import ResultMessage +from claude_agent_sdk import SystemMessage from claude_agent_sdk import HookMatcher from claude_agent_sdk._errors import MessageParseError from nemo_fabric_adapter_contract.codec import ContractValidationError @@ -43,6 +44,7 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import relay_artifacts from nemo_fabric_adapters.common import relay_gateway from nemo_fabric_adapters.common import relay_hooks @@ -209,16 +211,21 @@ def _settings(config: AgentConfig) -> dict[str, Any]: return config.harness.settings if config.harness is not None else {} +def _selected_model_role(config: AgentConfig) -> str: + """The model role Claude Code runs: ``default``, or the sole configured role.""" + + if "default" in config.models: + return "default" + if len(config.models) == 1: + return next(iter(config.models)) + raise AdapterConfigError( + "claude_invalid_configuration", + "Claude requires a default model or exactly one model", + ) + + def _selected_model_config(config: AgentConfig) -> AgentModelConfig: - model = config.models.get("default") - if model is None and len(config.models) == 1: - model = next(iter(config.models.values())) - if model is None: - raise AdapterConfigError( - "claude_invalid_configuration", - "Claude requires a default model or exactly one model", - ) - return model + return config.models[_selected_model_role(config)] def _resolve_path(base_dir: str, value: str | Path) -> Path: @@ -825,6 +832,27 @@ def sdk_failure(error: BaseException) -> dict[str, Any]: _DETAILED_TOKEN_USAGE_KEY = "nemo_fabric.detailed_token_usage.v1" +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-claude" +_HARNESS_NAME = "claude-code" +_METHOD_INIT_MESSAGE = "harness_init_message" + + +def init_message_version(messages: list[Message]) -> dict[str, Any] | None: + """The Claude Code version the latest init message names, as a fact. + + ``None`` when no init message arrived. An init message without a version + reads ``unavailable`` with reason ``empty_response``. + """ + + for message in reversed(messages): + if isinstance(message, SystemMessage) and message.subtype == "init": + version = message.data.get("claude_code_version") + if isinstance(version, str) and version: + return provenance.known_fact(version, _METHOD_INIT_MESSAGE) + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_INIT_MESSAGE + ) + return None # Claude Code's result usage keys and the detailed input counters they fill. # Anthropic's input_tokens excludes cache reads and cache writes, and its # output_tokens includes thinking, which Anthropic bills as output. @@ -871,7 +899,9 @@ def _detailed_token_usage(usage: dict[str, Any]) -> dict[str, int | bool]: return counters -def _agent_run_result(output: dict[str, Any]) -> AgentRunResult: +def _agent_run_result( + output: dict[str, Any], versions: dict[str, Any] | None = None +) -> AgentRunResult: normalized = dict(output) failed = bool(normalized.pop("failed", False)) reported_error = normalized.pop("error", None) @@ -914,6 +944,7 @@ def _agent_run_result(output: dict[str, Any]) -> AgentRunResult: output=normalized, error=error, usage=agent_usage, + extensions={provenance.VERSIONS_REPORT_KEY: versions} if versions else {}, ) @@ -1068,6 +1099,10 @@ def __init__(self) -> None: self._gateway_process: subprocess.Popen[Any] | None = None self._mcp_config_path: Path | None = None self._unusable = False + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_INIT_MESSAGE + ) async def start(self, payload: dict[str, Any]) -> None: if self._client is not None: @@ -1130,12 +1165,14 @@ async def invoke( "claude_runtime_mismatch", "Claude invocation does not match the connected runtime", ) + messages: list[Message] = [] if self._unusable: return _agent_run_result( _failure( "claude_runtime_unavailable", "Claude runtime cannot accept another invocation after a runtime failure", - ) + ), + self._versions_report(agent_config, messages), ) invocation_deadline = lifecycle.invocation_deadline(runtime_context) @@ -1157,6 +1194,7 @@ async def invoke( client, prompt, _remaining_timeout(invocation_deadline), + messages, ) if ( output.get("completed") @@ -1183,22 +1221,43 @@ async def invoke( ), relay, artifacts=[], - ) + ), + self._versions_report(agent_config, messages), ) if self._relay is not None: output = _relay_output(output, self._relay) - return _agent_run_result(output) + return _agent_run_result(output, self._versions_report(agent_config, messages)) + + def _versions_report( + self, config: AgentConfig, messages: list[Message] + ) -> dict[str, Any]: + """Report this runtime's versions, refreshed from any init message.""" + + version = init_message_version(messages) + if version is not None: + self._harness_version = version + return provenance.versions_report( + package_version=self._package_version, + harness_name=provenance.known_fact(_HARNESS_NAME, _METHOD_INIT_MESSAGE), + harness_version=self._harness_version, + served_models={role: provenance.no_source_fact() for role in config.models}, + ) async def _run_query( self, client: ClaudeSDKClient, prompt: str, remaining_timeout: float, + messages: list[Message] | None = None, ) -> dict[str, Any]: - """Run one SDK query and normalize its terminal result.""" + """Run one SDK query and normalize its terminal result. - messages: list[Message] = [] + Every message other than the terminal result is appended to + ``messages``, so the caller can read this invocation's messages. + """ + + messages = [] if messages is None else messages result: ResultMessage | None = None deadline = asyncio.timeout(remaining_timeout) try: diff --git a/adapters/python/codex/codex.fabric-adapter.json b/adapters/python/codex/codex.fabric-adapter.json index 03819ee12..3879c64f8 100644 --- a/adapters/python/codex/codex.fabric-adapter.json +++ b/adapters/python/codex/codex.fabric-adapter.json @@ -94,6 +94,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py index e463332b7..be0143a43 100644 --- a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py +++ b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py @@ -59,9 +59,13 @@ import nemo_fabric_adapters.common.relay_artifacts as relay_artifacts import nemo_fabric_adapters.common.utils as common_utils from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance INTERRUPT_TIMEOUT_SECONDS = 5.0 +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-codex" +_HARNESS_NAME = "codex" +_METHOD_HARNESS_API = "harness_api" CLOCK_RESOLUTION_SECONDS = time.get_clock_info("monotonic").resolution SANDBOXES = { "read-only": Sandbox.read_only, @@ -1078,8 +1082,48 @@ def _normalize_usage( return AgentUsage(**tokens, input_tokens_include_cache=True) +async def read_cli_version( + thread: Any, timeout: float = provenance.PROBE_TIMEOUT_SECONDS +) -> dict[str, Any]: + """The version of the Codex CLI that created ``thread``, from its thread data. + + One bounded ``thread/read`` request: a timeout reads ``collection_failed`` + with reason ``timeout``, and a failed request reads reason ``no_result``. + """ + + try: + async with asyncio.timeout(timeout): + response = await thread.read() + except TimeoutError: + return provenance.missing_fact("collection_failed", "timeout", _METHOD_HARNESS_API) + except (CodexError, RuntimeError, OSError, ValueError): + # ValueError covers a response the SDK's models reject. + return provenance.missing_fact( + "collection_failed", "no_result", _METHOD_HARNESS_API + ) + version = getattr(getattr(response, "thread", None), "cli_version", None) + if not isinstance(version, str) or not version: + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_HARNESS_API + ) + return provenance.known_fact(version, _METHOD_HARNESS_API) + + +def versions_report(config: AgentConfig, harness_version: dict[str, Any]) -> dict[str, Any]: + """Codex's versions report. Codex knows only the configured model.""" + + return provenance.versions_report( + package_version=provenance.package_version_fact(_PACKAGE_DISTRIBUTION), + harness_name=provenance.known_fact(_HARNESS_NAME, _METHOD_HARNESS_API), + harness_version=harness_version, + served_models={role: provenance.no_source_fact() for role in config.models}, + ) + + def _agent_run_result( - output: dict[str, Any], cumulative_totals: dict[str, int] | None = None + output: dict[str, Any], + cumulative_totals: dict[str, int] | None = None, + versions: dict[str, Any] | None = None, ) -> AgentRunResult: normalized = dict(output) failed = bool(normalized.pop("failed", False)) @@ -1105,6 +1149,7 @@ def _agent_run_result( output=normalized, error=error, usage=agent_usage, + extensions={provenance.VERSIONS_REPORT_KEY: versions} if versions else {}, ) @@ -1338,6 +1383,9 @@ def __init__(self) -> None: self._mcp_authentication_checked = False self._unusable = False self._usage_totals: dict[str, int] = dict.fromkeys(USAGE_FIELDS, 0) + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_HARNESS_API + ) async def start(self, payload: dict[str, Any]) -> None: if self._client is not None: @@ -1401,6 +1449,7 @@ async def start(self, payload: dict[str, Any]) -> None: base_dir, relay=relay, ) + harness_version = await read_cli_version(thread) except CodexAdapterError as error: await self._cleanup_failed_start() raise _as_lifecycle_error(error) from error @@ -1423,6 +1472,7 @@ async def start(self, payload: dict[str, Any]) -> None: self._fabric_runtime_id = fabric_runtime_id self._thread = thread self._usage_totals = dict.fromkeys(USAGE_FIELDS, 0) + self._harness_version = harness_version async def invoke( self, @@ -1454,7 +1504,8 @@ async def invoke( _failure( "codex_runtime_unavailable", "Codex runtime cannot accept another invocation after a runtime failure", - ) + ), + versions=versions_report(config, self._harness_version), ) invocation_deadline = lifecycle.invocation_deadline(runtime_context) @@ -1508,7 +1559,9 @@ async def invoke( ) failure["usage"] = output.get("usage") return _agent_run_result( - _relay_output(failure, relay, artifacts=[]), self._usage_totals + _relay_output(failure, relay, artifacts=[]), + self._usage_totals, + versions_report(config, self._harness_version), ) except AdapterRelayError as error: output = adapter_failure(error) @@ -1520,7 +1573,9 @@ async def invoke( self._unusable = not usable if self._relay is not None: output = _relay_output(output, self._relay) - return _agent_run_result(output, self._usage_totals) + return _agent_run_result( + output, self._usage_totals, versions_report(config, self._harness_version) + ) async def stop(self) -> None: client = self._client diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py new file mode 100644 index 000000000..43d41933d --- /dev/null +++ b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py @@ -0,0 +1,215 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build the ``nemo_fabric.versions.v1`` report an adapter returns with each result. + +NeMo Fabric core reads the report from ``AgentRunResult.extensions`` when the +adapter's descriptor names the key under its ``run_result`` extension schema, +and moves its facts into ``RunResult.what_ran``. Every fact is reported by the +adapter, so each carries ``source`` ``reported`` and ``collected_by`` +``adapter``. A fact holds ``value``, or ``state`` and ``reason``, never both. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import importlib.metadata +import sys +import threading +from collections.abc import Callable +from collections.abc import Mapping +from pathlib import Path +from typing import Any +from typing import TypeVar + +VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1" +PROBE_TIMEOUT_SECONDS = 5.0 + +METHOD_ADAPTER_REPORT = "adapter_report" +METHOD_PACKAGE_METADATA = "package_metadata" + +_MAX_VALUE_CHARS = 256 +_TRUNCATED_PREFIX_CHARS = 240 +_TRUNCATED_DIGEST_HEX_DIGITS = 15 +_MAX_MODEL_ROLES = 16 +_MAX_KEY_CHARS = 128 + +T = TypeVar("T") + + +def known_fact(value: str, method: str) -> dict[str, Any]: + """A reported fact with a value, bounded by the string length rule. + + A lone surrogate, which JSON text from a harness can carry, becomes ``?`` + first, because core rejects a result that holds one. + """ + + sanitized = value.encode("utf-8", "replace").decode("utf-8") + return { + "value": bounded_text(sanitized), + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +def missing_fact( + state: str, reason: str, method: str = METHOD_ADAPTER_REPORT +) -> dict[str, Any]: + """A reported fact without a value.""" + + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +def no_source_fact() -> dict[str, Any]: + """A fact this adapter has no source for.""" + + return missing_fact("unsupported", "no_source") + + +def bounded_text(text: str) -> str: + """Apply NeMo Fabric core's string length rule. + + A string of at most 256 characters is kept whole. A longer one keeps its + first 240 characters, then ``~``, then the first 15 hex digits of the + SHA-256 of its UTF-8 bytes, as core does. + """ + + if len(text) <= _MAX_VALUE_CHARS: + return text + digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + return ( + f"{text[:_TRUNCATED_PREFIX_CHARS]}~{digest[:_TRUNCATED_DIGEST_HEX_DIGITS]}" + ) + + +def package_version_fact(distribution: str) -> dict[str, Any]: + """The installed version of ``distribution``, read from its package metadata.""" + + try: + version = importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError: + return missing_fact("unavailable", "not_collected", METHOD_PACKAGE_METADATA) + if not version: + return missing_fact("unavailable", "empty_response", METHOD_PACKAGE_METADATA) + return known_fact(version, METHOD_PACKAGE_METADATA) + + +def loaded_module_version_fact(distribution: str, module_name: str) -> dict[str, Any]: + """The version of ``distribution`` when it installed the loaded ``module_name``. + + Another copy of the module can shadow the installed package, for example a + source checkout earlier on ``sys.path``. The version is reported only when + the distribution's file list holds the file the module was loaded from; + otherwise it reads ``unavailable`` with reason ``not_attributed``. A module + that is not loaded reads ``not_collected``. + """ + + module = sys.modules.get(module_name) + module_file = getattr(module, "__file__", None) + if not isinstance(module_file, str): + return missing_fact("unavailable", "not_collected", METHOD_PACKAGE_METADATA) + try: + installed = importlib.metadata.distribution(distribution) + except importlib.metadata.PackageNotFoundError: + return missing_fact("unavailable", "not_collected", METHOD_PACKAGE_METADATA) + loaded = Path(module_file).resolve() + if not any( + Path(str(installed.locate_file(file))).resolve() == loaded + for file in installed.files or () + if file.name == loaded.name + ): + return missing_fact("unavailable", "not_attributed", METHOD_PACKAGE_METADATA) + version = installed.version + if not version: + return missing_fact("unavailable", "empty_response", METHOD_PACKAGE_METADATA) + return known_fact(version, METHOD_PACKAGE_METADATA) + + +def versions_report( + *, + package_version: Mapping[str, Any], + harness_name: Mapping[str, Any], + harness_version: Mapping[str, Any], + served_models: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + """The ``nemo_fabric.versions.v1`` report object. + + ``served_models`` maps each plan role to its served-model fact. Like core, + the report keeps at most 16 roles in sorted order and skips a role longer + than 128 characters, so the object stays far below core's 64 KiB limit + for one report key. + """ + + roles = sorted(role for role in served_models if len(role) <= _MAX_KEY_CHARS) + return { + "adapter": {"package_version": dict(package_version)}, + "harness": {"name": dict(harness_name), "version": dict(harness_version)}, + "models": { + role: {"served": dict(served_models[role])} + for role in roles[:_MAX_MODEL_ROLES] + }, + } + + +class BlockingProbe: + """Run a runtime's blocking version calls off the event loop, one at a time. + + A blocking call cannot be cancelled, so it runs on a daemon thread and the + caller waits at most ``timeout`` seconds for it. While an earlier call + still runs, no new thread starts and the call times out at once, so a + runtime never has more than one outstanding probe. Stopping the runtime + never waits for the thread, and a daemon thread never delays process exit. + """ + + def __init__(self) -> None: + self._running = threading.Event() + + async def call( + self, function: Callable[[], T], *, timeout: float = PROBE_TIMEOUT_SECONDS + ) -> T: + """Return ``function()``, or raise ``TimeoutError`` past ``timeout``. + + An exception that ``function`` raises reaches the caller. + """ + + if self._running.is_set(): + raise TimeoutError("an earlier version probe is still running") + loop = asyncio.get_running_loop() + outcome: asyncio.Future[T] = loop.create_future() + + def deliver(result: T | None, error: Exception | None) -> None: + if outcome.done(): + return + if error is not None: + outcome.set_exception(error) + else: + outcome.set_result(result) # type: ignore[arg-type] + + def run() -> None: + result: T | None = None + error: Exception | None = None + try: + result = function() + except Exception as raised: + error = raised + finally: + self._running.clear() + try: + loop.call_soon_threadsafe(deliver, result, error) + except RuntimeError: + # The event loop closed while the probe ran; nobody waits for it. + return + + self._running.set() + threading.Thread(target=run, name="nemo-fabric-version-probe", daemon=True).start() + # On timeout, wait_for cancels only this future. The thread keeps + # running, and its late outcome is dropped. + return await asyncio.wait_for(outcome, timeout) diff --git a/adapters/python/deepagents/deepagents.fabric-adapter.json b/adapters/python/deepagents/deepagents.fabric-adapter.json index 12eb69898..796fbbfb2 100644 --- a/adapters/python/deepagents/deepagents.fabric-adapter.json +++ b/adapters/python/deepagents/deepagents.fabric-adapter.json @@ -195,6 +195,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 618957f1f..721e31221 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -36,6 +36,7 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance import nemo_fabric_adapters.common.utils as common_utils HARNESS = "deepagents" @@ -587,6 +588,11 @@ def _gated_subagents( return gated +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-deepagents" +_HARNESS_DISTRIBUTION = "deepagents" +_HARNESS_MODULE = "deepagents" + + class DeepAgentsRuntime: """One compiled Deep Agents graph and checkpointer owned by a Fabric runtime.""" @@ -609,6 +615,9 @@ def __init__(self) -> None: self._callback_handler_type: Any = None self._telemetry_quarantine: str | None = None self._telemetry_quarantine_cause: str | None = None + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", provenance.METHOD_PACKAGE_METADATA + ) async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -669,6 +678,9 @@ async def start(self, payload: dict[str, Any]) -> None: from deepagents import create_deep_agent + self._harness_version = provenance.loaded_module_version_fact( + _HARNESS_DISTRIBUTION, _HARNESS_MODULE + ) agent = create_deep_agent( **_supported_kwargs(create_deep_agent, agent_kwargs) ) @@ -799,6 +811,21 @@ async def invoke( if tokens or cost is not None else None ), + extensions={ + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=provenance.package_version_fact( + _PACKAGE_DISTRIBUTION + ), + harness_name=provenance.known_fact( + _HARNESS_DISTRIBUTION, provenance.METHOD_PACKAGE_METADATA + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() + for role in start_payload["config"].models + }, + ) + }, ) async def _invoke_with_telemetry( diff --git a/adapters/python/hermes/hermes.fabric-adapter.json b/adapters/python/hermes/hermes.fabric-adapter.json index 7ae06401d..f56822531 100644 --- a/adapters/python/hermes/hermes.fabric-adapter.json +++ b/adapters/python/hermes/hermes.fabric-adapter.json @@ -68,6 +68,14 @@ "provider": {"type": "string"} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 653304993..0f8fe2e18 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -15,6 +15,7 @@ import logging import os import sys +from collections.abc import Callable from contextlib import redirect_stdout from io import StringIO from pathlib import Path @@ -30,6 +31,7 @@ from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.hermes import configuration from nemo_fabric_adapters.hermes import telemetry @@ -37,9 +39,51 @@ # The actual internal hermes default as of v0.21 is `sys.maxsize` DEFAULT_MAX_ITERATIONS: int = 90 LOGGER = logging.getLogger(__name__) +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-hermes" +_HARNESS_NAME = "hermes-agent" +_METHOD_HARNESS_API = "harness_api" hermes_mcp_server_config = configuration.hermes_mcp_server_config +def hermes_version() -> str: + """The running Hermes Agent's version from its version call. + + ``get_version_info()`` can run several ``git`` commands, so callers run it + through a :class:`provenance.BlockingProbe`. + """ + + from hermes_cli.version_info import get_version_info + + return get_version_info().display_version + + +async def probe_hermes_version( + probe: provenance.BlockingProbe, + version_call: Callable[[], str] = hermes_version, + timeout: float = provenance.PROBE_TIMEOUT_SECONDS, +) -> dict[str, Any]: + """Hermes's version as a reported fact, bounded by the probe time limit.""" + + try: + version = await probe.call(version_call, timeout=timeout) + except TimeoutError: + return provenance.missing_fact("collection_failed", "timeout", _METHOD_HARNESS_API) + except ImportError: + # A Hermes Agent older than its version call. + return provenance.missing_fact("unsupported", "no_source", _METHOD_HARNESS_API) + except Exception: + # Provenance never fails a run, whatever Hermes's version call raises. + LOGGER.exception("Hermes version call failed") + return provenance.missing_fact( + "collection_failed", "no_result", _METHOD_HARNESS_API + ) + if not isinstance(version, str) or not version or version == "unknown": + return provenance.missing_fact( + "unavailable", "empty_response", _METHOD_HARNESS_API + ) + return provenance.known_fact(version, _METHOD_HARNESS_API) + + def main() -> None: """Serve the persistent local-host lifecycle protocol.""" @@ -66,6 +110,10 @@ def __init__(self) -> None: self._relay_plugin_config: dict[str, Any] | None = None self._relay_plugin_config_path: Path | None = None self._active_invoke_task: asyncio.Task[tuple[dict[str, Any], str]] | None = None + self._version_probe = provenance.BlockingProbe() + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_HARNESS_API + ) async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -151,6 +199,7 @@ async def start(self, payload: dict[str, Any]) -> None: from hermes_state import SessionDB from run_agent import AIAgent + self._harness_version = await probe_hermes_version(self._version_probe) with redirect_stdout(StringIO()): if relay_enabled: common_utils.reject_ambient_relay_plugin_config() @@ -368,10 +417,29 @@ def clear_active_invoke_task( code="hermes_reported_failure", message="Hermes did not complete the invocation", ) + if self._harness_version.get("reason") == "timeout": + # Pick up a version call that finished after its limit. While it + # still runs, the probe starts no new one. + self._harness_version = await probe_hermes_version(self._version_probe) return AgentRunResult( status=AgentRunStatus.FAILED if failed else AgentRunStatus.SUCCEEDED, output=output, error=error, + extensions={ + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=provenance.package_version_fact( + _PACKAGE_DISTRIBUTION + ), + harness_name=provenance.known_fact( + _HARNESS_NAME, _METHOD_HARNESS_API + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() + for role in agent_config.models + }, + ) + }, ) async def _authenticate_mcp_servers(self) -> None: diff --git a/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json b/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json index 70a6e0929..3b38b405d 100644 --- a/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json +++ b/adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json @@ -39,6 +39,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py b/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py index 15df4f5c4..148e924d0 100644 --- a/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py +++ b/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py @@ -12,6 +12,7 @@ from nemo_fabric_adapter_contract import models as contract from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import utils as common_utils SYSTEM_TEMPLATE = "{{system_instruction}}" @@ -25,6 +26,9 @@ your final response, so include it in the same command. """ _UNREADABLE = object() +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-mini-swe-agent" +_HARNESS_DISTRIBUTION = "mini-swe-agent" +_HARNESS_MODULE = "minisweagent" def _selected_model(config: contract.AgentConfig) -> contract.AgentModelConfig: @@ -46,6 +50,10 @@ def __init__(self) -> None: self._relay_plugin = self._relay_scope = self._relay_scope_type = None self._relay_plugin_config: dict[str, Any] | None = None self._telemetry_quarantine: str | None = None + self._model_roles: list[str] = [] + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", provenance.METHOD_PACKAGE_METADATA + ) async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] @@ -61,6 +69,10 @@ async def start(self, payload: dict[str, Any]) -> None: ) from nemo_fabric_adapters.mini_swe_agent.agents import RetainingDefaultAgent + self._model_roles = list(config.models) + self._harness_version = provenance.loaded_module_version_fact( + _HARNESS_DISTRIBUTION, _HARNESS_MODULE + ) runtime_context = contract.RuntimeContext.from_mapping( payload.get("runtime_context") ) @@ -179,6 +191,20 @@ async def invoke( ), output=output, error=error, + extensions={ + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=provenance.package_version_fact( + _PACKAGE_DISTRIBUTION + ), + harness_name=provenance.known_fact( + _HARNESS_DISTRIBUTION, provenance.METHOD_PACKAGE_METADATA + ), + harness_version=self._harness_version, + served_models={ + role: provenance.no_source_fact() for role in self._model_roles + }, + ) + }, ) async def _run_agent(self, task: str) -> dict[str, Any]: diff --git a/adapters/python/openclaw/openclaw.fabric-adapter.json b/adapters/python/openclaw/openclaw.fabric-adapter.json index 38ec42f4b..9c4d4e844 100644 --- a/adapters/python/openclaw/openclaw.fabric-adapter.json +++ b/adapters/python/openclaw/openclaw.fabric-adapter.json @@ -112,6 +112,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py index 75d99c9fa..719bf2abb 100644 --- a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py +++ b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py @@ -32,6 +32,7 @@ from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle from nemo_fabric_adapters.common import openai_chat +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import utils as common_utils from nemo_fabric_adapters.openclaw import _windows_job @@ -50,6 +51,12 @@ MAX_PORT_ATTEMPTS = 20 MAX_DERIVED_PORT_OFFSET = 110 MAX_TCP_PORT = 65_535 +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-openclaw" +_HARNESS_NAME = "openclaw" +# The local gateway's version comes from `openclaw --version`; an attached +# gateway's from its own startup endpoint. +_METHOD_VERSION_PROBE = "harness_probe" +_METHOD_GATEWAY_API = "harness_api" logger = logging.getLogger(__name__) @@ -759,6 +766,9 @@ def __init__(self) -> None: self._previous_signal_handlers: dict[int, Any] = {} self._signal_handler: Any = None self._windows_job: int | None = None + self._harness_version = provenance.missing_fact( + "unavailable", "not_collected", _METHOD_VERSION_PROBE + ) async def start(self, payload: dict[str, Any]) -> dict[str, Any] | None: config: contract.AgentConfig = payload["config"] @@ -806,6 +816,7 @@ async def start(self, payload: dict[str, Any]) -> dict[str, Any] | None: command, "--version", env=child_env, timeout=startup_timeout ) version = _validate_openclaw_version(version_output) + self._harness_version = provenance.known_fact(version, _METHOD_VERSION_PROBE) port = _select_port(settings) @@ -1029,7 +1040,7 @@ async def _connect_service_runtime( ) client = self._client_for(settings, token) try: - await self._probe_gateway(client, gateway_url, agent_id) + version = await self._probe_gateway(client, gateway_url, agent_id) except BaseException: await client.aclose() raise @@ -1038,6 +1049,7 @@ async def _connect_service_runtime( self._context = context self._gateway_url = gateway_url self._agent_id = agent_id + self._harness_version = provenance.known_fact(version, _METHOD_GATEWAY_API) async def _attach_service( self, @@ -1120,6 +1132,9 @@ async def _attach_service( self._context = context self._gateway_url = gateway_url self._agent_id = agent_id + self._harness_version = provenance.known_fact( + openclaw_version, _METHOD_GATEWAY_API + ) return self._service_info( gateway_url, agent_id, @@ -1370,6 +1385,7 @@ async def invoke( retryable=error.response.status_code >= 500 or error.response.status_code == 429, ), + extensions=self._versions_extensions(), ) except httpx.RequestError as error: raise lifecycle.LifecycleError( @@ -1386,8 +1402,24 @@ async def invoke( status=contract.AgentRunStatus.SUCCEEDED, output={"response": text}, usage=usage, + extensions=self._versions_extensions(), ) + def _versions_extensions(self) -> dict[str, Any]: + """The versions report, with the gateway version read at startup.""" + + models = self._config.models if self._config is not None else {} + return { + provenance.VERSIONS_REPORT_KEY: provenance.versions_report( + package_version=provenance.package_version_fact(_PACKAGE_DISTRIBUTION), + harness_name=provenance.known_fact( + _HARNESS_NAME, self._harness_version["method"] + ), + harness_version=self._harness_version, + served_models={role: provenance.no_source_fact() for role in models}, + ) + } + async def stop(self) -> None: cancellation: asyncio.CancelledError | None = None self._stopping = True diff --git a/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json b/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json index 5af1a312f..035cb01a1 100644 --- a/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/claude/claude.fabric-adapter.json @@ -99,6 +99,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json b/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json index 03819ee12..3879c64f8 100644 --- a/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/codex/codex.fabric-adapter.json @@ -94,6 +94,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json b/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json index 12eb69898..796fbbfb2 100644 --- a/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/deepagents/deepagents.fabric-adapter.json @@ -195,6 +195,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json b/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json index 7ae06401d..f56822531 100644 --- a/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json +++ b/crates/fabric-cli/assets/adapters/hermes/hermes.fabric-adapter.json @@ -68,6 +68,14 @@ "provider": {"type": "string"} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 667fe951a..a452c7f94 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -6865,6 +6865,10 @@ mod tests { fn invocation_extensions_are_fail_closed_and_schema_validated() { let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); let mut descriptor = load_adapter_descriptor(&path).expect("Claude descriptor"); + // Start from a descriptor that declares no result extension schema. + descriptor + .extension_schemas + .remove(&AdapterExtensionPoint::RunResult); let request = AgentRunRequest { input: serde_json::json!("review"), relay_session_root: None, diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index f9be07a25..a10a21695 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -2059,7 +2059,7 @@ mod tests { #[test] fn an_undeclared_report_key_stays_in_the_extensions_and_reads_key_not_declared() { - let plan = plan_with_secrets(); + let plan = plan_declaring(&[]); let extensions = json!({VERSIONS_REPORT_KEY: versions_report()}); let (what_ran, kept) = ingest(&plan, extensions.clone()); @@ -2077,7 +2077,7 @@ mod tests { #[test] fn a_report_key_declared_only_through_a_reference_is_not_declared() { - let mut plan = plan_with_secrets(); + let mut plan = plan_declaring(&[]); let schema = json!({"type": "object", "allOf": [{"properties": { VERSIONS_REPORT_KEY: {"type": "object"}}}]}); plan.adapter_descriptor diff --git a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json index 4c15e1112..14d79212e 100644 --- a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json +++ b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json @@ -42,6 +42,16 @@ }, "type": "object" }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + }, "usage": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -196,7 +206,7 @@ "name": "nemo-fabric-adapters-claude", "version": "0.6.0" }, - "sha256": "cded65f1e927f3b7154f79b3eb6312a5fa34551265ffe6df1ec271e55de9c407", + "sha256": "027fa0b3d396e93dce27d5848320165c27a52c909955f774f2a4372728d549d0", "source": "adapters/python/claude/claude.fabric-adapter.json" }, "nvidia.fabric.cline": { @@ -377,6 +387,16 @@ } }, "type": "object" + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" } }, "model_schema": { @@ -530,7 +550,7 @@ "name": "nemo-fabric-adapters-codex", "version": "0.6.0" }, - "sha256": "f66fb66d3a366977619bc9e88a4d926e0b71bbc28ac965b9a416bf2559106ce5", + "sha256": "019e21098570ba1a5725e7b6c2efe5f34b84e688cd38ce432b0562e444672cbd", "source": "adapters/python/codex/codex.fabric-adapter.json" }, "nvidia.fabric.droid": { @@ -697,6 +717,16 @@ } }, "type": "object" + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" } }, "requirements": {}, @@ -781,7 +811,7 @@ "name": "nemo-fabric-adapters-hermes", "version": "0.6.0" }, - "sha256": "e482a89dda834dc5bfdbcb7f6041ad5e457a73918d74d8a8215f0171b8947465", + "sha256": "75969f9ab81e0211c0248cec9d3193ef90dcf075af38265819f9d84adcb7f0c0", "source": "adapters/python/hermes/hermes.fabric-adapter.json" }, "nvidia.fabric.kilo": { @@ -907,6 +937,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1164,7 +1206,7 @@ "name": "nemo-fabric-adapters-deepagents", "version": "0.6.0" }, - "sha256": "819b1f9747af83011c98d5889c56dac9a735008b25e93bbc94c1cfde3050a44e", + "sha256": "1fb843947fc2383a7ef92946f0ba889b22557f4f7427e50bb17345f5f6ed206c", "source": "adapters/python/deepagents/deepagents.fabric-adapter.json" }, "nvidia.fabric.mini-swe-agent": { @@ -1192,6 +1234,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1270,7 +1324,7 @@ "name": "nemo-fabric-adapters-mini-swe-agent", "version": "0.6.0" }, - "sha256": "40ee9364117fd6d4d1fe4e9a822ecd7b9022a5f0f8053d8b550d065e3726df37", + "sha256": "e78d5665059801baafa9012fdb252c0ab110f065a47089996e14e280e385be77", "source": "adapters/python/mini-swe-agent/mini-swe-agent.fabric-adapter.json" }, "nvidia.fabric.nooa": { @@ -1490,6 +1544,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1646,7 +1712,7 @@ "name": "nemo-fabric-adapters-openclaw", "version": "0.6.0" }, - "sha256": "5ab8b29595214e214214b051b60bc314dc65f07add60a73283f434a62e3f7402", + "sha256": "eef2a0677ed569f2433a47886fd86bf05febb7e5ffa2d67b08b2ab2f14f206e0", "source": "adapters/python/openclaw/openclaw.fabric-adapter.json" }, "nvidia.fabric.opencode": { diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py new file mode 100644 index 000000000..2b3627f09 --- /dev/null +++ b/tests/adapters/test_adapters_common_provenance.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +import json +import sys +import threading +import time +import types +from pathlib import Path + +import deepagents +import minisweagent +import pytest +from nemo_fabric_adapters.common import provenance + + +def known(value: str, method: str = "package_metadata") -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +def missing(state: str, reason: str, method: str = "package_metadata") -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +# The same vectors as core's string length test in `what_ran.rs`. +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("é" * 256, "é" * 256), + ("m" * 300 + "a", "m" * 240 + "~96858eedeb3c8e6"), + ("é" * 300, "é" * 240 + "~7250b66610f8b7d"), + ], +) +def test_bounded_text_matches_core_string_length_rule(text, expected): + assert provenance.bounded_text(text) == expected + + +def test_known_fact_replaces_a_lone_surrogate_so_the_result_stays_valid_json(): + fact = provenance.known_fact("claude-\ud800", "harness_api") + + assert fact["value"] == "claude-?" + json.dumps(fact).encode("utf-8") + + +def test_package_version_fact_reads_installed_metadata(): + assert provenance.package_version_fact("nemo-fabric-adapters-common") == known( + "0.6.0" + ) + assert provenance.package_version_fact("nemo-fabric-no-such-package") == missing( + "unavailable", "not_collected" + ) + + +@pytest.mark.parametrize( + ("distribution", "module", "expected"), + [ + ("deepagents", deepagents.__name__, known("0.7.9")), + ("mini-swe-agent", minisweagent.__name__, known("2.4.6")), + # The distribution exists but did not install the loaded module. + ("mini-swe-agent", deepagents.__name__, missing("unavailable", "not_attributed")), + ("deepagents", "nemo_fabric_not_loaded", missing("unavailable", "not_collected")), + ], +) +def test_loaded_module_version_fact_accepts_only_the_module_the_package_installed( + distribution, module, expected +): + assert provenance.loaded_module_version_fact(distribution, module) == expected + + +def test_loaded_module_version_fact_rejects_a_shadowing_source_checkout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + shadow = types.ModuleType("deepagents") + shadow.__file__ = str(tmp_path / "deepagents" / "__init__.py") + monkeypatch.setitem(sys.modules, "deepagents", shadow) + + assert provenance.loaded_module_version_fact("deepagents", "deepagents") == missing( + "unavailable", "not_attributed" + ) + + +def test_versions_report_keeps_16_roles_in_sorted_order_and_bounds_values(): + served = {f"role-{index:02}": provenance.no_source_fact() for index in range(17)} + served["r" * 129] = provenance.no_source_fact() + + report = provenance.versions_report( + package_version=provenance.known_fact("0.6.0", "package_metadata"), + harness_name=provenance.known_fact("harness", "harness_api"), + harness_version=provenance.known_fact("v" * 300, "harness_api"), + served_models=served, + ) + + assert list(report["models"]) == [f"role-{index:02}" for index in range(16)] + assert report["harness"]["version"]["value"] == provenance.bounded_text("v" * 300) + assert report["adapter"]["package_version"] == known("0.6.0") + + +def probe_threads() -> int: + return sum( + thread.name == "nemo-fabric-version-probe" for thread in threading.enumerate() + ) + + +async def test_blocking_probe_bounds_a_function_that_never_returns(): + release = threading.Event() + calls: list[str] = [] + + def blocking() -> str: + calls.append("called") + release.wait() + return "1.0.0" + + probe = provenance.BlockingProbe() + ticks = 0 + + async def count_ticks() -> None: + nonlocal ticks + while True: + ticks += 1 + await asyncio.sleep(0.01) + + ticker = asyncio.create_task(count_ticks()) + started = time.monotonic() + try: + with pytest.raises(TimeoutError): + await probe.call(blocking, timeout=0.2) + elapsed = time.monotonic() - started + # A second call while the first still runs starts no new thread. + with pytest.raises(TimeoutError): + await probe.call(blocking, timeout=5) + assert time.monotonic() - started < 1 + assert calls == ["called"] + assert probe_threads() == 1 + finally: + ticker.cancel() + release.set() + + assert 0.2 <= elapsed < 1 + # The event loop kept running while the function blocked its thread. + assert ticks >= 5 + while probe_threads(): + await asyncio.sleep(0.01) + assert await probe.call(lambda: "2.0.0", timeout=1) == "2.0.0" + + +async def test_blocking_probe_returns_the_result_or_the_raised_error(): + probe = provenance.BlockingProbe() + + def fail() -> str: + raise RuntimeError("no version") + + assert await probe.call(lambda: "1.2.3") == "1.2.3" + with pytest.raises(RuntimeError, match="no version"): + await probe.call(fail) diff --git a/tests/adapters/test_claude_adapter.py b/tests/adapters/test_claude_adapter.py index e7b17060f..f5a176b47 100644 --- a/tests/adapters/test_claude_adapter.py +++ b/tests/adapters/test_claude_adapter.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import json import os import sys @@ -31,6 +32,7 @@ from claude_agent_sdk import SystemMessage from claude_agent_sdk import TextBlock from claude_agent_sdk._errors import MessageParseError +from claude_agent_sdk._internal.message_parser import parse_message from nemo_fabric_adapter_contract.models import AgentConfig from nemo_fabric_adapter_contract.models import AgentRunRequest from nemo_fabric_adapter_contract.models import AgentRunResult @@ -53,6 +55,8 @@ } DETAILED_TOKEN_USAGE_KEY = "nemo_fabric.detailed_token_usage.v1" +VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1" +CLAUDE_STREAMS = ROOT / "tests" / "fixtures" / "claude_streams" def lifecycle_invocation( @@ -258,6 +262,12 @@ def test_claude_descriptor_is_narrow_and_versioned(): }, "additionalProperties": False, }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"nemo_fabric.versions.v1": {"type": "object"}}, + "additionalProperties": False, + }, }, "config": { "accepts": [ @@ -2123,3 +2133,97 @@ def test_main_serves_persistent_runtime(monkeypatch): serve.assert_called_once_with( adapter.ClaudeRuntime, config_loader=AgentConfig.from_mapping ) + + +def stream_messages(name: str) -> list[Message]: + """Parse a saved Claude Code stream with the SDK's own message parser.""" + + lines = (CLAUDE_STREAMS / name).read_text(encoding="utf-8").splitlines() + return [parse_message(json.loads(line)) for line in lines] + + +async def versions_reports( + claude_payload: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + streams: list[list[Message]], +) -> list[dict[str, Any]]: + """Run one invocation per stream and return each result's versions report.""" + + install_scripted_client(monkeypatch, streams) + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(start_payload)) + reports = [] + for _ in streams: + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + reports.append(result.extensions[VERSIONS_REPORT_KEY]) + await runtime.stop() + return reports + + +def reported(value: str, method: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": method, + } + + +async def test_claude_reports_the_cli_version_from_its_init_message( + claude_payload, monkeypatch +): + reports = await versions_reports( + claude_payload, + monkeypatch, + [stream_messages("normal.jsonl"), [running_total_result(0.1)]], + ) + + package_version = importlib.metadata.version("nemo-fabric-adapters-claude") + for report in reports: + # A later invocation without an init message keeps the runtime's version. + assert report["adapter"] == { + "package_version": reported(package_version, "package_metadata") + } + assert report["harness"] == { + "name": reported("claude-code", "harness_init_message"), + "version": reported("2.1.294", "harness_init_message"), + } + + +async def test_claude_versions_report_matches_the_descriptor( + claude_payload, monkeypatch +): + install_scripted_client(monkeypatch, [stream_messages("normal.jsonl")]) + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(start_payload)) + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + await runtime.stop() + descriptor = json.loads( + (ROOT / "adapters/python/claude/claude.fabric-adapter.json").read_text( + encoding="utf-8" + ) + ) + + jsonschema.validate( + result.extensions, descriptor["extension_schemas"]["run_result"] + ) + + +async def test_claude_without_an_init_message_reads_the_version_as_not_collected( + claude_payload, monkeypatch +): + [report] = await versions_reports( + claude_payload, monkeypatch, [[running_total_result(0.1)]] + ) + + assert report["harness"]["version"] == { + "state": "unavailable", + "reason": "not_collected", + "source": "reported", + "collected_by": "adapter", + "method": "harness_init_message", + } diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index 32cb1bcfe..144309aa1 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -2103,3 +2103,65 @@ def test_main_serves_persistent_runtime(monkeypatch): serve.assert_called_once_with( adapter.CodexRuntime, config_loader=AgentConfig.from_mapping ) + + +def _version_fact(state: str, reason: str) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +async def test_codex_reads_the_cli_version_from_a_real_app_server(tmp_path: Path): + import importlib.metadata + + from openai_codex.client import CodexConfig + + env = {"CODEX_HOME": str(tmp_path), "HOME": str(tmp_path), "PATH": os.environ["PATH"]} + async with AsyncCodex(config=CodexConfig(cwd=str(tmp_path), env=env)) as codex: + thread = await codex.thread_start(cwd=str(tmp_path), model="gpt-test") + + fact = await adapter.read_cli_version(thread) + + assert fact == { + "value": importlib.metadata.version("openai-codex-cli-bin"), + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +async def _never_settles() -> None: + await asyncio.Event().wait() + + +@pytest.mark.parametrize( + ("read", "expected"), + [ + pytest.param( + AsyncMock(side_effect=_never_settles), + _version_fact("collection_failed", "timeout"), + id="never-settles", + ), + pytest.param( + AsyncMock(side_effect=JsonRpcError(-32603, "internal error")), + _version_fact("collection_failed", "no_result"), + id="rpc-error", + ), + pytest.param( + AsyncMock(return_value=SimpleNamespace(thread=SimpleNamespace(cli_version=""))), + _version_fact("unavailable", "empty_response"), + id="empty", + ), + ], +) +async def test_codex_cli_version_read_degrades_to_a_state(read, expected): + thread = MagicMock(spec=AsyncThread) + thread.read = read + started = time.monotonic() + + assert await adapter.read_cli_version(thread, timeout=0.1) == expected + assert time.monotonic() - started < 1 diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 9a466f0b1..420ca7018 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -1657,3 +1657,158 @@ def test_main_serves_persistent_runtime(monkeypatch): adapter.HermesRuntime, config_loader=AgentConfig.from_mapping, ) + + +def _reported(value: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +def _version_state(state: str, reason: str) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +async def test_hermes_reports_the_version_its_version_call_returns( + monkeypatch, tmp_path: Path +): + from hermes_cli.version_info import get_version_info + + mock_ai_agent_type = MagicMock(spec=AIAgent, return_value=MagicMock(spec=AIAgent)) + monkeypatch.setattr( + mock_ai_agent_type.__init__.__func__, + "__signature__", + inspect.signature(AIAgent.__init__), + raising=False, + ) + hermes_config = ModuleType("hermes_cli.config") + hermes_config.load_config = lambda: {} # type: ignore[attr-defined] + hermes_plugins = ModuleType("hermes_cli.plugins") + hermes_plugins.discover_plugins = lambda force=False: None # type: ignore[attr-defined] + hermes_state = ModuleType("hermes_state") + hermes_state.SessionDB = MagicMock(spec=SessionDB) # type: ignore[attr-defined] + run_agent = ModuleType("run_agent") + run_agent.AIAgent = mock_ai_agent_type # type: ignore[attr-defined] + # Hermes's own version call stays real. + monkeypatch.setitem(sys.modules, "hermes_cli.config", hermes_config) + monkeypatch.setitem(sys.modules, "hermes_cli.plugins", hermes_plugins) + monkeypatch.setitem(sys.modules, "hermes_state", hermes_state) + monkeypatch.setitem(sys.modules, "run_agent", run_agent) + monkeypatch.setenv("TEST_API_KEY", "secret") + mock_turn = MagicMock( + return_value=( + {"final_response": "done", "completed": True, "messages": []}, + "", + ) + ) + monkeypatch.setattr(adapter, "_invoke_hermes_turn", mock_turn) + monkeypatch.setattr( + configuration, "resolve_hermes_toolsets", lambda *_args: None + ) + runtime_context = _runtime_context( + runtime_id="runtime-version", + workspace=str(tmp_path), + artifact_root=str(tmp_path / "artifacts"), + ) + payload = { + "agent_name": "hermes-version", + "base_dir": str(tmp_path), + "runtime_context": runtime_context.to_mapping(), + "config": _agent_config( + { + "harness": {"settings": {}}, + "models": { + "default": { + "provider": "test-provider", + "model": "test-model", + "api_key_env": "TEST_API_KEY", + } + }, + } + ), + } + + runtime = adapter.HermesRuntime() + await runtime.start(payload) + try: + result = await runtime.invoke( + *_invocation( + { + "runtime_context": runtime_context.to_mapping(), + "request": {"input": "complete the task"}, + } + ) + ) + finally: + await runtime.stop() + + report = result.extensions["nemo_fabric.versions.v1"] + assert report["harness"] == { + "name": _reported("hermes-agent"), + "version": _reported(get_version_info().display_version), + } + assert report["adapter"]["package_version"]["value"] == "0.6.0" + + +def _raise_import_error() -> str: + raise ImportError("hermes_cli.version_info") + + +def _raise_runtime_error() -> str: + raise RuntimeError("malformed install stamp") + + +@pytest.mark.parametrize( + ("version_call", "expected"), + [ + pytest.param(lambda: "0.21.6+3", _reported("0.21.6+3"), id="version"), + pytest.param( + lambda: "unknown", + _version_state("unavailable", "empty_response"), + id="unknown", + ), + pytest.param( + _raise_import_error, + _version_state("unsupported", "no_source"), + id="no-version-call", + ), + pytest.param( + _raise_runtime_error, + _version_state("collection_failed", "no_result"), + id="raised", + ), + ], +) +async def test_hermes_version_probe_maps_each_outcome(version_call, expected): + probe = adapter.provenance.BlockingProbe() + + assert await adapter.probe_hermes_version(probe, version_call) == expected + + +async def test_hermes_version_probe_times_out_and_starts_no_second_call(): + release = threading.Event() + calls: list[str] = [] + + def blocking() -> str: + calls.append("called") + release.wait() + return "0.21.6" + + probe = adapter.provenance.BlockingProbe() + try: + first = await adapter.probe_hermes_version(probe, blocking, timeout=0.1) + second = await adapter.probe_hermes_version(probe, blocking, timeout=0.1) + finally: + release.set() + + assert first == second == _version_state("collection_failed", "timeout") + assert calls == ["called"] diff --git a/tests/adapters/test_openclaw.py b/tests/adapters/test_openclaw.py index 031624bf2..9c31c94c4 100644 --- a/tests/adapters/test_openclaw.py +++ b/tests/adapters/test_openclaw.py @@ -7,6 +7,7 @@ import asyncio import ctypes +import importlib.metadata import json import os import shutil @@ -287,6 +288,20 @@ async def test_openclaw_runtime_generates_config_invokes_and_cleans_up( assert result.status == "succeeded" assert result.output == {"response": "OpenClaw response"} assert result.usage.total_tokens == 5 + assert result.extensions["nemo_fabric.versions.v1"]["harness"] == { + "name": { + "value": "openclaw", + "source": "reported", + "collected_by": "adapter", + "method": "harness_probe", + }, + "version": { + "value": "2026.9.4", + "source": "reported", + "collected_by": "adapter", + "method": "harness_probe", + }, + } assert request == { "model": "openclaw/default", "messages": [ @@ -1473,6 +1488,12 @@ async def test_openclaw_plan_doctor_and_run_without_credentials( assert report.status == "pass" assert result.status == "succeeded" assert result.output == {"response": "OpenClaw response"} + # Core moved the adapter's versions report into what_ran. + assert "adapter" not in result.metadata + assert result.what_ran["harness"]["version"]["value"] == "2026.9.4" + assert result.what_ran["adapter"]["package_version"]["value"] == ( + importlib.metadata.version("nemo-fabric-adapters-openclaw") + ) @pytest.mark.skipif( diff --git a/tests/e2e/test_deepagents.py b/tests/e2e/test_deepagents.py index 30b07e9ef..ce96f6511 100644 --- a/tests/e2e/test_deepagents.py +++ b/tests/e2e/test_deepagents.py @@ -9,6 +9,7 @@ from __future__ import annotations +import importlib.metadata import importlib.util import json import os @@ -53,6 +54,25 @@ async def test_deepagents_persistent_host_with_mock_model(api_server, tmp_path): assert first["output"]["resumed"] is False, results assert second["output"]["resumed"] is True, results assert "user_count=2" in second["output"]["response"], results + # Core read the adapter's versions report into what_ran. + for result in (first, second): + assert result.what_ran["harness"] == { + "name": _reported("deepagents"), + "version": _reported(importlib.metadata.version("deepagents")), + }, results + assert result.what_ran["adapter"]["package_version"] == _reported( + importlib.metadata.version("nemo-fabric-adapters-deepagents") + ) + assert "adapter" not in result.metadata + + +def _reported(value: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "package_metadata", + } @pytest.mark.usefixtures("mock_nvidia_api_key", "nemo_relay") diff --git a/tests/e2e/test_mini_swe_agent.py b/tests/e2e/test_mini_swe_agent.py index 8e03b7ccf..f557de688 100644 --- a/tests/e2e/test_mini_swe_agent.py +++ b/tests/e2e/test_mini_swe_agent.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import sys import warnings from pathlib import Path @@ -92,6 +93,32 @@ async def test_mini_swe_agent_plans_diagnoses_and_runs( assert result.status == "succeeded", result.to_mapping() assert result.output["output"] == "done\n" assert result.output["usage"]["api_calls"] == 1 + what_ran = result.what_ran + assert what_ran["harness"] == { + "name": _reported("mini-swe-agent"), + "version": _reported(importlib.metadata.version("mini-swe-agent")), + } + assert what_ran["adapter"]["package_version"] == _reported( + importlib.metadata.version("nemo-fabric-adapters-mini-swe-agent") + ) + assert what_ran["models"]["default"]["served"] == { + "state": "unsupported", + "reason": "no_source", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + } + assert what_ran["models"]["default"]["requested"]["value"] == "gpt-4o-mini" + assert "adapter" not in result.metadata + + +def _reported(value: str) -> dict[str, str]: + return { + "value": value, + "source": "reported", + "collected_by": "adapter", + "method": "package_metadata", + } @pytest.mark.usefixtures("nemo_relay") diff --git a/tests/fixtures/claude_streams/api-error.jsonl b/tests/fixtures/claude_streams/api-error.jsonl new file mode 100644 index 000000000..655684b79 --- /dev/null +++ b/tests/fixtures/claude_streams/api-error.jsonl @@ -0,0 +1,3 @@ +{"type":"system","subtype":"init","session_id":"56f75096-555e-452e-9c99-e8bdf0c3f1e6","tools":["Bash","Read"],"mcp_servers":[],"model":"claude-sonnet-4-5","permissionMode":"default","claude_code_version":"2.1.294","uuid":"cef05a1a-6da4-4dc1-a94f-2b6c26dd46a4","cwd":"/workspace"} +{"type":"assistant","message":{"diagnostics":null,"id":"68fbbc6b-0c2f-4556-94cd-f68c2aff4a43","container":null,"model":"","role":"assistant","stop_details":null,"stop_reason":"stop_sequence","stop_sequence":"","type":"message","usage":{"output_tokens_details":null,"input_tokens":0,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":null,"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":null,"iterations":null,"speed":null,"fallback_credit":null},"content":[{"type":"text","text":"API Error: 400 mock bad request"}],"context_management":null},"parent_tool_use_id":null,"session_id":"56f75096-555e-452e-9c99-e8bdf0c3f1e6","uuid":"72126e34-8d7a-444b-8961-470bea8bb512","timestamp":"2026-10-20T05:19:34.225Z","error":"unknown","is_api_error_message":true} +{"duration_api_ms":0,"stop_reason":"stop_sequence","session_id":"56f75096-555e-452e-9c99-e8bdf0c3f1e6","total_cost_usd":0,"usage":{"output_tokens_details":{"thinking_tokens":0},"input_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":0,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard","fallback_credit":null},"modelUsage":{},"permission_denials":[],"terminal_reason":"api_error","is_error":true,"num_turns":1,"subtype":"success","result":"API Error: 400 mock bad request","type":"result","duration_ms":187,"uuid":"28f8d366-742e-4db2-a7eb-8bb34104492c"} diff --git a/tests/fixtures/claude_streams/normal.jsonl b/tests/fixtures/claude_streams/normal.jsonl new file mode 100644 index 000000000..341eca7fa --- /dev/null +++ b/tests/fixtures/claude_streams/normal.jsonl @@ -0,0 +1,3 @@ +{"type":"system","subtype":"init","session_id":"0760e0e7-374b-41a7-99f5-175a2af654ce","tools":["Bash","Read"],"mcp_servers":[],"model":"claude-sonnet-4-5","permissionMode":"default","claude_code_version":"2.1.294","uuid":"6fbe3966-4850-48f8-89db-81c3f98a7bb9","cwd":"/workspace"} +{"type":"assistant","message":{"id":"msg_mock01","type":"message","role":"assistant","model":"claude-served-snapshot-20991231","content":[{"type":"text","text":"hello"}],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1},"context_management":null},"parent_tool_use_id":null,"session_id":"0760e0e7-374b-41a7-99f5-175a2af654ce","uuid":"6614faf3-67b5-46ce-b5e0-acc00afd4c6e","timestamp":"2026-10-20T05:18:01.090Z"} +{"duration_api_ms":50,"stop_reason":"end_turn","session_id":"0760e0e7-374b-41a7-99f5-175a2af654ce","total_cost_usd":6e-05,"usage":{"input_tokens":10,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":2,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard","fallback_credit":null},"modelUsage":{"claude-sonnet-4-5":{"inputTokens":10,"outputTokens":2,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":6e-05,"contextWindow":200000,"maxOutputTokens":32000,"thinkingTokens":0,"canonicalModel":"claude-sonnet-4-5","provider":"firstParty","costBasis":"list"}},"permission_denials":[],"terminal_reason":"completed","is_error":false,"num_turns":1,"subtype":"success","result":"hello","type":"result","duration_ms":163,"uuid":"5c77634b-91b6-4d74-958a-b17cde293d82"} From c76d9fbb0f891d004ee4f9ba821028c97973704f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:53:42 -0700 Subject: [PATCH 072/159] feat: report the model that served each Claude Code invocation what_ran.models..served stayed unsupported for Claude Code even though each assistant message names the model that answered. The Claude Code adapter now reports, for the role it runs (default, or the sole configured role), the one distinct model named by this invocation's main-thread assistant messages. Messages with a parent_tool_use_id belong to subagents and are skipped, and so are Claude Code's API-error messages. Two distinct models read not_attributed, none reads empty_response, a failed stream reads collection_failed, and every invocation starts from its own messages. Tests feed saved Claude Code CLI streams through the SDK's parser. Signed-off-by: mnajafian-nv --- .../nemo_fabric_adapters/claude/adapter.py | 72 ++++++++- tests/adapters/test_claude_adapter.py | 138 ++++++++++++++++++ 2 files changed, 202 insertions(+), 8 deletions(-) diff --git a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py index ec9238886..2a1f68480 100644 --- a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py +++ b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py @@ -15,11 +15,13 @@ import subprocess from dataclasses import asdict from dataclasses import dataclass +from dataclasses import field from dataclasses import is_dataclass from hashlib import sha256 from pathlib import Path from typing import Any +from claude_agent_sdk import AssistantMessage from claude_agent_sdk import ClaudeAgentOptions from claude_agent_sdk import ClaudeSDKClient from claude_agent_sdk import ClaudeSDKError @@ -835,6 +837,50 @@ def sdk_failure(error: BaseException) -> dict[str, Any]: _PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-claude" _HARNESS_NAME = "claude-code" _METHOD_INIT_MESSAGE = "harness_init_message" +_METHOD_HARNESS_API = "harness_api" +# Claude Code names its own API-error messages with this model. +_SYNTHETIC_MODEL = "" + + +@dataclass +class InvocationMessages: + """One invocation's messages, and why collecting them failed, if it did.""" + + items: list[Message] = field(default_factory=list) + failure: str | None = None + + +def served_model_fact(messages: InvocationMessages | None) -> dict[str, Any]: + """The model that served this invocation's main thread, as a fact. + + Only assistant messages without a ``parent_tool_use_id`` count, so a + subagent's model never decides it, and Claude Code's ```` + API-error messages are skipped. One distinct model is reported. Two or + more read ``not_attributed``, and none reads ``empty_response``. A stream + that failed reads ``collection_failed``, and an invocation that never + queried Claude Code reads ``not_collected``. + """ + + if messages is None: + return provenance.missing_fact("unavailable", "not_collected", _METHOD_HARNESS_API) + if messages.failure is not None: + return provenance.missing_fact( + "collection_failed", messages.failure, _METHOD_HARNESS_API + ) + models = { + message.model + for message in messages.items + if isinstance(message, AssistantMessage) + and message.parent_tool_use_id is None + and isinstance(message.model, str) + and message.model + and message.model != _SYNTHETIC_MODEL + } + if not models: + return provenance.missing_fact("unavailable", "empty_response", _METHOD_HARNESS_API) + if len(models) > 1: + return provenance.missing_fact("unavailable", "not_attributed", _METHOD_HARNESS_API) + return provenance.known_fact(models.pop(), _METHOD_HARNESS_API) def init_message_version(messages: list[Message]) -> dict[str, Any] | None: @@ -1165,7 +1211,7 @@ async def invoke( "claude_runtime_mismatch", "Claude invocation does not match the connected runtime", ) - messages: list[Message] = [] + messages: InvocationMessages | None = None if self._unusable: return _agent_run_result( _failure( @@ -1183,6 +1229,7 @@ async def invoke( except ClaudeSDKError as error: output = sdk_failure(error) else: + messages = InvocationMessages() relay = self._relay atif_before = ( relay_artifacts.snapshot_atif_files(relay.plugin_config) @@ -1230,18 +1277,24 @@ async def invoke( return _agent_run_result(output, self._versions_report(agent_config, messages)) def _versions_report( - self, config: AgentConfig, messages: list[Message] + self, config: AgentConfig, messages: InvocationMessages | None ) -> dict[str, Any]: - """Report this runtime's versions, refreshed from any init message.""" + """Report this runtime's versions and this invocation's served model. + + The harness version is refreshed from any init message. The served + model belongs to the role Claude Code runs; another role has no source. + """ - version = init_message_version(messages) + version = init_message_version(messages.items if messages else []) if version is not None: self._harness_version = version + served = {role: provenance.no_source_fact() for role in config.models} + served[_selected_model_role(config)] = served_model_fact(messages) return provenance.versions_report( package_version=self._package_version, harness_name=provenance.known_fact(_HARNESS_NAME, _METHOD_INIT_MESSAGE), harness_version=self._harness_version, - served_models={role: provenance.no_source_fact() for role in config.models}, + served_models=served, ) async def _run_query( @@ -1249,15 +1302,16 @@ async def _run_query( client: ClaudeSDKClient, prompt: str, remaining_timeout: float, - messages: list[Message] | None = None, + collected: InvocationMessages | None = None, ) -> dict[str, Any]: """Run one SDK query and normalize its terminal result. Every message other than the terminal result is appended to - ``messages``, so the caller can read this invocation's messages. + ``collected``, which also records why the stream failed, if it did. """ - messages = [] if messages is None else messages + collected = InvocationMessages() if collected is None else collected + messages = collected.items result: ResultMessage | None = None deadline = asyncio.timeout(remaining_timeout) try: @@ -1270,6 +1324,7 @@ async def _run_query( messages.append(message) except (TimeoutError, ClaudeSDKError) as error: self._unusable = True + collected.failure = "timeout" if deadline.expired() else "no_result" await self._interrupt_failed_invocation() output = ( _failure("timeout", "Claude invocation timed out") @@ -1287,6 +1342,7 @@ async def _run_query( else: if result is None: self._unusable = True + collected.failure = "no_result" output = _failure( "claude_missing_result", "Claude returned no terminal result" ) diff --git a/tests/adapters/test_claude_adapter.py b/tests/adapters/test_claude_adapter.py index f5a176b47..7ac4ecb54 100644 --- a/tests/adapters/test_claude_adapter.py +++ b/tests/adapters/test_claude_adapter.py @@ -2227,3 +2227,141 @@ async def test_claude_without_an_init_message_reads_the_version_as_not_collected "collected_by": "adapter", "method": "harness_init_message", } + + +def wire_lines(name: str) -> list[dict[str, Any]]: + lines = (CLAUDE_STREAMS / name).read_text(encoding="utf-8").splitlines() + return [json.loads(line) for line in lines] + + +def assistant_line(model: str, parent_tool_use_id: str | None = None) -> dict[str, Any]: + """Claude Code's wire shape of an assistant message, from the saved stream.""" + + [line] = [line for line in wire_lines("normal.jsonl") if line["type"] == "assistant"] + line["message"]["model"] = model + line["parent_tool_use_id"] = parent_tool_use_id + return line + + +def parsed_stream(*middle: dict[str, Any]) -> list[Message]: + """The saved init and result lines around ``middle``, through the SDK parser.""" + + init, _, result = wire_lines("normal.jsonl") + return [parse_message(line) for line in (init, *middle, result)] + + +def served_state(state: str, reason: str) -> dict[str, str]: + return { + "state": state, + "reason": reason, + "source": "reported", + "collected_by": "adapter", + "method": "harness_api", + } + + +SERVED = "claude-served-snapshot-20991231" + + +@pytest.mark.parametrize( + ("stream", "expected"), + [ + pytest.param( + lambda: stream_messages("normal.jsonl"), + reported(SERVED, "harness_api"), + id="one-main-thread-model", + ), + pytest.param( + lambda: parsed_stream( + assistant_line("claude-a"), assistant_line("claude-b") + ), + served_state("unavailable", "not_attributed"), + id="two-main-thread-models", + ), + pytest.param( + lambda: parsed_stream( + assistant_line(SERVED), + assistant_line("claude-haiku-child", parent_tool_use_id="toolu_01"), + ), + reported(SERVED, "harness_api"), + id="main-and-child-models", + ), + pytest.param( + lambda: parsed_stream( + assistant_line(SERVED), assistant_line("") + ), + reported(SERVED, "harness_api"), + id="real-and-synthetic", + ), + pytest.param( + lambda: stream_messages("api-error.jsonl"), + served_state("unavailable", "empty_response"), + id="synthetic-only", + ), + pytest.param( + lambda: parsed_stream(), + served_state("unavailable", "empty_response"), + id="no-assistant-message", + ), + ], +) +async def test_claude_reports_the_main_thread_served_model( + claude_payload, monkeypatch, stream, expected +): + [report] = await versions_reports(claude_payload, monkeypatch, [stream()]) + + assert report["models"] == {"default": {"served": expected}} + + +async def test_claude_reports_the_served_model_under_a_sole_non_default_role( + claude_payload, monkeypatch +): + claude_payload["config"]["models"] = { + "primary": claude_payload["config"]["models"].pop("default") + } + + [report] = await versions_reports( + claude_payload, monkeypatch, [stream_messages("normal.jsonl")] + ) + + assert report["models"] == {"primary": {"served": reported(SERVED, "harness_api")}} + + +async def test_claude_served_model_is_isolated_per_invocation( + claude_payload, monkeypatch +): + reports = await versions_reports( + claude_payload, + monkeypatch, + [ + parsed_stream(assistant_line("claude-a")), + parsed_stream(assistant_line("claude-b")), + stream_messages("api-error.jsonl"), + ], + ) + + assert [report["models"]["default"]["served"] for report in reports] == [ + reported("claude-a", "harness_api"), + reported("claude-b", "harness_api"), + served_state("unavailable", "empty_response"), + ] + + +async def test_claude_served_model_reads_collection_failed_when_the_stream_fails( + claude_payload, monkeypatch +): + async def respond(_client: MagicMock) -> AsyncIterator[Message]: + yield parse_message(assistant_line(SERVED)) + raise CLIConnectionError("stream closed") + + install_fake_client(monkeypatch, respond) + start_payload = dict(claude_payload) + start_payload.pop("request") + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(start_payload)) + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + await runtime.stop() + + assert result.extensions[VERSIONS_REPORT_KEY]["models"]["default"]["served"] == ( + served_state("collection_failed", "no_result") + ) From 68b4d0fb04ebfe178bea1241638f00fd87868f7c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:59:13 -0700 Subject: [PATCH 073/159] docs: describe adapter-reported versions in what_ran The SDK guide and the integration reference said every adapter-reported fact reads key_not_declared. They now name the six adapters that report harness versions, the Claude Code served-model rule, the states a report key can produce and the tool_names cap. The adapter tutorial and the build-adapter skill show adapter authors how to declare and fill nemo_fabric.versions.v1. Signed-off-by: mnajafian-nv --- docs/adapter-contract/tutorials/results.md | 2 ++ docs/sdk/python.mdx | 35 +++++++++++++++---- skills/nemo-fabric-build-adapter/SKILL.md | 2 ++ .../references/results-and-errors.md | 8 +++-- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 7e8f6646b..1aa9fcce6 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -79,6 +79,8 @@ Report only observed invocation usage. For cumulative target counters, compute t `fabric.adapter/v1alpha2` usage is a closed object. Do not add cache-write, reasoning, peak request, or per-model fields at its root. Put that detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` and declare the extension with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized input, cached-input, output, and total token fields and can add `cache_write_input_tokens`, `reasoning_tokens`, `output_tokens_include_reasoning`, `peak_request_input_tokens`, and `models`. Within this object, `input_tokens_include_cache` applies to both cached reads and cache writes. Normalize `input_tokens` first if the provider includes only one component. Each model entry uses the same counters and requires a non-blank `model`; include `provider` when known. +To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. + NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat the detailed object as authoritative independently for each token dimension. Any of `input_tokens`, `cached_input_tokens`, `cache_write_input_tokens`, or `input_tokens_include_cache` selects the detailed input counters. Any of `output_tokens`, `reasoning_tokens`, or `output_tokens_include_reasoning` selects the detailed output counters. Consumers do not fall back to the top-level counters for a selected dimension, so the detailed and top-level views can diverge. If the selected counters do not contain enough information to determine an inclusive total, that total remains unknown even when the top-level view contains a value. **Success Check**: `invoke` returns one typed `AgentRunResult` whose explicit diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index cfb12c1ef..2e540e7bf 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1026,12 +1026,31 @@ carry `source`, `collected_by`, and `method`: | `mcp.servers.` | `origin`, `transport`, the configured `endpoint`, and a `runtime` block with the reported `connected`, `protocol_version`, `server_name`, `server_version`, and `tool_names`. | | `environment` | The environment `provider`, such as `local` or `openshell`. | -NeMo Fabric reads no adapter-reported facts yet, so every reported fact has -state `unsupported` with reason `key_not_declared` and method -`adapter_report`. - -Today every MCP entry has `origin` `plan`. An entry with `origin` `harness` -describes a server the harness reports but the plan does not configure. It has +An adapter reports its facts through the result keys `nemo_fabric.versions.v1` +(package version, harness name and version, and served models) and +`nemo_fabric.mcp.v1` (MCP `runtime` blocks) when its descriptor declares them. +The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and +mini-SWE-agent adapters report their package version and their harness name +and version. The Claude Code adapter also reports the served model of the role +it runs: the one model that this invocation's main-thread assistant messages +name, skipping subagent messages and Claude Code's `` API-error +messages. Two models give `unavailable` with reason `not_attributed`, and none +gives `unavailable` with reason `empty_response`. No bundled adapter reports +MCP runtime facts yet. + +A reported fact whose key the adapter does not declare has state `unsupported` +with reason `key_not_declared` and method `adapter_report`. A fact the adapter +has no source for has reason `no_source`. A declared key that the result omits +gives `unavailable` with reason `key_omitted`, and a fact the key omits gives +`fact_omitted`. A malformed fact, or one that does not name source `reported` +and collector `adapter`, gives `collection_failed` with reason +`invalid_report`. A key larger than 64 KiB of compact JSON gives +`collection_failed` with reason `too_large`. None of these changes the run's +status or error. + +An MCP entry has `origin` `plan` for a server the plan configures. An entry +with `origin` `harness` describes a server the harness reports but the plan +does not configure. It has the same shape, except that `endpoint` has state `not_applicable` with reason `not_configured`. A `plan` entry always has `transport`. An entry of any other origin can leave it out. @@ -1053,7 +1072,9 @@ URL or executable has state `unavailable` with reason `not_configured`. A string value longer than 256 characters keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. The rule applies to each -element of a list value too. +element of a list value too. `tool_names` keeps at most 64 names in sorted +order, and a `tool_names_truncated` marker in the `runtime` block counts the +names the adapter or NeMo Fabric dropped. `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, taken in sorted key order. An entry whose key is longer than 128 characters is dropped. diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 503448cba..15705843c 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -230,6 +230,8 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. +Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. + Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then prove another runtime cannot observe that context. Refer to the diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 3793b558a..71b4994c7 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -21,7 +21,7 @@ Every invocation that reaches the adapter boundary returns a normalized | `telemetry` | References to NVIDIA NeMo Relay or other telemetry streams from the run. | | `events` | Ordered normalized lifecycle and invocation events. | | `metadata` | Result-specific structured metadata. | -| `what_ran` | What executed for this invocation: core version, adapter ID, requested models, configured MCP servers, and environment provider, each fact with its source, or `None` when the core predates it. | +| `what_ran` | What executed for this invocation: core version, adapter ID and package version, harness name and version, requested and served models, configured MCP servers, and environment provider, each fact with its source, or `None` when the core predates it. | | `runtime_id`, `invocation_id`, `request_id` | Correlation IDs across runtimes, logs, telemetry, and artifacts. | Handle a result with this status-first pattern: @@ -51,8 +51,10 @@ Read `what_ran` as evidence of what executed, never as a grouping key. Each fact holds a `value`, or a `state` and `reason` when it has no value, plus `source`, `collected_by`, and `method`. Accept `collected_by`, `method`, and `reason` codes you do not recognize. Adapter-reported facts, such as the -harness version and the served model, currently read state `unsupported` with -reason `key_not_declared`. A malformed `what_ran` reads as `None` without +harness version and the served model, read state `unsupported` with reason +`key_not_declared` when the adapter does not report them. The Claude Code, +Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters report +the harness version, and the Claude Code adapter reports the served model. A malformed `what_ran` reads as `None` without failing the result. A `what_ran` over its 128 KiB budget can hold only `schema`, `version`, and `what_ran_truncated`, so check for that marker before reading a section. Refer to From 81044a63d0bc3c1afcf2ebfd1be55cf2d72649b1 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 01:59:14 -0700 Subject: [PATCH 074/159] docs: regenerate API references for adapter report keys Regenerated with just docs for the new report-key constants and the tool_names_truncated field. Signed-off-by: mnajafian-nv --- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 66 ++++++++++++++++++- .../runtime/enum-errorstage.mdx | 2 +- .../enum-openaichatcompletionchunkobject.mdx | 2 +- .../runtime/enum-openaistreamhost.mdx | 2 +- .../runtime/enum-openaistreamprofile.mdx | 2 +- .../enum-openaistreamprotocolversion.mdx | 2 +- .../runtime/enum-openaistreamrecord.mdx | 2 +- .../runtime/enum-runstatus.mdx | 2 +- .../runtime/enum-serviceownership.mdx | 2 +- .../runtime/fn-attach-service.mdx | 2 +- .../runtime/fn-invoke-openai-stream.mdx | 2 +- .../runtime/fn-invoke-runtime.mdx | 2 +- .../runtime/fn-prepare-service.mdx | 2 +- .../runtime/fn-release-service.mdx | 2 +- .../nemo-fabric-core/runtime/fn-run-plan.mdx | 2 +- .../runtime/fn-start-runtime-with-service.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../runtime/struct-environmenthandle.mdx | 4 +- .../runtime/struct-errorinfo.mdx | 2 +- .../runtime/struct-fabricevent.mdx | 2 +- .../runtime/struct-invocationhandle.mdx | 2 +- .../struct-openaichatcompletionchunk.mdx | 2 +- ...struct-openaichatcompletionchunkchoice.mdx | 2 +- .../struct-openaichatcompletionchunkdelta.mdx | 2 +- .../runtime/struct-openaistreaminvocation.mdx | 2 +- .../runtime/struct-openaistreamsink.mdx | 2 +- .../runtime/struct-openaistreamtransport.mdx | 2 +- .../runtime/struct-runrequest.mdx | 2 +- .../runtime/struct-runresult.mdx | 8 +-- .../runtime/struct-runtimecontext.mdx | 2 +- .../runtime/struct-runtimehandle.mdx | 2 +- .../struct-runtimetelemetrycontext.mdx | 2 +- .../runtime/struct-runusage.mdx | 2 +- .../runtime/struct-servicehandle.mdx | 2 +- .../runtime/struct-servicereference.mdx | 2 +- .../runtime/struct-telemetryref.mdx | 2 +- .../what-ran/constant-mcp-report-key.mdx | 15 +++++ .../what-ran/constant-versions-report-key.mdx | 15 +++++ .../what-ran/constant-what-ran-schema.mdx | 2 +- .../what-ran/constant-what-ran-version.mdx | 2 +- .../nemo-fabric-core/what-ran/index.mdx | 4 ++ .../what-ran/struct-mcpruntimefacts.mdx | 8 ++- 47 files changed, 151 insertions(+), 49 deletions(-) create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 52a3b6f57..1a5cbd1a3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 125 +position: 129 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 26a286a0c..b89e437a4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 126 +position: 130 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 2c89a0e39..3048bc293 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 127 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index fa7255361..595a1fd29 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 128 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index ea299ed94..b5f664a1a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 129 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 75900423a..975080702 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -497,6 +497,70 @@ Environment provider. Adapter kind. +### `UnsupportedEnvironmentRelease` + +
+ +An environment provider does not implement release. + +#### Fields + +##### `provider: String` + +Environment provider. + +### `EnvironmentProviderOperation` + +
+ +An environment provider process could not complete an operation. + +#### Fields + +##### `provider: String` + +Environment provider. + +##### `operation: String` + +Lifecycle operation. + +##### `code: String` + +Stable provider or host error code. + +##### `message: String` + +Human-readable message that excludes credentials and raw diagnostics. + +### `EnvironmentInUse` + +
+ +A runtime lifecycle operation is still active or has an uncertain outcome. + +#### Fields + +##### `environment_id: String` + +Environment whose runtime operation is unresolved. + +##### `runtime_id: String` + +Runtime identity, or `unconfirmed` if the start response was lost. + +### `EnvironmentReleaseInProgress` + +
+ +An environment release is already in progress. + +#### Fields + +##### `environment_id: String` + +Environment whose lifecycle is being released or detached. + ### `EnvironmentHandleRequired`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx index eb802320b..cdaccd776 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx @@ -2,7 +2,7 @@ title: "Enum Error Stage" sidebar-title: "ErrorStage" description: "NeMo Fabric lifecycle stage associated with an error." -position: 23 +position: 24 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx index cca22a793..83505d3b2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Chat Completion Chunk Object" sidebar-title: "OpenAiChatCompletionChunkObject" description: "Exact OpenAI object discriminator accepted by the v1 chunk profile." -position: 24 +position: 25 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx index f2d130b85..5e337a0e0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Host" sidebar-title: "OpenAiStreamHost" description: "Supported native-streaming listener host." -position: 25 +position: 26 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx index 1c3f839ae..eb3f8fe51 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Profile" sidebar-title: "OpenAiStreamProfile" description: "Supported OpenAI-compatible chunk profile." -position: 26 +position: 27 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx index 86fddd94c..ee69f77c3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Protocol Version" sidebar-title: "OpenAiStreamProtocolVersion" description: "Supported southbound native-streaming protocol version." -position: 27 +position: 28 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx index a3a933cdf..d02adc8bf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Record" sidebar-title: "OpenAiStreamRecord" description: "One correlated NDJSON record on the adapter-native stream channel." -position: 28 +position: 29 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx index 97954ce13..f7acd827c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx @@ -2,7 +2,7 @@ title: "Enum RunStatus" sidebar-title: "RunStatus" description: "Runtime completion status." -position: 29 +position: 30 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx index c5050a301..310cf8983 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx @@ -2,7 +2,7 @@ title: "Enum Service Ownership" sidebar-title: "ServiceOwnership" description: "Ownership of a service exposed through NeMo Fabric." -position: 30 +position: 31 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx index d8ff25825..494089eb6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx @@ -2,7 +2,7 @@ title: "Function attach_service" sidebar-title: "attach_service" description: "Attach to a caller-owned long-lived service for a resolved run plan." -position: 31 +position: 33 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx index b81268897..1fa28e020 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx @@ -2,7 +2,7 @@ title: "Function invoke_openai_stream" sidebar-title: "invoke_openai_stream" description: "Invoke a started harness runtime and pass through native OpenAI chat-completion chunks." -position: 32 +position: 34 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx index e1471de73..28b005c5d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx @@ -2,7 +2,7 @@ title: "Function invoke_runtime" sidebar-title: "invoke_runtime" description: "Invoke a started harness runtime." -position: 33 +position: 35 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx index a59f0067c..49e08a5bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx @@ -2,7 +2,7 @@ title: "Function prepare_service" sidebar-title: "prepare_service" description: "Prepare a Fabric-owned long-lived service for a resolved run plan." -position: 35 +position: 37 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx index b356d2db0..72ca1b83e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx @@ -2,7 +2,7 @@ title: "Function release_service" sidebar-title: "release_service" description: "Release a Fabric-owned service or detach from a caller-owned service." -position: 36 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 55a5a47cb..331c08b3c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -2,7 +2,7 @@ title: "Function run_plan" sidebar-title: "run_plan" description: "Invoke a NeMo Fabric run plan." -position: 37 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx index 547b47760..5acac20bd 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_with_service" sidebar-title: "start_runtime_with_service" description: "Start a runtime connected to an active service." -position: 39 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index 71febb037..ce8835391 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 40 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx index 55d47378d..67102fa82 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx @@ -29,11 +29,11 @@ Where NeMo Fabric control code runs. ### `workspace: Option` -Workspace visible to the harness runtime. +Workspace visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `artifacts: Option` -Artifact root visible to the harness runtime. +Artifact root visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `env: BTreeMap` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx index 9468739a0..376449e6a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx @@ -2,7 +2,7 @@ title: "Struct Error Info" sidebar-title: "ErrorInfo" description: "Normalized error metadata." -position: 5 +position: 6 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx index 5330e632a..5ec70a8ad 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx @@ -2,7 +2,7 @@ title: "Struct Fabric Event" sidebar-title: "FabricEvent" description: "NeMo Fabric lifecycle or progress event." -position: 6 +position: 7 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx index 3f3ac0d42..1df9629e6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx @@ -2,7 +2,7 @@ title: "Struct Invocation Handle" sidebar-title: "InvocationHandle" description: "One request sent to a runtime." -position: 7 +position: 8 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx index 026b2de72..96e965402 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk" sidebar-title: "OpenAiChatCompletionChunk" description: "OpenAI Chat Completions chunk accepted by the native streaming profile." -position: 8 +position: 9 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx index 94b2e70ee..f3dbf5424 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Choice" sidebar-title: "OpenAiChatCompletionChunkChoice" description: "One choice within an OpenAI Chat Completions streaming chunk." -position: 9 +position: 10 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx index 845afed45..53b0a6972 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Delta" sidebar-title: "OpenAiChatCompletionChunkDelta" description: "Incremental assistant message fields carried by one OpenAI choice." -position: 10 +position: 11 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx index 49b4c9c49..b5a3b7fcb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Invocation" sidebar-title: "OpenAiStreamInvocation" description: "One adapter-native OpenAI streaming invocation." -position: 11 +position: 12 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx index 8ad3827d5..a178c8113 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Sink" sidebar-title: "OpenAiStreamSink" description: "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric." -position: 12 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx index 2ed1e5c62..4a7f2e2f9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Transport" sidebar-title: "OpenAiStreamTransport" description: "SDK-owned loopback transport for one native OpenAI streaming invocation." -position: 13 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index 8dcc6950f..d0200756d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -2,7 +2,7 @@ title: "Struct RunRequest" sidebar-title: "RunRequest" description: "A request passed to a NeMo Fabric-managed harness runtime." -position: 14 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 9dfc3f1be..1ee527133 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -2,14 +2,14 @@ title: "Struct RunResult" sidebar-title: "RunResult" description: "Result from a NeMo Fabric-managed harness invocation." -position: 15 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
+
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Vec<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
Result from a NeMo Fabric-managed harness invocation. @@ -63,9 +63,9 @@ Normalized invocation usage when reported by the adapter target. Artifacts produced or collected by this run. -### `telemetry: Option` +### `telemetry: Vec` -Telemetry reference when available. +Ordered telemetry references when available. ### `events: Vec` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 28af565b2..625527702 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Context" sidebar-title: "RuntimeContext" description: "Context generated for one invocation of a started runtime." -position: 17 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx index b0392902d..0671b565e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Handle" sidebar-title: "RuntimeHandle" description: "Active or resumable harness runtime." -position: 18 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx index 1525ed89d..890abc95b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Telemetry Context" sidebar-title: "RuntimeTelemetryContext" description: "Runtime telemetry config passed to adapters." -position: 19 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx index 6f16e387b..864be9b27 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx @@ -2,7 +2,7 @@ title: "Struct RunUsage" sidebar-title: "RunUsage" description: "Normalized invocation usage exposed to NeMo Fabric consumers." -position: 16 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx index 8c36ff5de..dead906b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Service Handle" sidebar-title: "ServiceHandle" description: "Public identity and binding for one prepared or attached service." -position: 20 +position: 21 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx index e28440353..4a249074a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx @@ -2,7 +2,7 @@ title: "Struct Service Reference" sidebar-title: "ServiceReference" description: "Caller-supplied reference to an already-running service." -position: 21 +position: 22 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx index cd66e4025..94338baeb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Ref" sidebar-title: "TelemetryRef" description: "Reference to telemetry emitted by Relay or another configured telemetry path." -position: 22 +position: 23 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx new file mode 100644 index 000000000..612f219bf --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-mcp-report-key.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant MCP_REPORT_KEY" +sidebar-title: "MCP_REPORT_KEY" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-mcp-report-key" +description: "Report key for the `runtime` block of each MCP server, and for servers the harness reports that the plan does not configure." +position: 17 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo_fabric.mcp.v1\";"}} />
+ +Report key for the `runtime` block of each MCP server, and for servers the harness reports that the plan does not configure. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx new file mode 100644 index 000000000..b2938f6b4 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-versions-report-key.mdx @@ -0,0 +1,15 @@ +--- +title: "Constant VERSIONS_REPORT_KEY" +sidebar-title: "VERSIONS_REPORT_KEY" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-versions-report-key" +description: "Report key for the adapter package version, the harness name and version, and the served model of each role." +position: 18 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
str = \"nemo_fabric.versions.v1\";"}} />
+ +Report key for the adapter package version, the harness name and version, and the served model of each role. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx index 3b95258d3..ede3a0ea9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-schema.mdx @@ -3,7 +3,7 @@ title: "Constant WHAT_RAN_SCHEMA" sidebar-title: "WHAT_RAN_SCHEMA" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-schema" description: "Schema name written into every `WhatRan`." -position: 17 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx index f517163e1..e25ef5233 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/constant-what-ran-version.mdx @@ -3,7 +3,7 @@ title: "Constant WHAT_RAN_VERSION" sidebar-title: "WHAT_RAN_VERSION" slug: "/reference/api/rust-library-reference/nemo-fabric-core/what_ran/constant-what-ran-version" description: "Contract version written into every `WhatRan`." -position: 18 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx index b99298410..a8c31143e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -18,6 +18,8 @@ A string value longer than 256 characters, in a string fact or as an element of `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in sorted key order. An entry whose key is longer than 128 characters is dropped. While the compact JSON of the whole object is larger than 128 KiB, MCP server entries are dropped from the end of the sorted order, then model roles. Dropped entries are counted in a [`TruncatedEntries`](struct-truncatedentries.mdx) marker next to the map: `models_truncated` and `mcp.servers_truncated`. If the object is still too large with no MCP server and no model role, it is replaced by a minimal one that holds only the schema name, the version and the `what_ran_truncated` marker. +An adapter reports its own facts through the report keys [`VERSIONS_REPORT_KEY`](constant-versions-report-key.mdx) and [`MCP_REPORT_KEY`](constant-mcp-report-key.mdx) in its result's extensions, once its descriptor declares them. A reported list keeps at most 64 elements in sorted order, counted in a marker beside it, such as `tool_names_truncated`. + `what_ran` is evidence. It never decides which runs are compared. Compatibility rules for readers: @@ -51,5 +53,7 @@ Compatibility rules for readers: ## Constants +- [MCP_REPORT_KEY](constant-mcp-report-key.mdx): Report key for the `runtime` block of each MCP server, and for servers the harness reports that the plan does not configure. +- [VERSIONS_REPORT_KEY](constant-versions-report-key.mdx): Report key for the adapter package version, the harness name and version, and the served model of each role. - [WHAT_RAN_SCHEMA](constant-what-ran-schema.mdx): Schema name written into every [`WhatRan`](struct-whatran.mdx). - [WHAT_RAN_VERSION](constant-what-ran-version.mdx): Contract version written into every [`WhatRan`](struct-whatran.mdx). diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx index afa1bece5..1dca67789 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-mcpruntimefacts.mdx @@ -10,7 +10,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Fact<bool>,\n    pub protocol_version: Fact<String>,\n    pub server_name: Fact<String>,\n    pub server_version: Fact<String>,\n    pub tool_names: Fact<Vec<String>>,\n}"}} />
+
Fact<bool>,\n    pub protocol_version: Fact<String>,\n    pub server_name: Fact<String>,\n    pub server_version: Fact<String>,\n    pub tool_names: Fact<Vec<String>>,\n    pub tool_names_truncated: Option<TruncatedEntries>,\n}"}} />
What the harness learned about one MCP server when it connected. @@ -34,7 +34,11 @@ Version the server reports for itself. ### `tool_names: Fact>` -Raw tool names the server lists, each bounded by the string length rule. +Raw tool names the server lists, each bounded by the string length rule. At most 64 names, in sorted order. + +### `tool_names_truncated: Option` + +Tool names dropped from `tool_names`, by the adapter's helper or by core. Absent when none was dropped, never `null`. ## Trait Implementations From 57b64f35411976bade2fb3791ff9ad9eccb7d285 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 02:49:52 -0700 Subject: [PATCH 075/159] feat: report the package version from the other Python adapters NOOA, NOOA Bench, OpenHands and Remote Agent left adapter.package_version at key_not_declared, so a run with them could not say which adapter build ran. Each now declares nemo_fabric.versions.v1 and attaches its installed package version to every result. These adapters have no harness version or served-model source yet, so those facts read unsupported with reason no_source. The catalog is regenerated. Signed-off-by: mnajafian-nv --- .../nemo_fabric_adapters/common/provenance.py | 18 ++++ .../nooa/nooa-bench.fabric-adapter.json | 10 ++ adapters/python/nooa/nooa.fabric-adapter.json | 10 ++ .../src/nemo_fabric_adapters/nooa/adapter.py | 18 ++++ .../nooa/bench_adapter.py | 18 ++++ .../openhands/openhands.fabric-adapter.json | 10 ++ .../nemo_fabric_adapters/openhands/adapter.py | 18 ++++ .../remote-agent.fabric-adapter.json | 10 ++ .../remote_agent/adapter.py | 18 ++++ .../nemo_fabric_adapter_catalog/catalog.json | 56 ++++++++++- .../adapters/test_adapter_package_versions.py | 93 +++++++++++++++++++ .../test_adapters_common_provenance.py | 13 +++ 12 files changed, 288 insertions(+), 4 deletions(-) create mode 100644 tests/adapters/test_adapter_package_versions.py diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py index 43d41933d..b16a213b9 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py @@ -18,6 +18,7 @@ import sys import threading from collections.abc import Callable +from collections.abc import Iterable from collections.abc import Mapping from pathlib import Path from typing import Any @@ -159,6 +160,23 @@ def versions_report( } +def package_version_report( + distribution: str, model_roles: Iterable[str] +) -> dict[str, Any]: + """The report of an adapter that has a source only for its package version. + + The harness name and version and each role's served model read + ``unsupported`` with reason ``no_source``. + """ + + return versions_report( + package_version=package_version_fact(distribution), + harness_name=no_source_fact(), + harness_version=no_source_fact(), + served_models={role: no_source_fact() for role in model_roles}, + ) + + class BlockingProbe: """Run a runtime's blocking version calls off the event loop, one at a time. diff --git a/adapters/python/nooa/nooa-bench.fabric-adapter.json b/adapters/python/nooa/nooa-bench.fabric-adapter.json index 8ef2b3f71..529e391d3 100644 --- a/adapters/python/nooa/nooa-bench.fabric-adapter.json +++ b/adapters/python/nooa/nooa-bench.fabric-adapter.json @@ -35,6 +35,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/nooa/nooa.fabric-adapter.json b/adapters/python/nooa/nooa.fabric-adapter.json index 475c144c2..5b280f880 100644 --- a/adapters/python/nooa/nooa.fabric-adapter.json +++ b/adapters/python/nooa/nooa.fabric-adapter.json @@ -36,6 +36,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py index e4725af2c..670e075fd 100644 --- a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py +++ b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py @@ -26,12 +26,15 @@ from nemo_fabric_adapter_contract.models import AgentRunStatus from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance import nemo_fabric_adapters.common.utils as common_utils from nemo_fabric_adapters.nooa.model_support import build_models from nemo_fabric_adapters.nooa.model_support import close_models from nemo_fabric_adapters.nooa.telemetry import RelayReport from nemo_fabric_adapters.nooa.telemetry import RelayTelemetry +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-nooa" + ADAPTER = "python" HARNESS = "nooa" MODE = "interactive_agent" @@ -522,6 +525,7 @@ def __init__(self) -> None: self._target: InteractiveAgentTarget | None = None self._models: dict[str, Any] = {} self._telemetry: RelayTelemetry | None = None + self._model_roles: list[str] = [] async def start(self, payload: dict[str, Any]) -> None: if self._target is not None: @@ -531,6 +535,7 @@ async def start(self, payload: dict[str, Any]) -> None: ) config = _agent_config(payload) + self._model_roles = list(config.models) factory = _load_factory(_factory_ref(config)) models: dict[str, Any] = {} target: InteractiveAgentTarget | None = None @@ -586,6 +591,19 @@ async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, runtime_context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + ) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: if self._target is None or self._runtime_id is None: raise lifecycle.LifecycleError( diff --git a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py index ff103932b..a4175e94a 100644 --- a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py +++ b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py @@ -21,12 +21,15 @@ from nemo_fabric_adapter_contract.models import AgentUsage from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance import nemo_fabric_adapters.common.utils as common_utils from nemo_fabric_adapters.nooa.model_support import build_models from nemo_fabric_adapters.nooa.model_support import close_models from nemo_fabric_adapters.nooa.telemetry import RelayReport from nemo_fabric_adapters.nooa.telemetry import RelayTelemetry +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-nooa" + ADAPTER = "python" HARNESS = "nooa-bench" MODE = "bench_agent" @@ -201,6 +204,7 @@ def __init__(self) -> None: self._workspace: Path | None = None self._instruction: str | None = None self._telemetry: RelayTelemetry | None = None + self._model_roles: list[str] = [] async def start(self, payload: dict[str, Any]) -> None: if self._bench_agent_factory is not None: @@ -210,6 +214,7 @@ async def start(self, payload: dict[str, Any]) -> None: ) config = _agent_config(payload) + self._model_roles = list(config.models) role = _selected_model_role(config) models: dict[str, Any] = {} bench_agent_factory: Callable[..., Any] | None = None @@ -260,6 +265,19 @@ async def invoke( self, request: AgentRunRequest, runtime_context: RuntimeContext, + ) -> AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, runtime_context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + ) + return result + + async def _invoke( + self, + request: AgentRunRequest, + runtime_context: RuntimeContext, ) -> AgentRunResult: if ( self._bench_agent_factory is None diff --git a/adapters/python/openhands/openhands.fabric-adapter.json b/adapters/python/openhands/openhands.fabric-adapter.json index 1c3af5253..d0eb80c6e 100644 --- a/adapters/python/openhands/openhands.fabric-adapter.json +++ b/adapters/python/openhands/openhands.fabric-adapter.json @@ -39,6 +39,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py b/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py index 7ec53266a..646192bb5 100644 --- a/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py +++ b/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py @@ -16,8 +16,11 @@ from nemo_fabric_adapter_contract import models as contract from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import utils as common_utils +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-openhands" + HARNESS = "OpenHands" DEFAULT_MAX_ITERATIONS = 500 TOOL_ALIASES = { @@ -333,9 +336,11 @@ def __init__(self) -> None: self._conversation: Any = None self._profile_store_temp: tempfile.TemporaryDirectory[str] | None = None self._runtime_id: str | None = None + self._model_roles: list[str] = [] async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] + self._model_roles = list(config.models) context = contract.RuntimeContext.from_mapping(payload.get("runtime_context")) base_dir = common_utils.base_dir(payload) instruction = common_instructions.system_instruction( @@ -401,6 +406,19 @@ async def invoke( self, request: contract.AgentRunRequest, context: contract.RuntimeContext, + ) -> contract.AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + ) + return result + + async def _invoke( + self, + request: contract.AgentRunRequest, + context: contract.RuntimeContext, ) -> contract.AgentRunResult: if ( self._api is None diff --git a/adapters/python/remote-agent/remote-agent.fabric-adapter.json b/adapters/python/remote-agent/remote-agent.fabric-adapter.json index a6563914a..46e8cf458 100644 --- a/adapters/python/remote-agent/remote-agent.fabric-adapter.json +++ b/adapters/python/remote-agent/remote-agent.fabric-adapter.json @@ -69,6 +69,16 @@ "additionalProperties": false }, "requirements": {}, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "config": { "accepts": [ "models", diff --git a/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py b/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py index 89e82dd71..9425fec36 100644 --- a/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py +++ b/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py @@ -16,9 +16,12 @@ from nemo_fabric_adapter_contract import models as contract from nemo_fabric_adapters.common import instructions as common_instructions from nemo_fabric_adapters.common import lifecycle +from nemo_fabric_adapters.common import provenance from nemo_fabric_adapters.common import openai_chat from nemo_fabric_adapters.common import utils as common_utils +_PACKAGE_DISTRIBUTION = "nemo-fabric-adapters-remote-agent" + DEFAULT_API_TYPE = "openai-responses" DEFAULT_ANTHROPIC_MAX_TOKENS = 4096 @@ -127,9 +130,11 @@ def __init__(self) -> None: self._api_type = DEFAULT_API_TYPE self._messages: list[dict[str, str]] = [] self._relay_streaming = False + self._model_roles: list[str] = [] async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] + self._model_roles = list(config.models) context = contract.RuntimeContext.from_mapping(payload["runtime_context"]) settings = config.harness.settings if config.harness is not None else {} self._api_type = settings.get("api_type", DEFAULT_API_TYPE) @@ -210,6 +215,19 @@ async def invoke( self, request: contract.AgentRunRequest, context: contract.RuntimeContext, + ) -> contract.AgentRunResult: + """Run one invocation and report this adapter's package version.""" + + result = await self._invoke(request, context) + result.extensions[provenance.VERSIONS_REPORT_KEY] = ( + provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + ) + return result + + async def _invoke( + self, + request: contract.AgentRunRequest, + context: contract.RuntimeContext, ) -> contract.AgentRunResult: if self._client is None or self._config is None: raise lifecycle.LifecycleError( diff --git a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json index 14d79212e..bbcb85c6c 100644 --- a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json +++ b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json @@ -1348,6 +1348,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1421,7 +1433,7 @@ "name": "nemo-fabric-adapters-nooa", "version": "0.6.0" }, - "sha256": "80f632e09ea3279e7283a58790e53516d693162bc7742f2e932b8e7453373042", + "sha256": "c7905de4f20a376eac20061c9f5c49f6be6d8eb40c83497c919c3a53173e30fd", "source": "adapters/python/nooa/nooa.fabric-adapter.json" }, "nvidia.fabric.nooa.bench-agent": { @@ -1443,6 +1455,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1513,7 +1537,7 @@ "name": "nemo-fabric-adapters-nooa", "version": "0.6.0" }, - "sha256": "87e09e21e71d7b7bb081c357f1cbc2278a9b467ba76cc4db77d230713d126b20", + "sha256": "a70774d0926488f0a7f03bbf1a1f3d2962a939c7433795cc8cd618839b8c938e", "source": "adapters/python/nooa/nooa-bench.fabric-adapter.json" }, "nvidia.fabric.openclaw": { @@ -1874,6 +1898,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -1945,7 +1981,7 @@ "name": "nemo-fabric-adapters-openhands", "version": "0.6.0" }, - "sha256": "062ef09ba5e82dfdf0b4374632a8b39b7ff975160b0c6f62e68d2f4ac6f0b59e", + "sha256": "9f621c3c5308bce2104b7d17ff71e3c401b1823429aa9f70bac1053eaa420f21", "source": "adapters/python/openhands/openhands.fabric-adapter.json" }, "nvidia.fabric.pi": { @@ -2303,6 +2339,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, @@ -2414,7 +2462,7 @@ "name": "nemo-fabric-adapters-remote-agent", "version": "0.6.0" }, - "sha256": "39870c056b7aa15b8706b6cabf9190029f6c9165c5c9ce4b427f5f18bd7ba70a", + "sha256": "e5da937cefdfddf59720e9e04fbc22d3abd0f7418eaab99ffaeee8c0c3bc681b", "source": "adapters/python/remote-agent/remote-agent.fabric-adapter.json" } }, diff --git a/tests/adapters/test_adapter_package_versions.py b/tests/adapters/test_adapter_package_versions.py new file mode 100644 index 000000000..b3f0e339a --- /dev/null +++ b/tests/adapters/test_adapter_package_versions.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Python adapters without a harness version source still report their package version.""" + +from __future__ import annotations + +import importlib.metadata +import json +from pathlib import Path +from unittest.mock import AsyncMock +from unittest.mock import MagicMock + +import jsonschema +import pytest +from nemo_fabric_adapter_contract.models import AgentRunRequest +from nemo_fabric_adapter_contract.models import AgentRunResult +from nemo_fabric_adapter_contract.models import AgentRunStatus +from nemo_fabric_adapter_contract.models import RuntimeContext + +ROOT = Path(__file__).resolve().parents[2] +VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1" +NO_SOURCE = { + "state": "unsupported", + "reason": "no_source", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", +} + + +@pytest.mark.parametrize( + ("module", "runtime_type", "distribution", "descriptor"), + [ + ( + "nemo_fabric_adapters.nooa.adapter", + "NooaRuntime", + "nemo-fabric-adapters-nooa", + "adapters/python/nooa/nooa.fabric-adapter.json", + ), + ( + "nemo_fabric_adapters.nooa.bench_adapter", + "BenchRuntime", + "nemo-fabric-adapters-nooa", + "adapters/python/nooa/nooa-bench.fabric-adapter.json", + ), + ( + "nemo_fabric_adapters.openhands.adapter", + "OpenHandsRuntime", + "nemo-fabric-adapters-openhands", + "adapters/python/openhands/openhands.fabric-adapter.json", + ), + ( + "nemo_fabric_adapters.remote_agent.adapter", + "RemoteAgentRuntime", + "nemo-fabric-adapters-remote-agent", + "adapters/python/remote-agent/remote-agent.fabric-adapter.json", + ), + ], +) +async def test_adapter_reports_its_package_version_on_each_result( + module, runtime_type, distribution, descriptor, monkeypatch +): + runtime = getattr(pytest.importorskip(module), runtime_type)() + runtime._model_roles = ["default"] + returned = AgentRunResult( + status=AgentRunStatus.FAILED, + output={"response": None}, + error={"code": "target_failed", "message": "failed"}, + ) + monkeypatch.setattr(runtime, "_invoke", AsyncMock(return_value=returned)) + + result = await runtime.invoke( + AgentRunRequest(input="hello"), MagicMock(spec=RuntimeContext) + ) + + assert result.status is AgentRunStatus.FAILED + assert result.extensions[VERSIONS_REPORT_KEY] == { + "adapter": { + "package_version": { + "value": importlib.metadata.version(distribution), + "source": "reported", + "collected_by": "adapter", + "method": "package_metadata", + } + }, + "harness": {"name": NO_SOURCE, "version": NO_SOURCE}, + "models": {"default": {"served": NO_SOURCE}}, + } + schema = json.loads((ROOT / descriptor).read_text(encoding="utf-8")) + jsonschema.validate( + result.to_mapping()["extensions"], schema["extension_schemas"]["run_result"] + ) diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index 2b3627f09..fada78a86 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -166,3 +166,16 @@ def fail() -> str: assert await probe.call(lambda: "1.2.3") == "1.2.3" with pytest.raises(RuntimeError, match="no version"): await probe.call(fail) + + +def test_package_version_report_marks_harness_and_served_models_no_source(): + report = provenance.package_version_report( + "nemo-fabric-adapters-common", ["judge", "default"] + ) + + no_source = missing("unsupported", "no_source", "adapter_report") + assert report == { + "adapter": {"package_version": known("0.6.0")}, + "harness": {"name": no_source, "version": no_source}, + "models": {"default": {"served": no_source}, "judge": {"served": no_source}}, + } From a6a7290c4afec5c5a861f9525976b3ddc72deb1f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 02:50:00 -0700 Subject: [PATCH 076/159] feat: report the package version from the TypeScript adapters Cline, Droid, Kilo Code, OpenCode, Pi and Qwen Code left adapter.package_version at key_not_declared. A shared withPackageVersion wrapper in the TypeScript common package reads the adapter's own package.json synchronously at start and adds nemo_fabric.versions.v1 to every result, with the harness and served model facts unsupported/no_source until their sources land. Each CLI wraps its runtime, each descriptor declares the key, and the catalog is regenerated. A catalog test checks that all bundled descriptors declare the key. Signed-off-by: mnajafian-nv --- .../cline/cline.fabric-adapter.json | 1 + adapters/typescript/cline/src/cli.ts | 4 +- adapters/typescript/common/src/index.ts | 8 ++ adapters/typescript/common/src/provenance.ts | 98 +++++++++++++++++++ .../common/test/provenance.test.mjs | 72 ++++++++++++++ .../droid/droid.fabric-adapter.json | 1 + adapters/typescript/droid/src/cli.ts | 4 +- .../typescript/kilo/kilo.fabric-adapter.json | 10 ++ adapters/typescript/kilo/src/cli.ts | 4 +- .../opencode/opencode.fabric-adapter.json | 10 ++ adapters/typescript/opencode/src/cli.ts | 4 +- adapters/typescript/pi/pi.fabric-adapter.json | 1 + adapters/typescript/pi/src/cli.ts | 4 +- adapters/typescript/pi/test/runtime.test.mjs | 1 + .../typescript/qwen/qwen.fabric-adapter.json | 10 ++ adapters/typescript/qwen/src/cli.ts | 4 +- .../nemo_fabric_adapter_catalog/catalog.json | 57 +++++++++-- tests/adapters/test_opencode_adapter.py | 8 ++ tests/adapters/test_pi_adapter.py | 1 + tests/python/test_adapter_catalog.py | 14 +++ 20 files changed, 298 insertions(+), 18 deletions(-) create mode 100644 adapters/typescript/common/src/provenance.ts create mode 100644 adapters/typescript/common/test/provenance.test.mjs diff --git a/adapters/typescript/cline/cline.fabric-adapter.json b/adapters/typescript/cline/cline.fabric-adapter.json index 49c72f67c..e349f8f5d 100644 --- a/adapters/typescript/cline/cline.fabric-adapter.json +++ b/adapters/typescript/cline/cline.fabric-adapter.json @@ -42,6 +42,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "session_id": {"type": "string", "minLength": 1}, "finish_reason": { "type": "string", diff --git a/adapters/typescript/cline/src/cli.ts b/adapters/typescript/cline/src/cli.ts index 2d5a12025..79b2a3bd2 100644 --- a/adapters/typescript/cline/src/cli.ts +++ b/adapters/typescript/cline/src/cli.ts @@ -2,12 +2,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{ ClineSdkSessionFactory }, { ClineAdapterRuntime }] = await Promise.all([ import("./cline-sdk.js"), import("./runtime.js"), ]); - return new ClineAdapterRuntime(new ClineSdkSessionFactory()); + return withPackageVersion(new ClineAdapterRuntime(new ClineSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/common/src/index.ts b/adapters/typescript/common/src/index.ts index e207ca41a..94cee2995 100644 --- a/adapters/typescript/common/src/index.ts +++ b/adapters/typescript/common/src/index.ts @@ -15,3 +15,11 @@ export { } from "./lifecycle.js"; export { insecureHttpModelHosts, modelEndpointAllowed } from "./model-endpoint.js"; + +export { + VERSIONS_REPORT_KEY, + boundedText, + packageVersionFact, + packageVersionReport, + withPackageVersion, +} from "./provenance.js"; diff --git a/adapters/typescript/common/src/provenance.ts b/adapters/typescript/common/src/provenance.ts new file mode 100644 index 000000000..d0a4ef35d --- /dev/null +++ b/adapters/typescript/common/src/provenance.ts @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Reports an adapter's own package version in the `nemo_fabric.versions.v1` +// result extension. NeMo Fabric core reads the key when the adapter descriptor +// names it under `extension_schemas.run_result`, and moves it into +// `RunResult.what_ran`. + +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; + +import type { AgentRunRequest, AgentRunResult, JsonObject, RuntimeContext } from "nemo-fabric-adapter-contract"; + +import type { AdapterRuntime, AdapterStartInput } from "./lifecycle.js"; + +export const VERSIONS_REPORT_KEY = "nemo_fabric.versions.v1"; + +const MAX_VALUE_CHARS = 256; +const TRUNCATED_PREFIX_CHARS = 240; +const TRUNCATED_DIGEST_HEX_DIGITS = 15; +const MAX_MODEL_ROLES = 16; +const MAX_KEY_CHARS = 128; + +/** + * Apply NeMo Fabric core's string length rule, counting Unicode scalar values. + * A lone surrogate becomes `?` first, because core rejects a result that holds + * one. + */ +export function boundedText(text: string): string { + const chars = Array.from(text, (char) => { + const code = char.codePointAt(0) ?? 0; + return code >= 0xd800 && code <= 0xdfff ? "?" : char; + }); + if (chars.length <= MAX_VALUE_CHARS) return chars.join(""); + const digest = createHash("sha256").update(chars.join(""), "utf8").digest("hex"); + return `${chars.slice(0, TRUNCATED_PREFIX_CHARS).join("")}~${digest.slice(0, TRUNCATED_DIGEST_HEX_DIGITS)}`; +} + +function missingFact(state: string, reason: string, method = "adapter_report"): JsonObject { + return {state, reason, source: "reported", collected_by: "adapter", method}; +} + +/** + * The `version` field of the adapter's own `package.json`, read synchronously. + * A file that can't be read or parsed reads `unavailable` with reason + * `not_collected`, and an empty version reads `empty_response`. + */ +export function packageVersionFact(packageJson: URL): JsonObject { + let version: unknown; + try { + version = (JSON.parse(readFileSync(packageJson, "utf8")) as {version?: unknown}).version; + } catch { + return missingFact("unavailable", "not_collected", "package_metadata"); + } + if (typeof version !== "string" || version.length === 0) { + return missingFact("unavailable", "empty_response", "package_metadata"); + } + return {value: boundedText(version), source: "reported", collected_by: "adapter", method: "package_metadata"}; +} + +/** + * The report of an adapter that has a source only for its package version. + * The harness name and version and each role's served model read + * `unsupported` with reason `no_source`. Like core, it keeps at most 16 roles + * in sorted order and skips a role longer than 128 characters. + */ +export function packageVersionReport(packageVersion: JsonObject, modelRoles: readonly string[]): JsonObject { + const noSource = () => missingFact("unsupported", "no_source"); + const roles = modelRoles.filter((role) => Array.from(role).length <= MAX_KEY_CHARS).sort().slice(0, MAX_MODEL_ROLES); + return { + adapter: {package_version: packageVersion}, + harness: {name: noSource(), version: noSource()}, + models: Object.fromEntries(roles.map((role) => [role, {served: noSource()}])), + }; +} + +/** + * Wrap `runtime` so each result reports the adapter package version read from + * `packageJson` when the runtime starts. Results and errors are otherwise + * unchanged. + */ +export function withPackageVersion(runtime: AdapterRuntime, packageJson: URL): AdapterRuntime { + let report: JsonObject | undefined; + return { + async start(input: AdapterStartInput): Promise { + report = packageVersionReport(packageVersionFact(packageJson), Object.keys(input.config.models ?? {})); + await runtime.start(input); + }, + async invoke(request: AgentRunRequest, context: RuntimeContext): Promise { + const result = await runtime.invoke(request, context); + if (report === undefined) return result; + return {...result, extensions: {...(result.extensions ?? {}), [VERSIONS_REPORT_KEY]: report}}; + }, + stop(): Promise { + return runtime.stop(); + }, + }; +} diff --git a/adapters/typescript/common/test/provenance.test.mjs b/adapters/typescript/common/test/provenance.test.mjs new file mode 100644 index 000000000..178e305f1 --- /dev/null +++ b/adapters/typescript/common/test/provenance.test.mjs @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { pathToFileURL } from "node:url"; + +import { + VERSIONS_REPORT_KEY, + boundedText, + packageVersionFact, + withPackageVersion, +} from "../dist/index.js"; + +const noSource = {state: "unsupported", reason: "no_source", source: "reported", collected_by: "adapter", method: "adapter_report"}; + +// The same vectors as core's string length test in `what_ran.rs`. +test("bounded text matches core's string length rule", () => { + assert.equal(boundedText("é".repeat(256)), "é".repeat(256)); + assert.equal(boundedText(`${"m".repeat(300)}a`), `${"m".repeat(240)}~96858eedeb3c8e6`); + assert.equal(boundedText("é".repeat(300)), `${"é".repeat(240)}~7250b66610f8b7d`); + assert.equal(boundedText("v\ud800"), "v?"); +}); + +test("each TypeScript adapter reads its own package version", async () => { + for (const adapter of ["cline", "droid", "kilo", "opencode", "pi", "qwen"]) { + const packageJson = new URL(`../../${adapter}/package.json`, import.meta.url); + const {version} = JSON.parse(await readFile(packageJson, "utf8")); + assert.deepEqual(packageVersionFact(packageJson), { + value: version, source: "reported", collected_by: "adapter", method: "package_metadata", + }, adapter); + } +}); + +test("an unreadable package.json reads not_collected", () => { + assert.deepEqual(packageVersionFact(new URL("file:///nonexistent/package.json")), { + state: "unavailable", reason: "not_collected", source: "reported", collected_by: "adapter", method: "package_metadata", + }); +}); + +test("the wrapper adds the versions report and keeps the result", async () => { + const directory = await mkdtemp(join(tmpdir(), "fabric-provenance-")); + try { + const packageJson = pathToFileURL(join(directory, "package.json")); + await writeFile(packageJson, JSON.stringify({name: "adapter", version: "1.2.3"})); + const calls = []; + const runtime = withPackageVersion({ + async start(input) { calls.push(["start", input.agentName]); }, + async invoke() { return {status: "succeeded", output: "done", extensions: {session_id: "s-1"}}; }, + async stop() { calls.push(["stop"]); }, + }, packageJson); + + await runtime.start({agentName: "agent", baseDir: directory, config: {models: {default: {}, judge: {}}}, runtimeContext: {}}); + const result = await runtime.invoke({input: "hi"}, {}); + await runtime.stop(); + + assert.deepEqual(calls, [["start", "agent"], ["stop"]]); + assert.equal(result.status, "succeeded"); + assert.equal(result.output, "done"); + assert.equal(result.extensions.session_id, "s-1"); + assert.deepEqual(result.extensions[VERSIONS_REPORT_KEY], { + adapter: {package_version: {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}}, + harness: {name: noSource, version: noSource}, + models: {default: {served: noSource}, judge: {served: noSource}}, + }); + } finally { + await rm(directory, {recursive: true, force: true}); + } +}); diff --git a/adapters/typescript/droid/droid.fabric-adapter.json b/adapters/typescript/droid/droid.fabric-adapter.json index 0c11ce867..b037aab74 100644 --- a/adapters/typescript/droid/droid.fabric-adapter.json +++ b/adapters/typescript/droid/droid.fabric-adapter.json @@ -40,6 +40,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "session_id": {"type": "string", "minLength": 1}, "finish_reason": { "type": "string", diff --git a/adapters/typescript/droid/src/cli.ts b/adapters/typescript/droid/src/cli.ts index fe522cdac..42c063af5 100644 --- a/adapters/typescript/droid/src/cli.ts +++ b/adapters/typescript/droid/src/cli.ts @@ -2,12 +2,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{ DroidSdkSessionFactory }, { DroidAdapterRuntime }] = await Promise.all([ import("./droid-sdk.js"), import("./runtime.js"), ]); - return new DroidAdapterRuntime(new DroidSdkSessionFactory()); + return withPackageVersion(new DroidAdapterRuntime(new DroidSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/kilo/kilo.fabric-adapter.json b/adapters/typescript/kilo/kilo.fabric-adapter.json index 5ee7eb9d2..c58c8f99e 100644 --- a/adapters/typescript/kilo/kilo.fabric-adapter.json +++ b/adapters/typescript/kilo/kilo.fabric-adapter.json @@ -44,5 +44,15 @@ }, "additionalProperties": false }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "capabilities": {"streaming": false, "cancellation": false, "updates": false, "service": false} } diff --git a/adapters/typescript/kilo/src/cli.ts b/adapters/typescript/kilo/src/cli.ts index 05bb8e687..fdbc9a378 100644 --- a/adapters/typescript/kilo/src/cli.ts +++ b/adapters/typescript/kilo/src/cli.ts @@ -2,11 +2,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{KiloSdkSessionFactory}, {KiloAdapterRuntime}] = await Promise.all([ import("./kilo-sdk.js"), import("./runtime.js"), ]); - return new KiloAdapterRuntime(new KiloSdkSessionFactory()); + return withPackageVersion(new KiloAdapterRuntime(new KiloSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/opencode/opencode.fabric-adapter.json b/adapters/typescript/opencode/opencode.fabric-adapter.json index 3558508b7..c102cc824 100644 --- a/adapters/typescript/opencode/opencode.fabric-adapter.json +++ b/adapters/typescript/opencode/opencode.fabric-adapter.json @@ -62,6 +62,16 @@ }, "additionalProperties": false }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "capabilities": { "streaming": false, "cancellation": false, diff --git a/adapters/typescript/opencode/src/cli.ts b/adapters/typescript/opencode/src/cli.ts index e53dbdb6c..4fd730268 100644 --- a/adapters/typescript/opencode/src/cli.ts +++ b/adapters/typescript/opencode/src/cli.ts @@ -8,7 +8,7 @@ import { createRequire } from "node:module"; -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; import { assertSupportedBunVersion } from "./bun-version.js"; @@ -30,5 +30,5 @@ await serve(async () => { import("./opencode-sdk.js"), import("./runtime.js"), ]); - return new OpenCodeAdapterRuntime(new OpenCodeSdkSessionFactory()); + return withPackageVersion(new OpenCodeAdapterRuntime(new OpenCodeSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/pi/pi.fabric-adapter.json b/adapters/typescript/pi/pi.fabric-adapter.json index db31bde6c..5fdd440bc 100644 --- a/adapters/typescript/pi/pi.fabric-adapter.json +++ b/adapters/typescript/pi/pi.fabric-adapter.json @@ -97,6 +97,7 @@ "run_result": { "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "pi_turn_count": {"type": "integer", "minimum": 0}, "pi_turn_started": {"type": "boolean"} }, diff --git a/adapters/typescript/pi/src/cli.ts b/adapters/typescript/pi/src/cli.ts index b495c41e9..74d1d9421 100644 --- a/adapters/typescript/pi/src/cli.ts +++ b/adapters/typescript/pi/src/cli.ts @@ -8,7 +8,7 @@ import { createRequire } from "node:module"; -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; import { assertSupportedNodeVersion } from "./node-version.js"; @@ -22,5 +22,5 @@ await serve(async () => { import("./pi-sdk.js"), import("./runtime.js"), ]); - return new PiAdapterRuntime(new PiSdkSessionFactory()); + return withPackageVersion(new PiAdapterRuntime(new PiSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/adapters/typescript/pi/test/runtime.test.mjs b/adapters/typescript/pi/test/runtime.test.mjs index 1726e6201..581485bb1 100644 --- a/adapters/typescript/pi/test/runtime.test.mjs +++ b/adapters/typescript/pi/test/runtime.test.mjs @@ -42,6 +42,7 @@ test("declares the cumulative Pi turn count in every run result", async () => { assert.deepEqual(descriptor.extension_schemas.run_result, { type: "object", properties: { + "nemo_fabric.versions.v1": { type: "object" }, pi_turn_count: { type: "integer", minimum: 0 }, pi_turn_started: { type: "boolean" }, }, diff --git a/adapters/typescript/qwen/qwen.fabric-adapter.json b/adapters/typescript/qwen/qwen.fabric-adapter.json index 19ea58016..cd2c3cf97 100644 --- a/adapters/typescript/qwen/qwen.fabric-adapter.json +++ b/adapters/typescript/qwen/qwen.fabric-adapter.json @@ -44,5 +44,15 @@ }, "additionalProperties": false }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false + } + }, "capabilities": {"streaming": false, "cancellation": false, "updates": false, "service": false} } diff --git a/adapters/typescript/qwen/src/cli.ts b/adapters/typescript/qwen/src/cli.ts index 66a8d7064..8f5291ca4 100644 --- a/adapters/typescript/qwen/src/cli.ts +++ b/adapters/typescript/qwen/src/cli.ts @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; const [major, minor] = process.versions.node.split(".").map(Number); if (major === undefined || minor === undefined || major < 22 || (major === 22 && minor < 19)) { @@ -14,5 +14,5 @@ await serve(async () => { import("./qwen-sdk.js"), import("./runtime.js"), ]); - return new QwenAdapterRuntime(new QwenSdkSessionFactory()); + return withPackageVersion(new QwenAdapterRuntime(new QwenSdkSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json index bbcb85c6c..7bbfe3eac 100644 --- a/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json +++ b/sdk/python/nemo-fabric-adapter-catalog/src/nemo_fabric_adapter_catalog/catalog.json @@ -269,6 +269,9 @@ ], "type": "object" }, + "nemo_fabric.versions.v1": { + "type": "object" + }, "session_id": { "minLength": 1, "type": "string" @@ -346,7 +349,7 @@ "name": "nemo-fabric-adapters-cline", "version": "0.6.0" }, - "sha256": "4c695e93660eafc004482c1b1e5ac951c33dad22bdb3752791f184e02735e684", + "sha256": "c7e93c832ce8e88401fa2548577cb04a95b2431dd3068692a4e79211074f5646", "source": "adapters/typescript/cline/cline.fabric-adapter.json" }, "nvidia.fabric.codex": { @@ -596,6 +599,9 @@ ], "type": "string" }, + "nemo_fabric.versions.v1": { + "type": "object" + }, "session_id": { "minLength": 1, "type": "string" @@ -680,7 +686,7 @@ "name": "nemo-fabric-adapters-droid", "version": "0.6.0" }, - "sha256": "6d1b3e67bf4d7b803c292057a27aa6c1c272f1160118cf0bb349ee273b067f14", + "sha256": "5aae42b43dbe867ef43444f3796e392143e652be714de649ee75d351d1a5a65f", "source": "adapters/typescript/droid/droid.fabric-adapter.json" }, "nvidia.fabric.hermes": { @@ -842,6 +848,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "additionalProperties": false, "properties": { @@ -911,7 +929,7 @@ "name": "nemo-fabric-adapters-kilo", "version": "0.6.0" }, - "sha256": "608aeacfa0cfa5ff7ccd28e3f66cadd53f82780bc882b8577e7e8687dfd3dd07", + "sha256": "a7e4124a236408686856225f908813a3197efde6bb932a0ac2eb45e0462fe6d1", "source": "adapters/typescript/kilo/kilo.fabric-adapter.json" }, "nvidia.fabric.langchain.deepagents": { @@ -1764,6 +1782,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "additionalProperties": false, "allOf": [ @@ -1865,7 +1895,7 @@ "name": "nemo-fabric-adapters-opencode", "version": "0.6.0" }, - "sha256": "86175ea45671c92601bf1a132fc0fe91f3de82f19ce8bf4249e9993125c32641", + "sha256": "498d0c0fa0e4f835711d4990b571969e2aebf70ccf0a31fe1260a5ebc0a0d850", "source": "adapters/typescript/opencode/opencode.fabric-adapter.json" }, "nvidia.fabric.openhands": { @@ -2091,6 +2121,9 @@ "run_result": { "additionalProperties": false, "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + }, "pi_turn_count": { "minimum": 0, "type": "integer" @@ -2203,7 +2236,7 @@ "name": "nemo-fabric-adapters-pi", "version": "0.6.0" }, - "sha256": "3cb85c7a0e52e49827e889631da96ca7936c7260d52aac8b52835484a0cafad9", + "sha256": "413cb592ffe6d4724dfa0e4cd35ad8be1474acde8cd0cf33ce8073d666db0a9a", "source": "adapters/typescript/pi/pi.fabric-adapter.json" }, "nvidia.fabric.qwen": { @@ -2234,6 +2267,18 @@ ] }, "contract_version": "fabric.adapter/v1alpha2", + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "nemo_fabric.versions.v1": { + "type": "object" + } + }, + "type": "object" + } + }, "model_schema": { "additionalProperties": false, "properties": { @@ -2313,7 +2358,7 @@ "name": "nemo-fabric-adapters-qwen", "version": "0.6.0" }, - "sha256": "ffd6ed76db07ac79e04edc6cdc183729b1e569cb67015503b93aea50f951b3d2", + "sha256": "691728864183e7bffdfcfc6eb520743b14684f69a404256a9b13f239f7f6ed13", "source": "adapters/typescript/qwen/qwen.fabric-adapter.json" }, "nvidia.fabric.remote-agent": { diff --git a/tests/adapters/test_opencode_adapter.py b/tests/adapters/test_opencode_adapter.py index 109955f72..d32b97a50 100644 --- a/tests/adapters/test_opencode_adapter.py +++ b/tests/adapters/test_opencode_adapter.py @@ -143,6 +143,14 @@ def test_opencode_descriptor_declares_the_minimum_supported_surface(): }, "additionalProperties": False, }, + "extension_schemas": { + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"nemo_fabric.versions.v1": {"type": "object"}}, + "additionalProperties": False, + } + }, "capabilities": { "streaming": False, "cancellation": False, diff --git a/tests/adapters/test_pi_adapter.py b/tests/adapters/test_pi_adapter.py index c5fca8ec5..ac344918c 100644 --- a/tests/adapters/test_pi_adapter.py +++ b/tests/adapters/test_pi_adapter.py @@ -77,6 +77,7 @@ def test_pi_descriptor_declares_the_supported_surface(): assert descriptor["extension_schemas"]["run_result"] == { "type": "object", "properties": { + "nemo_fabric.versions.v1": {"type": "object"}, "pi_turn_count": {"type": "integer", "minimum": 0}, "pi_turn_started": {"type": "boolean"}, }, diff --git a/tests/python/test_adapter_catalog.py b/tests/python/test_adapter_catalog.py index 3288ddc26..50c431c57 100644 --- a/tests/python/test_adapter_catalog.py +++ b/tests/python/test_adapter_catalog.py @@ -84,3 +84,17 @@ def test_invalid_resources_fail_clearly( monkeypatch.setattr(catalog, "files", Mock(return_value=resource)) with pytest.raises(error): catalog.get_adapter_descriptor("not.a.bundled.id") + + +def test_every_bundled_descriptor_declares_the_versions_report(bundle): + adapters = bundle["adapters"] + + assert adapters + for adapter_id, entry in adapters.items(): + run_result = entry["descriptor"]["extension_schemas"]["run_result"] + assert run_result["properties"]["nemo_fabric.versions.v1"] == { + "type": "object" + }, adapter_id + assert "nemo_fabric.versions.v1" not in run_result.get("required", []), ( + adapter_id + ) From b852eacacea65056dcf55d02af3806d8e1468ca2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 02:50:09 -0700 Subject: [PATCH 077/159] docs: note that every bundled adapter reports its package version The SDK guide and the integration reference named only the six adapters with a harness version source. The TypeScript common README documents withPackageVersion. Signed-off-by: mnajafian-nv --- adapters/typescript/common/README.md | 2 ++ docs/sdk/python.mdx | 7 ++++--- .../nemo-fabric-integrate/references/results-and-errors.md | 7 ++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/adapters/typescript/common/README.md b/adapters/typescript/common/README.md index ed30c0d93..2da8912a4 100644 --- a/adapters/typescript/common/README.md +++ b/adapters/typescript/common/README.md @@ -46,3 +46,5 @@ contract. Published package manifests use the registry version instead. ## Model Endpoint Policy `insecureHttpModelHosts(settings, errorCode)` validates and normalizes the explicit `insecure_http_model_hosts` setting. `modelEndpointAllowed(url, hosts, allowPrivateHttp)` applies the shared HTTPS, loopback HTTP, and exact HTTP host policy used by OpenCode, Qwen, and Kilo. The optional private IPv4 argument preserves Kilo's existing opt-in. These utilities do not apply to MCP endpoints. + +`withPackageVersion(runtime, packageJson)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 2e540e7bf..6f5a3c383 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1029,9 +1029,10 @@ carry `source`, `collected_by`, and `method`: An adapter reports its facts through the result keys `nemo_fabric.versions.v1` (package version, harness name and version, and served models) and `nemo_fabric.mcp.v1` (MCP `runtime` blocks) when its descriptor declares them. -The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and -mini-SWE-agent adapters report their package version and their harness name -and version. The Claude Code adapter also reports the served model of the role +Every bundled adapter reports its package version. The Claude Code, Codex, +Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters also report +their harness name and version. The other adapters report those facts with +state `unsupported` and reason `no_source`. The Claude Code adapter also reports the served model of the role it runs: the one model that this invocation's main-thread assistant messages name, skipping subagent messages and Claude Code's `` API-error messages. Two models give `unavailable` with reason `not_attributed`, and none diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 71b4994c7..9fe5bd040 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -52,9 +52,10 @@ fact holds a `value`, or a `state` and `reason` when it has no value, plus `source`, `collected_by`, and `method`. Accept `collected_by`, `method`, and `reason` codes you do not recognize. Adapter-reported facts, such as the harness version and the served model, read state `unsupported` with reason -`key_not_declared` when the adapter does not report them. The Claude Code, -Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters report -the harness version, and the Claude Code adapter reports the served model. A malformed `what_ran` reads as `None` without +`key_not_declared` when the adapter does not report them. Every bundled +adapter reports its package version. The Claude Code, Codex, Hermes Agent, +OpenClaw, Deep Agents, and mini-SWE-agent adapters report the harness version, +and the Claude Code adapter reports the served model. A malformed `what_ran` reads as `None` without failing the result. A `what_ran` over its 128 KiB budget can hold only `schema`, `version`, and `what_ran_truncated`, so check for that marker before reading a section. Refer to From 98ba43b14316a47a8709c44380c394ec16f48264 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:21:00 -0700 Subject: [PATCH 078/159] fix: declare the versions report in the Harbor SWE-bench descriptors The Claude and Hermes adapters attach nemo_fabric.versions.v1 to every result, and core rejects a result extension that the descriptor's run_result schema does not declare. The Harbor SWE-bench copies of both descriptors lacked the block, so runs through them failed. A new test finds every checked-in copy of a bundled descriptor and checks that it declares the key. Signed-off-by: mnajafian-nv --- .../claude/claude.fabric-adapter.json | 8 ++++ .../hermes/hermes.fabric-adapter.json | 8 ++++ tests/python/test_adapter_catalog.py | 47 +++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json b/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json index 5af1a312f..035cb01a1 100644 --- a/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json +++ b/examples/harbor/swebench/adapters/claude/claude.fabric-adapter.json @@ -99,6 +99,14 @@ "timeout_seconds": {"type": "number", "minimum": 0} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json b/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json index 7ae06401d..f56822531 100644 --- a/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json +++ b/examples/harbor/swebench/adapters/hermes/hermes.fabric-adapter.json @@ -68,6 +68,14 @@ "provider": {"type": "string"} }, "additionalProperties": false + }, + "run_result": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nemo_fabric.versions.v1": {"type": "object"} + }, + "additionalProperties": false } }, "config": { diff --git a/tests/python/test_adapter_catalog.py b/tests/python/test_adapter_catalog.py index 50c431c57..2142ebad3 100644 --- a/tests/python/test_adapter_catalog.py +++ b/tests/python/test_adapter_catalog.py @@ -98,3 +98,50 @@ def test_every_bundled_descriptor_declares_the_versions_report(bundle): assert "nemo_fabric.versions.v1" not in run_result.get("required", []), ( adapter_id ) + + +SKIPPED_DIRECTORIES = {"node_modules", "target", "build", "dist"} + + +def _checked_in_descriptors(repo_root: Path): + for directory, subdirectories, filenames in os.walk(repo_root): + subdirectories[:] = [ + name + for name in subdirectories + if not name.startswith(".") and name not in SKIPPED_DIRECTORIES + ] + for filename in filenames: + if filename.endswith(".fabric-adapter.json"): + yield Path(directory) / filename + + +def test_every_copy_of_a_bundled_descriptor_declares_the_versions_report( + bundle, repo_root: Path +): + # Bundled adapters always attach the versions report, and core fails a run + # whose descriptor does not declare it, so copies used by examples and the + # CLI presets must declare it too. + bundled_ids = set(bundle["adapters"]) + bundled_modules = { + entry["descriptor"]["runner"]["module"] + for entry in bundle["adapters"].values() + if "module" in entry["descriptor"]["runner"] + } + sources = { + (repo_root / entry["source"]).resolve() for entry in bundle["adapters"].values() + } + + copies = [] + for path in _checked_in_descriptors(repo_root): + descriptor = json.loads(path.read_bytes()) + if ( + descriptor.get("adapter_id") in bundled_ids + or descriptor.get("runner", {}).get("module") in bundled_modules + ): + copies.append(path) + run_result = descriptor.get("extension_schemas", {}).get("run_result", {}) + assert run_result.get("properties", {}).get("nemo_fabric.versions.v1") == { + "type": "object" + }, path + + assert {path.resolve() for path in copies} - sources From 2d43065237ae3a941dc46b221aadd150b7691c68 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:28:08 -0700 Subject: [PATCH 079/159] fix: report the Hermes version that hermes --version prints display_version drops the commit and the dirty marker of a source checkout, so two different Hermes builds could report the same version. Report derived_version, which Hermes's own --version output uses. Signed-off-by: mnajafian-nv --- .../src/nemo_fabric_adapters/hermes/adapter.py | 8 +++++--- tests/adapters/test_hermes_adapter.py | 12 +++++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 0f8fe2e18..4b978dc13 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -48,13 +48,15 @@ def hermes_version() -> str: """The running Hermes Agent's version from its version call. - ``get_version_info()`` can run several ``git`` commands, so callers run it - through a :class:`provenance.BlockingProbe`. + Returns the same ``derived_version`` that ``hermes --version`` prints, which + keeps the commit and dirty marker of a source checkout. ``get_version_info()`` + can run several ``git`` commands, so callers run it through a + :class:`provenance.BlockingProbe`. """ from hermes_cli.version_info import get_version_info - return get_version_info().display_version + return get_version_info().derived_version async def probe_hermes_version( diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 420ca7018..959895ce6 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -1754,7 +1754,7 @@ async def test_hermes_reports_the_version_its_version_call_returns( report = result.extensions["nemo_fabric.versions.v1"] assert report["harness"] == { "name": _reported("hermes-agent"), - "version": _reported(get_version_info().display_version), + "version": _reported(get_version_info().derived_version), } assert report["adapter"]["package_version"]["value"] == "0.6.0" @@ -1812,3 +1812,13 @@ def blocking() -> str: assert first == second == _version_state("collection_failed", "timeout") assert calls == ["called"] + + +def test_hermes_version_keeps_the_commit_and_dirty_marker(monkeypatch): + version_info = ModuleType("hermes_cli.version_info") + version_info.get_version_info = lambda: MagicMock( # type: ignore[attr-defined] + derived_version="0.21.6+3.gabc1234.dirty", display_version="0.21.6+3" + ) + monkeypatch.setitem(sys.modules, "hermes_cli.version_info", version_info) + + assert adapter.hermes_version() == "0.21.6+3.gabc1234.dirty" From a021cdf9f63c2ecf7636c671cea10ac457de74c6 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:28:40 -0700 Subject: [PATCH 080/159] docs: keep only the API reference pages this branch changes The regenerated references also picked up changes that come from the base branch, such as new FabricError variants, the telemetry field type and page positions. Restore those pages so the branch carries only the what_ran pages for the new report keys. Signed-off-by: mnajafian-nv --- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 66 +------------------ .../runtime/enum-errorstage.mdx | 2 +- .../enum-openaichatcompletionchunkobject.mdx | 2 +- .../runtime/enum-openaistreamhost.mdx | 2 +- .../runtime/enum-openaistreamprofile.mdx | 2 +- .../enum-openaistreamprotocolversion.mdx | 2 +- .../runtime/enum-openaistreamrecord.mdx | 2 +- .../runtime/enum-runstatus.mdx | 2 +- .../runtime/enum-serviceownership.mdx | 2 +- .../runtime/fn-attach-service.mdx | 2 +- .../runtime/fn-invoke-openai-stream.mdx | 2 +- .../runtime/fn-invoke-runtime.mdx | 2 +- .../runtime/fn-prepare-service.mdx | 2 +- .../runtime/fn-release-service.mdx | 2 +- .../nemo-fabric-core/runtime/fn-run-plan.mdx | 2 +- .../runtime/fn-start-runtime-with-service.mdx | 2 +- .../runtime/fn-stop-runtime.mdx | 2 +- .../runtime/struct-environmenthandle.mdx | 4 +- .../runtime/struct-errorinfo.mdx | 2 +- .../runtime/struct-fabricevent.mdx | 2 +- .../runtime/struct-invocationhandle.mdx | 2 +- .../struct-openaichatcompletionchunk.mdx | 2 +- ...struct-openaichatcompletionchunkchoice.mdx | 2 +- .../struct-openaichatcompletionchunkdelta.mdx | 2 +- .../runtime/struct-openaistreaminvocation.mdx | 2 +- .../runtime/struct-openaistreamsink.mdx | 2 +- .../runtime/struct-openaistreamtransport.mdx | 2 +- .../runtime/struct-runrequest.mdx | 2 +- .../runtime/struct-runresult.mdx | 8 +-- .../runtime/struct-runtimecontext.mdx | 2 +- .../runtime/struct-runtimehandle.mdx | 2 +- .../struct-runtimetelemetrycontext.mdx | 2 +- .../runtime/struct-runusage.mdx | 2 +- .../runtime/struct-servicehandle.mdx | 2 +- .../runtime/struct-servicereference.mdx | 2 +- .../runtime/struct-telemetryref.mdx | 2 +- 41 files changed, 45 insertions(+), 109 deletions(-) diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 1a5cbd1a3..52a3b6f57 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 129 +position: 125 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index b89e437a4..26a286a0c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 130 +position: 126 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 3048bc293..2c89a0e39 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 131 +position: 127 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index 595a1fd29..fa7255361 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 132 +position: 128 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index b5f664a1a..ea299ed94 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 133 +position: 129 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 975080702..75900423a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -497,70 +497,6 @@ Environment provider. Adapter kind. -### `UnsupportedEnvironmentRelease` - -
- -An environment provider does not implement release. - -#### Fields - -##### `provider: String` - -Environment provider. - -### `EnvironmentProviderOperation` - -
- -An environment provider process could not complete an operation. - -#### Fields - -##### `provider: String` - -Environment provider. - -##### `operation: String` - -Lifecycle operation. - -##### `code: String` - -Stable provider or host error code. - -##### `message: String` - -Human-readable message that excludes credentials and raw diagnostics. - -### `EnvironmentInUse` - -
- -A runtime lifecycle operation is still active or has an uncertain outcome. - -#### Fields - -##### `environment_id: String` - -Environment whose runtime operation is unresolved. - -##### `runtime_id: String` - -Runtime identity, or `unconfirmed` if the start response was lost. - -### `EnvironmentReleaseInProgress` - -
- -An environment release is already in progress. - -#### Fields - -##### `environment_id: String` - -Environment whose lifecycle is being released or detached. - ### `EnvironmentHandleRequired`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx index cdaccd776..eb802320b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx @@ -2,7 +2,7 @@ title: "Enum Error Stage" sidebar-title: "ErrorStage" description: "NeMo Fabric lifecycle stage associated with an error." -position: 24 +position: 23 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx index 83505d3b2..cca22a793 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Chat Completion Chunk Object" sidebar-title: "OpenAiChatCompletionChunkObject" description: "Exact OpenAI object discriminator accepted by the v1 chunk profile." -position: 25 +position: 24 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx index 5e337a0e0..f2d130b85 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Host" sidebar-title: "OpenAiStreamHost" description: "Supported native-streaming listener host." -position: 26 +position: 25 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx index eb3f8fe51..1c3f839ae 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Profile" sidebar-title: "OpenAiStreamProfile" description: "Supported OpenAI-compatible chunk profile." -position: 27 +position: 26 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx index ee69f77c3..86fddd94c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Protocol Version" sidebar-title: "OpenAiStreamProtocolVersion" description: "Supported southbound native-streaming protocol version." -position: 28 +position: 27 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx index d02adc8bf..a3a933cdf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Record" sidebar-title: "OpenAiStreamRecord" description: "One correlated NDJSON record on the adapter-native stream channel." -position: 29 +position: 28 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx index f7acd827c..97954ce13 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx @@ -2,7 +2,7 @@ title: "Enum RunStatus" sidebar-title: "RunStatus" description: "Runtime completion status." -position: 30 +position: 29 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx index 310cf8983..c5050a301 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx @@ -2,7 +2,7 @@ title: "Enum Service Ownership" sidebar-title: "ServiceOwnership" description: "Ownership of a service exposed through NeMo Fabric." -position: 31 +position: 30 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx index 494089eb6..d8ff25825 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx @@ -2,7 +2,7 @@ title: "Function attach_service" sidebar-title: "attach_service" description: "Attach to a caller-owned long-lived service for a resolved run plan." -position: 33 +position: 31 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx index 1fa28e020..b81268897 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx @@ -2,7 +2,7 @@ title: "Function invoke_openai_stream" sidebar-title: "invoke_openai_stream" description: "Invoke a started harness runtime and pass through native OpenAI chat-completion chunks." -position: 34 +position: 32 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx index 28b005c5d..e1471de73 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx @@ -2,7 +2,7 @@ title: "Function invoke_runtime" sidebar-title: "invoke_runtime" description: "Invoke a started harness runtime." -position: 35 +position: 33 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx index 49e08a5bb..a59f0067c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx @@ -2,7 +2,7 @@ title: "Function prepare_service" sidebar-title: "prepare_service" description: "Prepare a Fabric-owned long-lived service for a resolved run plan." -position: 37 +position: 35 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx index 72ca1b83e..b356d2db0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx @@ -2,7 +2,7 @@ title: "Function release_service" sidebar-title: "release_service" description: "Release a Fabric-owned service or detach from a caller-owned service." -position: 39 +position: 36 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 331c08b3c..55a5a47cb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -2,7 +2,7 @@ title: "Function run_plan" sidebar-title: "run_plan" description: "Invoke a NeMo Fabric run plan." -position: 40 +position: 37 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx index 5acac20bd..547b47760 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_with_service" sidebar-title: "start_runtime_with_service" description: "Start a runtime connected to an active service." -position: 43 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index ce8835391..71febb037 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 44 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx index 67102fa82..55d47378d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx @@ -29,11 +29,11 @@ Where NeMo Fabric control code runs. ### `workspace: Option` -Workspace visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. +Workspace visible to the harness runtime. ### `artifacts: Option` -Artifact root visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. +Artifact root visible to the harness runtime. ### `env: BTreeMap` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx index 376449e6a..9468739a0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx @@ -2,7 +2,7 @@ title: "Struct Error Info" sidebar-title: "ErrorInfo" description: "Normalized error metadata." -position: 6 +position: 5 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx index 5ec70a8ad..5330e632a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx @@ -2,7 +2,7 @@ title: "Struct Fabric Event" sidebar-title: "FabricEvent" description: "NeMo Fabric lifecycle or progress event." -position: 7 +position: 6 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx index 1df9629e6..3f3ac0d42 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx @@ -2,7 +2,7 @@ title: "Struct Invocation Handle" sidebar-title: "InvocationHandle" description: "One request sent to a runtime." -position: 8 +position: 7 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx index 96e965402..026b2de72 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk" sidebar-title: "OpenAiChatCompletionChunk" description: "OpenAI Chat Completions chunk accepted by the native streaming profile." -position: 9 +position: 8 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx index f3dbf5424..94b2e70ee 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Choice" sidebar-title: "OpenAiChatCompletionChunkChoice" description: "One choice within an OpenAI Chat Completions streaming chunk." -position: 10 +position: 9 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx index 53b0a6972..845afed45 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Delta" sidebar-title: "OpenAiChatCompletionChunkDelta" description: "Incremental assistant message fields carried by one OpenAI choice." -position: 11 +position: 10 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx index b5a3b7fcb..49b4c9c49 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Invocation" sidebar-title: "OpenAiStreamInvocation" description: "One adapter-native OpenAI streaming invocation." -position: 12 +position: 11 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx index a178c8113..8ad3827d5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Sink" sidebar-title: "OpenAiStreamSink" description: "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric." -position: 13 +position: 12 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx index 4a7f2e2f9..2ed1e5c62 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Transport" sidebar-title: "OpenAiStreamTransport" description: "SDK-owned loopback transport for one native OpenAI streaming invocation." -position: 14 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index d0200756d..8dcc6950f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -2,7 +2,7 @@ title: "Struct RunRequest" sidebar-title: "RunRequest" description: "A request passed to a NeMo Fabric-managed harness runtime." -position: 15 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 1ee527133..9dfc3f1be 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -2,14 +2,14 @@ title: "Struct RunResult" sidebar-title: "RunResult" description: "Result from a NeMo Fabric-managed harness invocation." -position: 16 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Vec<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
+
String,\n    pub harness: String,\n    pub adapter_kind: AdapterKind,\n    pub adapter_id: Option<String>,\n    pub runtime_id: String,\n    pub invocation_id: String,\n    pub request_id: String,\n    pub status: RunStatus,\n    pub output: Value,\n    pub error: Option<ErrorInfo>,\n    pub usage: Option<RunUsage>,\n    pub artifacts: ArtifactManifest,\n    pub telemetry: Option<TelemetryRef>,\n    pub events: Vec<FabricEvent>,\n    pub metadata: BTreeMap<String, Value>,\n    pub what_ran: Option<WhatRan>,\n}"}} />
Result from a NeMo Fabric-managed harness invocation. @@ -63,9 +63,9 @@ Normalized invocation usage when reported by the adapter target. Artifacts produced or collected by this run. -### `telemetry: Vec` +### `telemetry: Option` -Ordered telemetry references when available. +Telemetry reference when available. ### `events: Vec` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 625527702..28af565b2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Context" sidebar-title: "RuntimeContext" description: "Context generated for one invocation of a started runtime." -position: 18 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx index 0671b565e..b0392902d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Handle" sidebar-title: "RuntimeHandle" description: "Active or resumable harness runtime." -position: 19 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx index 890abc95b..1525ed89d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Telemetry Context" sidebar-title: "RuntimeTelemetryContext" description: "Runtime telemetry config passed to adapters." -position: 20 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx index 864be9b27..6f16e387b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx @@ -2,7 +2,7 @@ title: "Struct RunUsage" sidebar-title: "RunUsage" description: "Normalized invocation usage exposed to NeMo Fabric consumers." -position: 17 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx index dead906b7..8c36ff5de 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Service Handle" sidebar-title: "ServiceHandle" description: "Public identity and binding for one prepared or attached service." -position: 21 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx index 4a249074a..e28440353 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx @@ -2,7 +2,7 @@ title: "Struct Service Reference" sidebar-title: "ServiceReference" description: "Caller-supplied reference to an already-running service." -position: 22 +position: 21 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx index 94338baeb..cd66e4025 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Ref" sidebar-title: "TelemetryRef" description: "Reference to telemetry emitted by Relay or another configured telemetry path." -position: 23 +position: 22 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} From 12f8f9ebd714033e0f7f699fe257be5a2a1d5a72 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:29:46 -0700 Subject: [PATCH 081/159] fix: let the version probe recover when its thread cannot start BlockingProbe marked a probe as running before starting its thread. If the start raised, nothing cleared the mark, and every later probe in the runtime timed out at once. Clear the mark and re-raise. Signed-off-by: mnajafian-nv --- .../src/nemo_fabric_adapters/common/provenance.py | 10 +++++++++- tests/adapters/test_adapters_common_provenance.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py index b16a213b9..29095edd5 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py @@ -227,7 +227,15 @@ def run() -> None: return self._running.set() - threading.Thread(target=run, name="nemo-fabric-version-probe", daemon=True).start() + try: + threading.Thread( + target=run, name="nemo-fabric-version-probe", daemon=True + ).start() + except Exception: + # No thread runs to clear the flag, so clear it here or every later + # call would time out at once. + self._running.clear() + raise # On timeout, wait_for cancels only this future. The thread keeps # running, and its late outcome is dropped. return await asyncio.wait_for(outcome, timeout) diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index fada78a86..3e9f00453 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -168,6 +168,20 @@ def fail() -> str: await probe.call(fail) +async def test_blocking_probe_recovers_when_its_thread_cannot_start(monkeypatch): + probe = provenance.BlockingProbe() + + def refuse_to_start(_thread: threading.Thread) -> None: + raise RuntimeError("can't start new thread") + + with monkeypatch.context() as patch: + patch.setattr(threading.Thread, "start", refuse_to_start) + with pytest.raises(RuntimeError, match="can't start new thread"): + await probe.call(lambda: "1.0.0", timeout=1) + + assert await probe.call(lambda: "2.0.0", timeout=1) == "2.0.0" + + def test_package_version_report_marks_harness_and_served_models_no_source(): report = provenance.package_version_report( "nemo-fabric-adapters-common", ["judge", "default"] From bd48fc02be0c5d18884313f092dfb8d58fc58a74 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:35:17 -0700 Subject: [PATCH 082/159] fix: report hard-coded harness names with the adapter_report method Each adapter knows its harness name as a constant, so no collection technique stands behind it. The names reused the version's method, such as harness_init_message or package_metadata, which claimed a source the name never came from. Report them with adapter_report, as the design requires for an adapter fact with no technique behind it. Signed-off-by: mnajafian-nv --- .../nemo_fabric_adapters/claude/adapter.py | 4 +++- .../src/nemo_fabric_adapters/codex/adapter.py | 2 +- .../deepagents/adapter.py | 2 +- .../nemo_fabric_adapters/hermes/adapter.py | 2 +- .../mini_swe_agent/adapter.py | 2 +- .../nemo_fabric_adapters/openclaw/adapter.py | 2 +- tests/adapters/test_claude_adapter.py | 2 +- tests/adapters/test_codex_adapter.py | 19 +++++++++++++++++++ tests/adapters/test_hermes_adapter.py | 2 +- tests/adapters/test_openclaw.py | 2 +- tests/e2e/test_deepagents.py | 2 +- tests/e2e/test_mini_swe_agent.py | 2 +- 12 files changed, 32 insertions(+), 11 deletions(-) diff --git a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py index 2a1f68480..c334b3427 100644 --- a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py +++ b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py @@ -1292,7 +1292,9 @@ def _versions_report( served[_selected_model_role(config)] = served_model_fact(messages) return provenance.versions_report( package_version=self._package_version, - harness_name=provenance.known_fact(_HARNESS_NAME, _METHOD_INIT_MESSAGE), + harness_name=provenance.known_fact( + _HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT + ), harness_version=self._harness_version, served_models=served, ) diff --git a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py index be0143a43..87868c54d 100644 --- a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py +++ b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py @@ -1114,7 +1114,7 @@ def versions_report(config: AgentConfig, harness_version: dict[str, Any]) -> dic return provenance.versions_report( package_version=provenance.package_version_fact(_PACKAGE_DISTRIBUTION), - harness_name=provenance.known_fact(_HARNESS_NAME, _METHOD_HARNESS_API), + harness_name=provenance.known_fact(_HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT), harness_version=harness_version, served_models={role: provenance.no_source_fact() for role in config.models}, ) diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 721e31221..65774bc6e 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -817,7 +817,7 @@ async def invoke( _PACKAGE_DISTRIBUTION ), harness_name=provenance.known_fact( - _HARNESS_DISTRIBUTION, provenance.METHOD_PACKAGE_METADATA + _HARNESS_DISTRIBUTION, provenance.METHOD_ADAPTER_REPORT ), harness_version=self._harness_version, served_models={ diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 4b978dc13..0bbc07589 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -433,7 +433,7 @@ def clear_active_invoke_task( _PACKAGE_DISTRIBUTION ), harness_name=provenance.known_fact( - _HARNESS_NAME, _METHOD_HARNESS_API + _HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT ), harness_version=self._harness_version, served_models={ diff --git a/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py b/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py index 148e924d0..57b6c0c9a 100644 --- a/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py +++ b/adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py @@ -197,7 +197,7 @@ async def invoke( _PACKAGE_DISTRIBUTION ), harness_name=provenance.known_fact( - _HARNESS_DISTRIBUTION, provenance.METHOD_PACKAGE_METADATA + _HARNESS_DISTRIBUTION, provenance.METHOD_ADAPTER_REPORT ), harness_version=self._harness_version, served_models={ diff --git a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py index 719bf2abb..1b1ddf207 100644 --- a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py +++ b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py @@ -1413,7 +1413,7 @@ def _versions_extensions(self) -> dict[str, Any]: provenance.VERSIONS_REPORT_KEY: provenance.versions_report( package_version=provenance.package_version_fact(_PACKAGE_DISTRIBUTION), harness_name=provenance.known_fact( - _HARNESS_NAME, self._harness_version["method"] + _HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT ), harness_version=self._harness_version, served_models={role: provenance.no_source_fact() for role in models}, diff --git a/tests/adapters/test_claude_adapter.py b/tests/adapters/test_claude_adapter.py index 7ac4ecb54..cd40aa1dd 100644 --- a/tests/adapters/test_claude_adapter.py +++ b/tests/adapters/test_claude_adapter.py @@ -2187,7 +2187,7 @@ async def test_claude_reports_the_cli_version_from_its_init_message( "package_version": reported(package_version, "package_metadata") } assert report["harness"] == { - "name": reported("claude-code", "harness_init_message"), + "name": reported("claude-code", "adapter_report"), "version": reported("2.1.294", "harness_init_message"), } diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index 144309aa1..01ad47577 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -2165,3 +2165,22 @@ async def test_codex_cli_version_read_degrades_to_a_state(read, expected): assert await adapter.read_cli_version(thread, timeout=0.1) == expected assert time.monotonic() - started < 1 + + +def test_codex_versions_report_names_the_harness_as_an_adapter_report(): + config = AgentConfig.from_mapping( + {"models": {"default": {"provider": "openai", "model": "gpt-test"}}} + ) + harness_version = _version_fact("unavailable", "not_collected") + + report = adapter.versions_report(config, harness_version) + + assert report["harness"] == { + "name": { + "value": "codex", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + }, + "version": harness_version, + } diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 959895ce6..6b9892e73 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -1753,7 +1753,7 @@ async def test_hermes_reports_the_version_its_version_call_returns( report = result.extensions["nemo_fabric.versions.v1"] assert report["harness"] == { - "name": _reported("hermes-agent"), + "name": {**_reported("hermes-agent"), "method": "adapter_report"}, "version": _reported(get_version_info().derived_version), } assert report["adapter"]["package_version"]["value"] == "0.6.0" diff --git a/tests/adapters/test_openclaw.py b/tests/adapters/test_openclaw.py index 9c31c94c4..bc1ebe23f 100644 --- a/tests/adapters/test_openclaw.py +++ b/tests/adapters/test_openclaw.py @@ -293,7 +293,7 @@ async def test_openclaw_runtime_generates_config_invokes_and_cleans_up( "value": "openclaw", "source": "reported", "collected_by": "adapter", - "method": "harness_probe", + "method": "adapter_report", }, "version": { "value": "2026.9.4", diff --git a/tests/e2e/test_deepagents.py b/tests/e2e/test_deepagents.py index ce96f6511..81ea30aee 100644 --- a/tests/e2e/test_deepagents.py +++ b/tests/e2e/test_deepagents.py @@ -57,7 +57,7 @@ async def test_deepagents_persistent_host_with_mock_model(api_server, tmp_path): # Core read the adapter's versions report into what_ran. for result in (first, second): assert result.what_ran["harness"] == { - "name": _reported("deepagents"), + "name": {**_reported("deepagents"), "method": "adapter_report"}, "version": _reported(importlib.metadata.version("deepagents")), }, results assert result.what_ran["adapter"]["package_version"] == _reported( diff --git a/tests/e2e/test_mini_swe_agent.py b/tests/e2e/test_mini_swe_agent.py index f557de688..cfac06e35 100644 --- a/tests/e2e/test_mini_swe_agent.py +++ b/tests/e2e/test_mini_swe_agent.py @@ -95,7 +95,7 @@ async def test_mini_swe_agent_plans_diagnoses_and_runs( assert result.output["usage"]["api_calls"] == 1 what_ran = result.what_ran assert what_ran["harness"] == { - "name": _reported("mini-swe-agent"), + "name": {**_reported("mini-swe-agent"), "method": "adapter_report"}, "version": _reported(importlib.metadata.version("mini-swe-agent")), } assert what_ran["adapter"]["package_version"] == _reported( From bc4d30fbd1095fd1c02e03ea781d4924d0ef1da5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:37:50 -0700 Subject: [PATCH 083/159] feat: report Cline's served model as not attributed The Cline SDK's result names a model, but nothing yet shows that it is the model that served the invocation, so the design reads Cline's served model as unavailable with reason not_attributed rather than unsupported. packageVersionReport and withPackageVersion take an optional served-model fact for one role, and the Cline entry point passes it for the role Cline runs. Signed-off-by: mnajafian-nv --- adapters/typescript/cline/src/cli.ts | 17 ++++++-- adapters/typescript/cline/src/cline-sdk.ts | 16 +++++-- .../typescript/cline/test/cline-sdk.test.mjs | 10 ++++- adapters/typescript/common/README.md | 2 +- adapters/typescript/common/src/index.ts | 2 + adapters/typescript/common/src/provenance.ts | 43 ++++++++++++++----- .../common/test/provenance.test.mjs | 37 ++++++++++++++++ docs/sdk/python.mdx | 5 ++- .../references/results-and-errors.md | 3 +- 9 files changed, 115 insertions(+), 20 deletions(-) diff --git a/adapters/typescript/cline/src/cli.ts b/adapters/typescript/cline/src/cli.ts index 79b2a3bd2..60f1237a8 100644 --- a/adapters/typescript/cline/src/cli.ts +++ b/adapters/typescript/cline/src/cli.ts @@ -2,12 +2,23 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; +import { missingFact, serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { - const [{ ClineSdkSessionFactory }, { ClineAdapterRuntime }] = await Promise.all([ + const [{ ClineSdkSessionFactory, selectedModelRole }, { ClineAdapterRuntime }] = await Promise.all([ import("./cline-sdk.js"), import("./runtime.js"), ]); - return withPackageVersion(new ClineAdapterRuntime(new ClineSdkSessionFactory()), new URL("../package.json", import.meta.url)); + // The SDK's result names a model, but nothing yet shows that it is the model + // that served this invocation, so the selected role reads not_attributed. + return withPackageVersion( + new ClineAdapterRuntime(new ClineSdkSessionFactory()), + new URL("../package.json", import.meta.url), + (input) => { + const role = selectedModelRole(input.config); + return role === undefined + ? undefined + : { role, fact: missingFact("unavailable", "not_attributed", "harness_api") }; + }, + ); }); diff --git a/adapters/typescript/cline/src/cline-sdk.ts b/adapters/typescript/cline/src/cline-sdk.ts index 5c8ca5e5f..e9a99e14d 100644 --- a/adapters/typescript/cline/src/cline-sdk.ts +++ b/adapters/typescript/cline/src/cline-sdk.ts @@ -134,12 +134,22 @@ async function loadClineSdk(): Promise { return sdk; } +/** + * The model role Cline runs: `default`, or the only configured role. Returns + * `undefined` when no role or several roles without `default` are configured. + */ +export function selectedModelRole(config: AgentConfig): string | undefined { + const roles = Object.keys(config.models ?? {}); + if (roles.includes("default")) return "default"; + return roles.length === 1 ? roles[0] : undefined; +} + function selectModel(config: AgentConfig): AgentModelConfig { - const entries = Object.entries(config.models ?? {}); - if (entries.length === 0) { + if (Object.keys(config.models ?? {}).length === 0) { throw new LifecycleError("cline_model_required", "The Cline adapter requires one configured model"); } - const selected = config.models?.default ?? (entries.length === 1 ? entries[0]?.[1] : undefined); + const role = selectedModelRole(config); + const selected = role === undefined ? undefined : config.models?.[role]; if (selected === undefined) { throw new LifecycleError( "cline_model_ambiguous", diff --git a/adapters/typescript/cline/test/cline-sdk.test.mjs b/adapters/typescript/cline/test/cline-sdk.test.mjs index 2aac26163..b5a44a327 100644 --- a/adapters/typescript/cline/test/cline-sdk.test.mjs +++ b/adapters/typescript/cline/test/cline-sdk.test.mjs @@ -7,7 +7,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import test from "node:test"; -import { ClineSdkSessionFactory } from "../dist/cline-sdk.js"; +import { ClineSdkSessionFactory, selectedModelRole } from "../dist/cline-sdk.js"; function input(overrides = {}) { return { @@ -444,3 +444,11 @@ test("rejects unsupported MCP authentication and tool filters", async () => { (error) => error.code === "cline_mcp_blocklist_unsupported", ); }); + +test("the selected model role is default, or the only configured role", () => { + assert.equal(selectedModelRole({ models: { default: {}, judge: {} } }), "default"); + assert.equal(selectedModelRole({ models: { primary: {} } }), "primary"); + assert.equal(selectedModelRole({ models: { primary: {}, judge: {} } }), undefined); + assert.equal(selectedModelRole({ models: {} }), undefined); + assert.equal(selectedModelRole({}), undefined); +}); diff --git a/adapters/typescript/common/README.md b/adapters/typescript/common/README.md index 2da8912a4..700ae129c 100644 --- a/adapters/typescript/common/README.md +++ b/adapters/typescript/common/README.md @@ -47,4 +47,4 @@ contract. Published package manifests use the registry version instead. `insecureHttpModelHosts(settings, errorCode)` validates and normalizes the explicit `insecure_http_model_hosts` setting. `modelEndpointAllowed(url, hosts, allowPrivateHttp)` applies the shared HTTPS, loopback HTTP, and exact HTTP host policy used by OpenCode, Qwen, and Kilo. The optional private IPv4 argument preserves Kilo's existing opt-in. These utilities do not apply to MCP endpoints. -`withPackageVersion(runtime, packageJson)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. +`withPackageVersion(runtime, packageJson)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. The optional third argument, `servedModel(input)`, returns the `role` and `fact` of one role's served model from the start input, such as `missingFact("unavailable", "not_attributed", "harness_api")`; every other role keeps `no_source`. diff --git a/adapters/typescript/common/src/index.ts b/adapters/typescript/common/src/index.ts index 94cee2995..7d8d3f909 100644 --- a/adapters/typescript/common/src/index.ts +++ b/adapters/typescript/common/src/index.ts @@ -19,7 +19,9 @@ export { insecureHttpModelHosts, modelEndpointAllowed } from "./model-endpoint.j export { VERSIONS_REPORT_KEY, boundedText, + missingFact, packageVersionFact, packageVersionReport, withPackageVersion, + type ServedModelFact, } from "./provenance.js"; diff --git a/adapters/typescript/common/src/provenance.ts b/adapters/typescript/common/src/provenance.ts index d0a4ef35d..8e084519d 100644 --- a/adapters/typescript/common/src/provenance.ts +++ b/adapters/typescript/common/src/provenance.ts @@ -36,10 +36,17 @@ export function boundedText(text: string): string { return `${chars.slice(0, TRUNCATED_PREFIX_CHARS).join("")}~${digest.slice(0, TRUNCATED_DIGEST_HEX_DIGITS)}`; } -function missingFact(state: string, reason: string, method = "adapter_report"): JsonObject { +/** A reported fact without a value. */ +export function missingFact(state: string, reason: string, method = "adapter_report"): JsonObject { return {state, reason, source: "reported", collected_by: "adapter", method}; } +/** The served-model fact an adapter reports for one model role. */ +export interface ServedModelFact { + role: string; + fact: JsonObject; +} + /** * The `version` field of the adapter's own `package.json`, read synchronously. * A file that can't be read or parsed reads `unavailable` with reason @@ -60,30 +67,46 @@ export function packageVersionFact(packageJson: URL): JsonObject { /** * The report of an adapter that has a source only for its package version. - * The harness name and version and each role's served model read - * `unsupported` with reason `no_source`. Like core, it keeps at most 16 roles - * in sorted order and skips a role longer than 128 characters. + * The harness name and version read `unsupported` with reason `no_source`, and + * so does each role's served model, except the role that `servedModel` names, + * which gets its fact. Like core, it keeps at most 16 roles in sorted order + * and skips a role longer than 128 characters. */ -export function packageVersionReport(packageVersion: JsonObject, modelRoles: readonly string[]): JsonObject { +export function packageVersionReport( + packageVersion: JsonObject, + modelRoles: readonly string[], + servedModel?: ServedModelFact, +): JsonObject { const noSource = () => missingFact("unsupported", "no_source"); const roles = modelRoles.filter((role) => Array.from(role).length <= MAX_KEY_CHARS).sort().slice(0, MAX_MODEL_ROLES); return { adapter: {package_version: packageVersion}, harness: {name: noSource(), version: noSource()}, - models: Object.fromEntries(roles.map((role) => [role, {served: noSource()}])), + models: Object.fromEntries( + roles.map((role) => [role, {served: role === servedModel?.role ? {...servedModel.fact} : noSource()}]), + ), }; } /** * Wrap `runtime` so each result reports the adapter package version read from - * `packageJson` when the runtime starts. Results and errors are otherwise - * unchanged. + * `packageJson` when the runtime starts. `servedModel`, when given, picks the + * served-model fact of one role from the start input. Results and errors are + * otherwise unchanged. */ -export function withPackageVersion(runtime: AdapterRuntime, packageJson: URL): AdapterRuntime { +export function withPackageVersion( + runtime: AdapterRuntime, + packageJson: URL, + servedModel?: (input: AdapterStartInput) => ServedModelFact | undefined, +): AdapterRuntime { let report: JsonObject | undefined; return { async start(input: AdapterStartInput): Promise { - report = packageVersionReport(packageVersionFact(packageJson), Object.keys(input.config.models ?? {})); + report = packageVersionReport( + packageVersionFact(packageJson), + Object.keys(input.config.models ?? {}), + servedModel?.(input), + ); await runtime.start(input); }, async invoke(request: AgentRunRequest, context: RuntimeContext): Promise { diff --git a/adapters/typescript/common/test/provenance.test.mjs b/adapters/typescript/common/test/provenance.test.mjs index 178e305f1..9746b3a97 100644 --- a/adapters/typescript/common/test/provenance.test.mjs +++ b/adapters/typescript/common/test/provenance.test.mjs @@ -11,7 +11,9 @@ import { pathToFileURL } from "node:url"; import { VERSIONS_REPORT_KEY, boundedText, + missingFact, packageVersionFact, + packageVersionReport, withPackageVersion, } from "../dist/index.js"; @@ -70,3 +72,38 @@ test("the wrapper adds the versions report and keeps the result", async () => { await rm(directory, {recursive: true, force: true}); } }); + +test("a served-model override reaches only the role it names", () => { + const packageVersion = {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}; + const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); + + assert.deepEqual(packageVersionReport(packageVersion, ["judge", "default"], {role: "default", fact: notAttributed}).models, { + default: {served: notAttributed}, + judge: {served: noSource}, + }); + // A role that is not configured adds no entry. + assert.deepEqual(packageVersionReport(packageVersion, ["default"], {role: "absent", fact: notAttributed}).models, { + default: {served: noSource}, + }); +}); + +test("the wrapper picks the served-model role from the start input", async () => { + const directory = await mkdtemp(join(tmpdir(), "fabric-provenance-")); + try { + const packageJson = pathToFileURL(join(directory, "package.json")); + await writeFile(packageJson, JSON.stringify({name: "adapter", version: "1.2.3"})); + const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); + const runtime = withPackageVersion({ + async start() {}, + async invoke() { return {status: "succeeded", output: "done"}; }, + async stop() {}, + }, packageJson, (input) => ({role: Object.keys(input.config.models)[0], fact: notAttributed})); + + await runtime.start({agentName: "agent", baseDir: directory, config: {models: {primary: {}}}, runtimeContext: {}}); + const result = await runtime.invoke({input: "hi"}, {}); + + assert.deepEqual(result.extensions[VERSIONS_REPORT_KEY].models, {primary: {served: notAttributed}}); + } finally { + await rm(directory, {recursive: true, force: true}); + } +}); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 6f5a3c383..f181293e2 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1036,7 +1036,10 @@ state `unsupported` and reason `no_source`. The Claude Code adapter also reports it runs: the one model that this invocation's main-thread assistant messages name, skipping subagent messages and Claude Code's `` API-error messages. Two models give `unavailable` with reason `not_attributed`, and none -gives `unavailable` with reason `empty_response`. No bundled adapter reports +gives `unavailable` with reason `empty_response`. The Cline adapter reports the +served model of the role it runs as `unavailable` with reason `not_attributed`, +because nothing yet shows that the model its SDK names served this invocation. +No bundled adapter reports MCP runtime facts yet. A reported fact whose key the adapter does not declare has state `unsupported` diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 9fe5bd040..08979aabf 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -55,7 +55,8 @@ harness version and the served model, read state `unsupported` with reason `key_not_declared` when the adapter does not report them. Every bundled adapter reports its package version. The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters report the harness version, -and the Claude Code adapter reports the served model. A malformed `what_ran` reads as `None` without +and the Claude Code adapter reports the served model. The Cline adapter reports +the served model as `unavailable` with reason `not_attributed`. A malformed `what_ran` reads as `None` without failing the result. A `what_ran` over its 128 KiB budget can hold only `schema`, `version`, and `what_ran_truncated`, so check for that marker before reading a section. Refer to From ea23194210898ada982b2711ff545a4102dd1ed3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 03:38:02 -0700 Subject: [PATCH 084/159] docs: narrow when an adapter fact reads key_not_declared The integrate skill said key_not_declared applies whenever an adapter does not report a fact. Core uses it only when the descriptor does not declare the report key; a declared key that the result omits reads key_omitted, and a declared key that omits the fact reads fact_omitted. Signed-off-by: mnajafian-nv --- .../nemo-fabric-integrate/references/results-and-errors.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 08979aabf..6abc28ea5 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -52,7 +52,10 @@ fact holds a `value`, or a `state` and `reason` when it has no value, plus `source`, `collected_by`, and `method`. Accept `collected_by`, `method`, and `reason` codes you do not recognize. Adapter-reported facts, such as the harness version and the served model, read state `unsupported` with reason -`key_not_declared` when the adapter does not report them. Every bundled +`key_not_declared` when the adapter descriptor does not declare their report +key. When the descriptor declares the key, a result that omits the key reads +`unavailable` with reason `key_omitted`, and a key that omits the fact reads +`unavailable` with reason `fact_omitted`. Every bundled adapter reports its package version. The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters report the harness version, and the Claude Code adapter reports the served model. The Cline adapter reports From fbddfd8b169217fb36c51a19e3b925c97b2ca955 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 04:54:35 -0700 Subject: [PATCH 085/159] fix: read no_source for Hermes only when its version call is missing The probe mapped any ImportError to no_source, including one raised inside get_version_info(), and labeled the fact harness_api. Only a failed import of the version call now reads no_source, through the shared no_source fact with method adapter_report. An ImportError inside the call reads collection_failed with reason no_result like any other error. Signed-off-by: mnajafian-nv --- .../nemo_fabric_adapters/hermes/adapter.py | 18 +++++-- tests/adapters/test_hermes_adapter.py | 50 +++++++++++++++---- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 0bbc07589..b051b2138 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -45,16 +45,25 @@ hermes_mcp_server_config = configuration.hermes_mcp_server_config +class HermesVersionCallMissingError(Exception): + """The installed Hermes Agent predates its version call.""" + + def hermes_version() -> str: """The running Hermes Agent's version from its version call. Returns the same ``derived_version`` that ``hermes --version`` prints, which keeps the commit and dirty marker of a source checkout. ``get_version_info()`` can run several ``git`` commands, so callers run it through a - :class:`provenance.BlockingProbe`. + :class:`provenance.BlockingProbe`. Raises + :class:`HermesVersionCallMissingError` only when the version call itself + can't be imported; an error raised inside the call propagates unchanged. """ - from hermes_cli.version_info import get_version_info + try: + from hermes_cli.version_info import get_version_info + except ImportError as error: + raise HermesVersionCallMissingError from error return get_version_info().derived_version @@ -70,9 +79,8 @@ async def probe_hermes_version( version = await probe.call(version_call, timeout=timeout) except TimeoutError: return provenance.missing_fact("collection_failed", "timeout", _METHOD_HARNESS_API) - except ImportError: - # A Hermes Agent older than its version call. - return provenance.missing_fact("unsupported", "no_source", _METHOD_HARNESS_API) + except HermesVersionCallMissingError: + return provenance.no_source_fact() except Exception: # Provenance never fails a run, whatever Hermes's version call raises. LOGGER.exception("Hermes version call failed") diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 6b9892e73..04bfe3336 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -1759,10 +1759,6 @@ async def test_hermes_reports_the_version_its_version_call_returns( assert report["adapter"]["package_version"]["value"] == "0.6.0" -def _raise_import_error() -> str: - raise ImportError("hermes_cli.version_info") - - def _raise_runtime_error() -> str: raise RuntimeError("malformed install stamp") @@ -1776,11 +1772,6 @@ def _raise_runtime_error() -> str: _version_state("unavailable", "empty_response"), id="unknown", ), - pytest.param( - _raise_import_error, - _version_state("unsupported", "no_source"), - id="no-version-call", - ), pytest.param( _raise_runtime_error, _version_state("collection_failed", "no_result"), @@ -1794,6 +1785,47 @@ async def test_hermes_version_probe_maps_each_outcome(version_call, expected): assert await adapter.probe_hermes_version(probe, version_call) == expected +def _version_module_raising_import_error() -> ModuleType: + version_info = ModuleType("hermes_cli.version_info") + + def get_version_info() -> object: + raise ImportError("a module the version call needs") + + version_info.get_version_info = get_version_info # type: ignore[attr-defined] + return version_info + + +@pytest.mark.parametrize( + ("version_module", "expected"), + [ + # None in sys.modules makes the import itself raise ImportError. + pytest.param( + None, + { + "state": "unsupported", + "reason": "no_source", + "source": "reported", + "collected_by": "adapter", + "method": "adapter_report", + }, + id="no-version-call", + ), + pytest.param( + _version_module_raising_import_error(), + _version_state("collection_failed", "no_result"), + id="import-error-inside-the-call", + ), + ], +) +async def test_hermes_version_probe_reads_no_source_only_without_a_version_call( + monkeypatch, version_module, expected +): + monkeypatch.setitem(sys.modules, "hermes_cli.version_info", version_module) + probe = adapter.provenance.BlockingProbe() + + assert await adapter.probe_hermes_version(probe) == expected + + async def test_hermes_version_probe_times_out_and_starts_no_second_call(): release = threading.Event() calls: list[str] = [] From 0dca1d2df257a5d8295dd68c5640660d6011df17 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 04:56:09 -0700 Subject: [PATCH 086/159] test: require descriptor copies to match their bundled source byte for byte The copy check read every file on disk, skipped build, dist and target folders, and looked only for the versions key. It now lists tracked descriptors with git ls-files, walking the tree only outside a git checkout, matches copies by adapter ID, runner module or runner script, and requires each to equal its source byte for byte, as the CLI's packaged-descriptor test already does for its own copies. Signed-off-by: mnajafian-nv --- tests/python/test_adapter_catalog.py | 90 ++++++++++++++++++---------- 1 file changed, 59 insertions(+), 31 deletions(-) diff --git a/tests/python/test_adapter_catalog.py b/tests/python/test_adapter_catalog.py index 2142ebad3..3b464fefe 100644 --- a/tests/python/test_adapter_catalog.py +++ b/tests/python/test_adapter_catalog.py @@ -5,6 +5,7 @@ import json import os +import subprocess from importlib.resources import files from pathlib import Path from unittest.mock import Mock @@ -100,48 +101,75 @@ def test_every_bundled_descriptor_declares_the_versions_report(bundle): ) -SKIPPED_DIRECTORIES = {"node_modules", "target", "build", "dist"} - - -def _checked_in_descriptors(repo_root: Path): +def _descriptors_on_disk(repo_root: Path) -> list[Path]: + found = [] for directory, subdirectories, filenames in os.walk(repo_root): + # Installed dependencies are not part of the source tree. subdirectories[:] = [ - name - for name in subdirectories - if not name.startswith(".") and name not in SKIPPED_DIRECTORIES + name for name in subdirectories if name not in {".git", "node_modules"} ] - for filename in filenames: - if filename.endswith(".fabric-adapter.json"): - yield Path(directory) / filename + found.extend( + Path(directory) / filename + for filename in filenames + if filename.endswith(".fabric-adapter.json") + ) + return found + +def _checked_in_descriptors(repo_root: Path) -> list[Path]: + """Every tracked descriptor, or every descriptor on disk outside a git checkout.""" -def test_every_copy_of_a_bundled_descriptor_declares_the_versions_report( + try: + listed = subprocess.run( + ["git", "-C", str(repo_root), "ls-files", "-z", "*.fabric-adapter.json"], + capture_output=True, + check=True, + ).stdout + except (OSError, subprocess.CalledProcessError): + return _descriptors_on_disk(repo_root) + return [ + repo_root / name + for name in listed.decode("utf-8").split("\0") + if name and (repo_root / name).is_file() + ] + + +def test_every_checked_in_copy_of_a_bundled_descriptor_matches_its_source( bundle, repo_root: Path ): # Bundled adapters always attach the versions report, and core fails a run - # whose descriptor does not declare it, so copies used by examples and the - # CLI presets must declare it too. - bundled_ids = set(bundle["adapters"]) - bundled_modules = { - entry["descriptor"]["runner"]["module"] - for entry in bundle["adapters"].values() - if "module" in entry["descriptor"]["runner"] - } - sources = { - (repo_root / entry["source"]).resolve() for entry in bundle["adapters"].values() - } + # whose result carries an undeclared key, so a copy used by an example or a + # CLI preset must stay byte-identical to the descriptor it copies, as the + # CLI's packaged-descriptor test already requires for its own copies. + sources_by_id: dict[str, Path] = {} + sources_by_module: dict[str, Path] = {} + sources_by_script: dict[str, list[Path]] = {} + for adapter_id, entry in bundle["adapters"].items(): + source = repo_root / entry["source"] + runner = entry["descriptor"]["runner"] + sources_by_id[adapter_id] = source + if "module" in runner: + sources_by_module[runner["module"]] = source + if "script" in runner: + sources_by_script.setdefault(runner["script"], []).append(source) copies = [] for path in _checked_in_descriptors(repo_root): descriptor = json.loads(path.read_bytes()) - if ( - descriptor.get("adapter_id") in bundled_ids - or descriptor.get("runner", {}).get("module") in bundled_modules - ): - copies.append(path) - run_result = descriptor.get("extension_schemas", {}).get("run_result", {}) - assert run_result.get("properties", {}).get("nemo_fabric.versions.v1") == { - "type": "object" - }, path + runner = descriptor.get("runner", {}) + if descriptor.get("adapter_id") in sources_by_id: + candidates = [sources_by_id[descriptor["adapter_id"]]] + elif runner.get("module") in sources_by_module: + candidates = [sources_by_module[runner["module"]]] + elif runner.get("script") in sources_by_script: + # Several TypeScript adapters share one script path. + candidates = sources_by_script[runner["script"]] + else: + continue + copies.append(path) + assert any(path.read_bytes() == source.read_bytes() for source in candidates), ( + f"{path} drifted from {[str(source) for source in candidates]}" + ) + sources = {path.resolve() for path in sources_by_id.values()} assert {path.resolve() for path in copies} - sources From 14ff8c6a7ac8fb689ca4a86a0dae1cb2221c2aa7 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 04:56:37 -0700 Subject: [PATCH 087/159] docs: document the served-model argument of the TypeScript report helpers The common package README described withPackageVersion's callback but not packageVersionReport's third parameter, the ServedModelFact type, or missingFact, which the Cline adapter uses to build its fact. Signed-off-by: mnajafian-nv --- adapters/typescript/common/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/adapters/typescript/common/README.md b/adapters/typescript/common/README.md index 700ae129c..93cb2803a 100644 --- a/adapters/typescript/common/README.md +++ b/adapters/typescript/common/README.md @@ -47,4 +47,8 @@ contract. Published package manifests use the registry version instead. `insecureHttpModelHosts(settings, errorCode)` validates and normalizes the explicit `insecure_http_model_hosts` setting. `modelEndpointAllowed(url, hosts, allowPrivateHttp)` applies the shared HTTPS, loopback HTTP, and exact HTTP host policy used by OpenCode, Qwen, and Kilo. The optional private IPv4 argument preserves Kilo's existing opt-in. These utilities do not apply to MCP endpoints. -`withPackageVersion(runtime, packageJson)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. The optional third argument, `servedModel(input)`, returns the `role` and `fact` of one role's served model from the start input, such as `missingFact("unavailable", "not_attributed", "harness_api")`; every other role keeps `no_source`. +`withPackageVersion(runtime, packageJson, servedModel?)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. The optional `servedModel(input)` callback receives the start input and returns a `ServedModelFact`, or `undefined` to keep `no_source` for every role. + +`ServedModelFact` is `{role: string; fact: JsonObject}`: the served-model fact for one model role. Build a fact without a value with `missingFact(state, reason, method)`, for example `missingFact("unavailable", "not_attributed", "harness_api")`; `method` defaults to `adapter_report`. + +`packageVersionReport(packageVersion, modelRoles, servedModel?)` builds the report object itself. `servedModel`, a `ServedModelFact`, gives its role that fact, and every other role keeps `no_source`; a role that is not in `modelRoles` adds no entry. Like NeMo Fabric core, the report keeps at most 16 roles in sorted order and skips a role longer than 128 characters. From 4531f8aa1def7519bf31b2e70b2e01a96b494d52 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 06:56:01 -0700 Subject: [PATCH 088/159] fix: keep plan MCP servers ahead of harness entries under the caps The entry cap kept the first 32 servers in sorted order whatever their origin, and the byte budget dropped the last one. A report naming servers that sort early could therefore push plan servers, and the endpoint facts core owns for them, out of what_ran. Plan servers now take their places first and harness entries fill only the places left, and the byte budget drops harness entries before plan servers. Both cases are counted in servers_truncated as before. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 188 +++++++++++++++++- .../nemo-fabric-core/what-ran/index.mdx | 2 +- docs/sdk/python.mdx | 9 +- 3 files changed, 185 insertions(+), 14 deletions(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index a10a21695..fdddf0ec7 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -16,9 +16,11 @@ //! above are never cut. //! //! `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in -//! sorted key order. An entry whose key is longer than 128 characters is -//! dropped. While the compact JSON of the whole object is larger than -//! 128 KiB, MCP server entries are dropped from the end of the sorted order, +//! sorted key order. `plan` servers take their places first, and servers +//! only the harness reports fill the places left. An entry whose key is +//! longer than 128 characters is dropped. While the compact JSON of the +//! whole object is larger than 128 KiB, MCP server entries are dropped from +//! the end of the sorted order, `harness` entries before `plan` entries, //! then model roles. Dropped entries are counted in a [`TruncatedEntries`] //! marker next to the map: `models_truncated` and `mcp.servers_truncated`. //! If the object is still too large with no MCP server and no model role, @@ -593,14 +595,15 @@ impl WhatRan { } /// Keep the compact JSON of the whole object within 128 KiB. While it - /// is larger, drop the last MCP server in sorted key order, then the - /// last model role, counting each in its map's marker with reason - /// `byte_budget`. With neither left and the object still too large, - /// replace it with [`WhatRan::minimal`]. + /// is larger, drop the last MCP server in sorted key order, a server of + /// any other origin before a `plan` server, then the last model role, + /// counting each in its map's marker with reason `byte_budget`. With + /// neither left and the object still too large, replace it with + /// [`WhatRan::minimal`]. fn apply_byte_budget(&mut self) { while self.serialized_len() > MAX_WHAT_RAN_BYTES { if let Some(mcp) = self.mcp.as_mut() - && mcp.servers.pop_last().is_some() + && pop_last_server(&mut mcp.servers).is_some() { mcp.servers_truncated = Some(one_more_byte_budget_drop(&mcp.servers_truncated)); } else if let Some(models) = self.models.as_mut() @@ -623,8 +626,7 @@ impl WhatRan { self.models_truncated = merged_marker(self.models_truncated.take(), dropped); } if let Some(mcp) = self.mcp.as_mut() { - let (kept, dropped) = - bounded_entries(&mcp.servers, MAX_MCP_SERVERS, McpServerFacts::clone); + let (kept, dropped) = bounded_servers(&mcp.servers); mcp.servers = kept; mcp.servers_truncated = merged_marker(mcp.servers_truncated.take(), dropped); } @@ -773,6 +775,48 @@ fn bounded_entries( (kept, truncated) } +/// At most 32 MCP servers, `plan` servers first: the cap and the key rule +/// of [`bounded_entries`] apply to the `plan` servers, and servers of any +/// other origin, such as `harness`, fill only the places left, so a report +/// never pushes out a server the plan configures. One marker counts every +/// dropped server. +fn bounded_servers( + servers: &BTreeMap, +) -> (BTreeMap, Option) { + let (plan, others): (BTreeMap<_, _>, BTreeMap<_, _>) = servers + .iter() + .map(|(name, server)| (name.clone(), server.clone())) + .partition(|(_, server)| server.origin == ORIGIN_PLAN); + let (mut kept, plan_dropped) = bounded_entries(&plan, MAX_MCP_SERVERS, McpServerFacts::clone); + let (others_kept, others_dropped) = + bounded_entries(&others, MAX_MCP_SERVERS - kept.len(), McpServerFacts::clone); + kept.extend(others_kept); + let dropped = match (plan_dropped, others_dropped) { + (Some(plan), Some(others)) => Some(TruncatedEntries { + reason: if plan.reason == REASON_ENTRY_CAP { + plan.reason + } else { + others.reason + }, + dropped: plan.dropped.saturating_add(others.dropped), + }), + (plan, others) => plan.or(others), + }; + (kept, dropped) +} + +/// Remove the last server in sorted key order whose origin is not `plan`, +/// or the last server when every one is a `plan` server. +fn pop_last_server(servers: &mut BTreeMap) -> Option { + let name = servers + .iter() + .rev() + .find(|(_, server)| server.origin != ORIGIN_PLAN) + .or_else(|| servers.last_key_value()) + .map(|(name, _)| name.clone())?; + servers.remove(&name) +} + /// `marker` with one more dropped entry and reason `byte_budget`. fn one_more_byte_budget_drop(marker: &Option) -> TruncatedEntries { TruncatedEntries { @@ -2414,4 +2458,128 @@ mod tests { json!({"reason": "entry_cap", "dropped": 1}) ); } + + /// A plan declaring the MCP report key, with MCP servers under the given + /// names and, when `heavy_roles` is set, 16 model roles whose providers + /// take most of the 128 KiB budget. + fn mcp_plan(server_names: &[String], heavy_roles: bool) -> RunPlan { + let mut plan = plan_declaring(&[MCP_REPORT_KEY]); + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.capability_plan.mcp_servers = server_names + .iter() + .map(|name| (name.clone(), server.clone())) + .collect(); + if heavy_roles { + let model = plan.config.models["default"].clone(); + plan.config.models = numbered_keys("role", 16) + .into_iter() + .map(|role| (role, model.clone())) + .collect(); + for model in plan.config.models.values_mut() { + model.provider = "p".repeat(7_000); + } + } + plan + } + + /// An MCP report naming each server in `names` with a `runtime` block + /// that lists `tool_count` long tool names. + fn mcp_report(names: &[String], tool_count: usize) -> Value { + let tools: Vec = (0..tool_count) + .map(|index| format!("{index:03}{}", "t".repeat(250))) + .collect(); + let servers: serde_json::Map = names + .iter() + .map(|name| { + ( + name.clone(), + json!({"transport": "stdio", "runtime": { + "connected": reported(json!(true), "harness_status"), + "tool_names": reported(json!(tools), "harness_api")}}), + ) + }) + .collect(); + json!({MCP_REPORT_KEY: {"servers": servers}}) + } + + fn origins(what_ran: &Value) -> Vec<(String, String)> { + what_ran["mcp"]["servers"] + .as_object() + .expect("servers") + .iter() + .map(|(name, server)| { + ( + name.clone(), + server["origin"].as_str().expect("origin").to_string(), + ) + }) + .collect() + } + + #[test] + fn harness_entries_fill_only_the_places_plan_servers_leave_under_the_cap() { + let plan_names = numbered_keys("server", 30); + let harness_names = numbered_keys("aa", 5); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, false), mcp_report(&harness_names, 0)); + + let kept = origins(&what_ran); + let plan_kept: Vec<_> = kept + .iter() + .filter(|(_, origin)| origin == "plan") + .map(|(name, _)| name.clone()) + .collect(); + let harness_kept: Vec<_> = kept + .iter() + .filter(|(_, origin)| origin == "harness") + .map(|(name, _)| name.clone()) + .collect(); + assert_eq!(plan_kept, plan_names); + assert_eq!(harness_kept, harness_names[..2]); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 3}) + ); + // Every kept plan server keeps the endpoint core configured. + for name in &plan_names { + assert_eq!( + what_ran["mcp"]["servers"][name]["endpoint"]["source"], + "configured" + ); + } + } + + #[test] + fn the_byte_budget_drops_harness_entries_before_plan_servers() { + // The plan servers that fit beside 16 heavy model roles. + let collected = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let plan_names = map_keys(&collected["mcp"]["servers"]); + assert!(!plan_names.is_empty()); + // Harness names sort before every plan server. Each entry is about + // 10 KiB, so the five cannot all fit, and the report stays under its + // own 64 KiB limit. + let harness_names = numbered_keys("aa", 5); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, true), mcp_report(&harness_names, 40)); + + assert!(json_len(&what_ran) <= 128 * 1024, "{}", json_len(&what_ran)); + let kept = origins(&what_ran); + let plan_kept: Vec<_> = kept + .iter() + .filter(|(_, origin)| origin == "plan") + .map(|(name, _)| name.clone()) + .collect(); + let harness_kept = kept.len() - plan_kept.len(); + assert_eq!(plan_kept, plan_names); + assert!(harness_kept < harness_names.len(), "{kept:?}"); + assert!( + json_len(&mcp_report(&harness_names, 40)[MCP_REPORT_KEY]) < 64 * 1024, + "the report must be ingested, not marked too_large" + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": harness_names.len() - harness_kept}) + ); + assert_eq!(map_keys(&what_ran["models"]).len(), 16); + } } diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx index a8c31143e..4c475c9d6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/index.mdx @@ -16,7 +16,7 @@ What executed for one invocation, and how NeMo Fabric learned each fact. A string value longer than 256 characters, in a string fact or as an element of a string-list fact, keeps its first 240 characters, then `~`, then the first 15 hex digits of its SHA-256. Map keys and the plain names above are never cut. -`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in sorted key order. An entry whose key is longer than 128 characters is dropped. While the compact JSON of the whole object is larger than 128 KiB, MCP server entries are dropped from the end of the sorted order, then model roles. Dropped entries are counted in a [`TruncatedEntries`](struct-truncatedentries.mdx) marker next to the map: `models_truncated` and `mcp.servers_truncated`. If the object is still too large with no MCP server and no model role, it is replaced by a minimal one that holds only the schema name, the version and the `what_ran_truncated` marker. +`models` keeps at most 16 roles and `mcp.servers` at most 32 servers, in sorted key order. `plan` servers take their places first, and servers only the harness reports fill the places left. An entry whose key is longer than 128 characters is dropped. While the compact JSON of the whole object is larger than 128 KiB, MCP server entries are dropped from the end of the sorted order, `harness` entries before `plan` entries, then model roles. Dropped entries are counted in a [`TruncatedEntries`](struct-truncatedentries.mdx) marker next to the map: `models_truncated` and `mcp.servers_truncated`. If the object is still too large with no MCP server and no model role, it is replaced by a minimal one that holds only the schema name, the version and the `what_ran_truncated` marker. An adapter reports its own facts through the report keys [`VERSIONS_REPORT_KEY`](constant-versions-report-key.mdx) and [`MCP_REPORT_KEY`](constant-mcp-report-key.mdx) in its result's extensions, once its descriptor declares them. A reported list keeps at most 64 elements in sorted order, counted in a marker beside it, such as `tool_names_truncated`. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index f181293e2..bf743891c 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1081,9 +1081,12 @@ order, and a `tool_names_truncated` marker in the `runtime` block counts the names the adapter or NeMo Fabric dropped. `models` keeps at most 16 roles and `mcp.servers` at most 32 servers, taken in -sorted key order. An entry whose key is longer than 128 characters is dropped. -While the whole `what_ran` is larger than 128 KiB of compact JSON, MCP servers -are dropped from the end of the sorted order, and then model roles. A map that +sorted key order. Servers the plan configures take their places first, and +servers only the harness reports fill the places left. An entry whose key is +longer than 128 characters is dropped. While the whole `what_ran` is larger +than 128 KiB of compact JSON, MCP servers are dropped from the end of the +sorted order, harness-reported servers before plan servers, and then model +roles. A map that lost entries has a marker next to it, `models_truncated` or `mcp.servers_truncated`, with the `reason` (`key_too_long`, `entry_cap`, or `byte_budget`) and the `dropped` count. Each map has one marker: when more than From c9d6fad1f292d90488ad82432092d6e3331934e3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 06:56:32 -0700 Subject: [PATCH 089/159] docs: name the state together with each what_ran reason code The adapter tutorial said core marks a fact invalid_report or too_large, which reads as a state. Core sets state collection_failed and uses those codes as the reason. Name the state with its reason there, and in the SDK guide and common TypeScript README sentences that gave only a reason. Signed-off-by: mnajafian-nv --- adapters/typescript/common/README.md | 4 ++-- docs/adapter-contract/tutorials/results.md | 2 +- docs/sdk/python.mdx | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/adapters/typescript/common/README.md b/adapters/typescript/common/README.md index 93cb2803a..acaacab2d 100644 --- a/adapters/typescript/common/README.md +++ b/adapters/typescript/common/README.md @@ -47,8 +47,8 @@ contract. Published package manifests use the registry version instead. `insecureHttpModelHosts(settings, errorCode)` validates and normalizes the explicit `insecure_http_model_hosts` setting. `modelEndpointAllowed(url, hosts, allowPrivateHttp)` applies the shared HTTPS, loopback HTTP, and exact HTTP host policy used by OpenCode, Qwen, and Kilo. The optional private IPv4 argument preserves Kilo's existing opt-in. These utilities do not apply to MCP endpoints. -`withPackageVersion(runtime, packageJson, servedModel?)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. The optional `servedModel(input)` callback receives the start input and returns a `ServedModelFact`, or `undefined` to keep `no_source` for every role. +`withPackageVersion(runtime, packageJson, servedModel?)` wraps a runtime so that each result carries the `nemo_fabric.versions.v1` extension with the adapter's package version, read synchronously from `packageJson` when the runtime starts. Declare that key as `{"type": "object"}` under `extension_schemas.run_result` in the adapter descriptor. The harness name and version and the served models read `unsupported` with reason `no_source`. The optional `servedModel(input)` callback receives the start input and returns a `ServedModelFact`, or `undefined` to keep `unsupported` with reason `no_source` for every role. `ServedModelFact` is `{role: string; fact: JsonObject}`: the served-model fact for one model role. Build a fact without a value with `missingFact(state, reason, method)`, for example `missingFact("unavailable", "not_attributed", "harness_api")`; `method` defaults to `adapter_report`. -`packageVersionReport(packageVersion, modelRoles, servedModel?)` builds the report object itself. `servedModel`, a `ServedModelFact`, gives its role that fact, and every other role keeps `no_source`; a role that is not in `modelRoles` adds no entry. Like NeMo Fabric core, the report keeps at most 16 roles in sorted order and skips a role longer than 128 characters. +`packageVersionReport(packageVersion, modelRoles, servedModel?)` builds the report object itself. `servedModel`, a `ServedModelFact`, gives its role that fact, and every other role keeps `unsupported` with reason `no_source`; a role that is not in `modelRoles` adds no entry. Like NeMo Fabric core, the report keeps at most 16 roles in sorted order and skips a role longer than 128 characters. diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 1aa9fcce6..cfa6d58d7 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -79,7 +79,7 @@ Report only observed invocation usage. For cumulative target counters, compute t `fabric.adapter/v1alpha2` usage is a closed object. Do not add cache-write, reasoning, peak request, or per-model fields at its root. Put that detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` and declare the extension with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized input, cached-input, output, and total token fields and can add `cache_write_input_tokens`, `reasoning_tokens`, `output_tokens_include_reasoning`, `peak_request_input_tokens`, and `models`. Within this object, `input_tokens_include_cache` applies to both cached reads and cache writes. Normalize `input_tokens` first if the provider includes only one component. Each model entry uses the same counters and requires a non-blank `model`; include `provider` when known. -To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. +To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat the detailed object as authoritative independently for each token dimension. Any of `input_tokens`, `cached_input_tokens`, `cache_write_input_tokens`, or `input_tokens_include_cache` selects the detailed input counters. Any of `output_tokens`, `reasoning_tokens`, or `output_tokens_include_reasoning` selects the detailed output counters. Consumers do not fall back to the top-level counters for a selected dimension, so the detailed and top-level views can diverge. If the selected counters do not contain enough information to determine an inclusive total, that total remains unknown even when the top-level view contains a value. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index bf743891c..2e62492ae 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1044,9 +1044,9 @@ MCP runtime facts yet. A reported fact whose key the adapter does not declare has state `unsupported` with reason `key_not_declared` and method `adapter_report`. A fact the adapter -has no source for has reason `no_source`. A declared key that the result omits -gives `unavailable` with reason `key_omitted`, and a fact the key omits gives -`fact_omitted`. A malformed fact, or one that does not name source `reported` +has no source for has state `unsupported` with reason `no_source`. A declared +key that the result omits gives `unavailable` with reason `key_omitted`, and a +fact the key omits gives `unavailable` with reason `fact_omitted`. A malformed fact, or one that does not name source `reported` and collector `adapter`, gives `collection_failed` with reason `invalid_report`. A key larger than 64 KiB of compact JSON gives `collection_failed` with reason `too_large`. None of these changes the run's From b35c3ff24b32d14d6d722bb15dcac325a2efd363 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 06:57:45 -0700 Subject: [PATCH 090/159] test: read expected versions in the provenance tests from package metadata The helper tests compared reported versions with literal package and harness versions, so a routine version bump of Fabric, Deep Agents or mini-SWE-agent would fail them. Read the expected values from the installed package metadata; literal versions remain only as test input. Signed-off-by: mnajafian-nv --- .../adapters/test_adapters_common_provenance.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index 3e9f00453..d9dda1acf 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import json import sys import threading @@ -58,7 +59,7 @@ def test_known_fact_replaces_a_lone_surrogate_so_the_result_stays_valid_json(): def test_package_version_fact_reads_installed_metadata(): assert provenance.package_version_fact("nemo-fabric-adapters-common") == known( - "0.6.0" + importlib.metadata.version("nemo-fabric-adapters-common") ) assert provenance.package_version_fact("nemo-fabric-no-such-package") == missing( "unavailable", "not_collected" @@ -68,8 +69,12 @@ def test_package_version_fact_reads_installed_metadata(): @pytest.mark.parametrize( ("distribution", "module", "expected"), [ - ("deepagents", deepagents.__name__, known("0.7.9")), - ("mini-swe-agent", minisweagent.__name__, known("2.4.6")), + ("deepagents", deepagents.__name__, known(importlib.metadata.version("deepagents"))), + ( + "mini-swe-agent", + minisweagent.__name__, + known(importlib.metadata.version("mini-swe-agent")), + ), # The distribution exists but did not install the loaded module. ("mini-swe-agent", deepagents.__name__, missing("unavailable", "not_attributed")), ("deepagents", "nemo_fabric_not_loaded", missing("unavailable", "not_collected")), @@ -189,7 +194,11 @@ def test_package_version_report_marks_harness_and_served_models_no_source(): no_source = missing("unsupported", "no_source", "adapter_report") assert report == { - "adapter": {"package_version": known("0.6.0")}, + "adapter": { + "package_version": known( + importlib.metadata.version("nemo-fabric-adapters-common") + ) + }, "harness": {"name": no_source, "version": no_source}, "models": {"default": {"served": no_source}, "judge": {"served": no_source}}, } From b047245d9db0d1a212d47e4eb3c63aa310a61419 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 06:57:45 -0700 Subject: [PATCH 091/159] test: read the Hermes adapter version from package metadata The Hermes versions-report test expected the literal 0.6.0, which fails after a normal version bump. Read it from the installed package metadata as the OpenClaw and end-to-end tests do. Signed-off-by: mnajafian-nv --- tests/adapters/test_hermes_adapter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index 04bfe3336..c200cc0ce 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +import importlib.metadata import importlib.util import inspect import json @@ -1756,7 +1757,9 @@ async def test_hermes_reports_the_version_its_version_call_returns( "name": {**_reported("hermes-agent"), "method": "adapter_report"}, "version": _reported(get_version_info().derived_version), } - assert report["adapter"]["package_version"]["value"] == "0.6.0" + assert report["adapter"]["package_version"]["value"] == importlib.metadata.version( + "nemo-fabric-adapters-hermes" + ) def _raise_runtime_error() -> str: From d35a1e94fa8ca20289ba9d3e805ac894ba89f227 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 06:59:15 -0700 Subject: [PATCH 092/159] test: fix the common package check and the Windows probe timing The common TypeScript package now ships dist/provenance.js and its declarations, so its pack check lists them. On Windows the monotonic clock ticks about every 15.6 ms, and the version-probe timeout test measured 0.187 s for a 0.2 s timeout, so its lower bound now allows that resolution. Signed-off-by: mnajafian-nv --- adapters/typescript/scripts/check-package.mjs | 2 ++ tests/adapters/test_adapters_common_provenance.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/adapters/typescript/scripts/check-package.mjs b/adapters/typescript/scripts/check-package.mjs index ef6c77b2f..f6a861645 100644 --- a/adapters/typescript/scripts/check-package.mjs +++ b/adapters/typescript/scripts/check-package.mjs @@ -28,6 +28,8 @@ const expectedByPackage = { "dist/lifecycle.js", "dist/model-endpoint.d.ts", "dist/model-endpoint.js", + "dist/provenance.d.ts", + "dist/provenance.js", "package.json", ], "nemo-fabric-adapters-cline": [ diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index d9dda1acf..ef2c67d18 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -154,7 +154,9 @@ async def count_ticks() -> None: ticker.cancel() release.set() - assert 0.2 <= elapsed < 1 + # Windows' monotonic clock ticks about every 15.6 ms, so the measured wait + # can read slightly under the 0.2 s timeout. + assert 0.15 <= elapsed < 1 # The event loop kept running while the function blocked its thread. assert ticks >= 5 while probe_threads(): From 57a98746848e288cea047f7d7fcb8346f4752f25 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 07:22:49 -0700 Subject: [PATCH 093/159] fix: order TypeScript report roles by code point like core packageVersionReport sorted roles with Array.prototype.sort, which compares UTF-16 code units. Core and the Python helper order by code point, which is UTF-8 byte order, so past the 16-role cap a role above U+FFFF and one in U+E000 to U+FFFF swapped places. The TypeScript helper then kept a role core drops, and a served-model override on the role core keeps was lost. Sort by code point, and check all three implementations against one shared role-order fixture. Signed-off-by: mnajafian-nv --- adapters/typescript/common/src/provenance.ts | 22 +++++++++- .../common/test/provenance.test.mjs | 12 ++++++ crates/fabric-core/src/what_ran.rs | 18 +++++++++ .../test_adapters_common_provenance.py | 21 ++++++++++ tests/fixtures/what_ran/role-order.json | 40 +++++++++++++++++++ 5 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/what_ran/role-order.json diff --git a/adapters/typescript/common/src/provenance.ts b/adapters/typescript/common/src/provenance.ts index 8e084519d..1b24e3fb5 100644 --- a/adapters/typescript/common/src/provenance.ts +++ b/adapters/typescript/common/src/provenance.ts @@ -65,6 +65,23 @@ export function packageVersionFact(packageJson: URL): JsonObject { return {value: boundedText(version), source: "reported", collected_by: "adapter", method: "package_metadata"}; } +/** + * Order two strings by Unicode code point, which is the UTF-8 byte order that + * NeMo Fabric core and the Python helper use. `Array.prototype.sort` compares + * UTF-16 code units, which puts a character above U+FFFF before one in + * U+E000 to U+FFFF. + */ +function compareCodePoints(left: string, right: string): number { + const leftPoints = Array.from(left, (char) => char.codePointAt(0) ?? 0); + const rightPoints = Array.from(right, (char) => char.codePointAt(0) ?? 0); + for (const [index, point] of leftPoints.entries()) { + const other = rightPoints[index]; + if (other === undefined) return 1; + if (point !== other) return point - other; + } + return leftPoints.length - rightPoints.length; +} + /** * The report of an adapter that has a source only for its package version. * The harness name and version read `unsupported` with reason `no_source`, and @@ -78,7 +95,10 @@ export function packageVersionReport( servedModel?: ServedModelFact, ): JsonObject { const noSource = () => missingFact("unsupported", "no_source"); - const roles = modelRoles.filter((role) => Array.from(role).length <= MAX_KEY_CHARS).sort().slice(0, MAX_MODEL_ROLES); + const roles = modelRoles + .filter((role) => Array.from(role).length <= MAX_KEY_CHARS) + .sort(compareCodePoints) + .slice(0, MAX_MODEL_ROLES); return { adapter: {package_version: packageVersion}, harness: {name: noSource(), version: noSource()}, diff --git a/adapters/typescript/common/test/provenance.test.mjs b/adapters/typescript/common/test/provenance.test.mjs index 9746b3a97..187886cd8 100644 --- a/adapters/typescript/common/test/provenance.test.mjs +++ b/adapters/typescript/common/test/provenance.test.mjs @@ -107,3 +107,15 @@ test("the wrapper picks the served-model role from the start input", async () => await rm(directory, {recursive: true, force: true}); } }); + +test("the report keeps the roles core keeps, in code point order", async () => { + // The vectors Rust core and the Python helper also read. + const vectors = JSON.parse(await readFile(new URL("../../../../tests/fixtures/what_ran/role-order.json", import.meta.url), "utf8")); + const packageVersion = {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}; + const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); + + const report = packageVersionReport(packageVersion, vectors.roles, {role: "\u{E000}", fact: notAttributed}); + + assert.deepEqual(Object.keys(report.models), vectors.kept); + assert.deepEqual(report.models["\u{E000}"], {served: notAttributed}); +}); diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index fdddf0ec7..1603c7e39 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1490,6 +1490,24 @@ mod tests { assert!(what_ran_validator().is_valid(&what_ran), "{what_ran}"); } + #[test] + fn model_roles_past_the_cap_keep_the_code_point_order_the_helpers_share() { + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/role-order.json" + )) + .expect("role order JSON"); + let roles: Vec = serde_json::from_value(vectors["roles"].clone()).expect("roles"); + let expected: Vec = serde_json::from_value(vectors["kept"].clone()).expect("kept"); + + let what_ran = collect(&plan_with_entries(&roles, &[])); + + assert_eq!(map_keys(&what_ran["models"]), expected); + assert_eq!( + what_ran["models_truncated"], + json!({"reason": "entry_cap", "dropped": 1}) + ); + } + #[test] fn keys_longer_than_128_characters_are_dropped_without_taking_a_place() { let longest = "é".repeat(128); diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index ef2c67d18..b649ee10f 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -114,6 +114,27 @@ def test_versions_report_keeps_16_roles_in_sorted_order_and_bounds_values(): assert report["adapter"]["package_version"] == known("0.6.0") + +def test_versions_report_keeps_the_roles_core_keeps_in_code_point_order(): + vectors = json.loads( + ( + Path(__file__).resolve().parents[1] / "fixtures/what_ran/role-order.json" + ).read_text(encoding="utf-8") + ) + override = missing("unavailable", "not_attributed", "harness_api") + served = {role: provenance.no_source_fact() for role in vectors["roles"]} + served["\ue000"] = override + + report = provenance.versions_report( + package_version=provenance.known_fact("0.6.0", "package_metadata"), + harness_name=provenance.no_source_fact(), + harness_version=provenance.no_source_fact(), + served_models=served, + ) + + assert list(report["models"]) == vectors["kept"] + assert report["models"]["\ue000"] == {"served": override} + def probe_threads() -> int: return sum( thread.name == "nemo-fabric-version-probe" for thread in threading.enumerate() diff --git a/tests/fixtures/what_ran/role-order.json b/tests/fixtures/what_ran/role-order.json new file mode 100644 index 000000000..012e9ea4a --- /dev/null +++ b/tests/fixtures/what_ran/role-order.json @@ -0,0 +1,40 @@ +{ + "description": "Model roles past the 16-role cap. Rust core, the Python helper and the TypeScript helper sort role keys by Unicode code point (UTF-8 byte order) and keep the first 16, so all three keep the same roles. U+10000 sorts after U+E000 by code point but before it by UTF-16 code unit.", + "roles": [ + "\ud800\udc00", + "\ue000", + "role-14", + "role-13", + "role-12", + "role-11", + "role-10", + "role-09", + "role-08", + "role-07", + "role-06", + "role-05", + "role-04", + "role-03", + "role-02", + "role-01", + "role-00" + ], + "kept": [ + "role-00", + "role-01", + "role-02", + "role-03", + "role-04", + "role-05", + "role-06", + "role-07", + "role-08", + "role-09", + "role-10", + "role-11", + "role-12", + "role-13", + "role-14", + "\ue000" + ] +} From 9af928bb79d5f1f149a194a39c7ed6807aee002d Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 07:23:04 -0700 Subject: [PATCH 094/159] docs: say when an undeclared report key fails the run The guide said none of the ingestion outcomes changes a run's status, right after describing undeclared keys. A result that carries a key the descriptor's run_result schema rejects fails before ingestion and returns no RunResult. Say so, and limit the status sentence to a returned run. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 2e62492ae..e9a6bfcb9 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1042,15 +1042,18 @@ because nothing yet shows that the model its SDK names served this invocation. No bundled adapter reports MCP runtime facts yet. -A reported fact whose key the adapter does not declare has state `unsupported` -with reason `key_not_declared` and method `adapter_report`. A fact the adapter -has no source for has state `unsupported` with reason `no_source`. A declared -key that the result omits gives `unavailable` with reason `key_omitted`, and a -fact the key omits gives `unavailable` with reason `fact_omitted`. A malformed fact, or one that does not name source `reported` +When the descriptor does not declare a report key, ingestion leaves the +corresponding facts `unsupported` with reason `key_not_declared`. +Result-extension validation runs first: if the schema rejects an emitted +undeclared key, the run fails before ingestion and no `RunResult` is returned. +A fact the adapter has no source for has state `unsupported` with reason +`no_source`. A declared key that the result omits gives `unavailable` with +reason `key_omitted`, and a fact the key omits gives `unavailable` with reason +`fact_omitted`. A malformed fact, or one that does not name source `reported` and collector `adapter`, gives `collection_failed` with reason `invalid_report`. A key larger than 64 KiB of compact JSON gives -`collection_failed` with reason `too_large`. None of these changes the run's -status or error. +`collection_failed` with reason `too_large`. These ingestion outcomes do not +change a returned run's status or error. An MCP entry has `origin` `plan` for a server the plan configures. An entry with `origin` `harness` describes a server the harness reports but the plan From 3023e80cc42be616ff91354f9489b6134885acc2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 07:52:14 -0700 Subject: [PATCH 095/159] fix: add a report's harness server drops to core's servers marker An adapter's helper can drop MCP servers from its report to keep the key under 64 KiB, and count them in the report's servers_truncated marker. Core ignored that marker, so harness-only servers the helper dropped vanished from what_ran without a count. Core now adds the marker's count, less the plan servers the report omits (those keep their entries), to its own marker. A malformed marker makes the servers report read invalid_report, as a malformed tool_names marker does. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 131 ++++++++++++++++++++++++++++- docs/sdk/python.mdx | 15 ++-- 2 files changed, 136 insertions(+), 10 deletions(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 1603c7e39..2724f85d1 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -977,8 +977,9 @@ const REASON_TOO_LARGE: &str = "too_large"; /// Inside a report, a key may hold only adapter-owned facts, each with /// source `reported` and collector `adapter`: the adapter package version, /// the harness name and version, each plan role's served model, each MCP -/// server's `runtime` block with its `tool_names_truncated` marker, and a -/// `harness` server's transport. A malformed or forged fact reads +/// server's `runtime` block with its `tool_names_truncated` marker, a +/// `harness` server's transport, and the `servers_truncated` marker beside +/// `servers`, whose count core adds to its own only for `harness` entries. A malformed or forged fact reads /// `collection_failed` with reason `invalid_report`, and an omitted one /// reads `unavailable` with reason `fact_omitted`. Core ignores every other /// property, so a report never changes a fact core owns. The string length @@ -1114,18 +1115,53 @@ fn apply_versions_report(what_ran: &mut WhatRan, report: &Report<'_>) { /// Fill each plan server's `runtime` block, and add a `harness` entry for /// each reported server the plan does not configure. +/// +/// The report's `servers_truncated` marker counts servers the adapter's +/// helper dropped from the report. A plan server the helper dropped keeps +/// its entry, with facts that read `fact_omitted`, so core adds to its own +/// marker only the count left after the plan servers the report omits: +/// the `harness` entries that disappear from `what_ran`. A malformed marker +/// makes the whole `servers` report read `invalid_report`, as a malformed +/// `tool_names_truncated` does for `tool_names`. fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) { let Some(mcp) = what_ran.mcp.as_mut() else { return; }; - let servers = report.child("servers"); + let mut servers = report.child("servers"); + let mut helper_marker = None; + if let (Report::Object(_), Report::Object(object)) = (&servers, report) + && let Some(marker) = object.get("servers_truncated") + { + match TruncatedEntries::deserialize(marker) { + Ok(marker) => helper_marker = Some(marker), + Err(_) => { + servers = Report::missing(FactState::CollectionFailed, REASON_INVALID_REPORT); + } + } + } + let plan_servers = &plan.capability_plan.mcp_servers; + if let (Some(marker), Report::Object(reported)) = (helper_marker, &servers) { + let plan_servers_omitted = plan_servers + .keys() + .filter(|name| !reported.contains_key(*name)) + .count() as u64; + let harness_dropped = marker.dropped.saturating_sub(plan_servers_omitted); + if harness_dropped > 0 { + mcp.servers_truncated = merged_marker( + mcp.servers_truncated.take(), + Some(TruncatedEntries { + reason: marker.reason, + dropped: harness_dropped, + }), + ); + } + } for (name, server) in &mut mcp.servers { server.runtime = runtime_facts(&servers.child(name).child("runtime")); } let Report::Object(reported) = servers else { return; }; - let plan_servers = &plan.capability_plan.mcp_servers; for (name, entry) in reported { if plan_servers.contains_key(name) { continue; @@ -2600,4 +2636,91 @@ mod tests { ); assert_eq!(map_keys(&what_ran["models"]).len(), 16); } + + fn connected_runtime() -> Value { + json!({"runtime": {"connected": reported(json!(true), "harness_status")}}) + } + + #[test] + fn a_helper_servers_marker_adds_its_harness_drops_to_core_marker() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + // Every plan server is reported, so all three helper drops were + // harness entries. + let report = json!({ + "servers": {"events": connected_runtime(), "files": connected_runtime(), + "github": connected_runtime(), "plugin-docs": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 3} + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + ["events", "files", "github", "plugin-docs"] + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + } + + #[test] + fn a_helper_servers_marker_never_counts_plan_servers_the_report_omits() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + // The helper dropped `github`, a plan server that keeps its entry, + // and two harness entries. + let report = json!({ + "servers": {"events": connected_runtime(), "files": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 3} + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert_eq!( + what_ran["mcp"]["servers"]["github"]["runtime"]["connected"], + report_state("unavailable", "fact_omitted") + ); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 2}) + ); + + // A marker that counts only omitted plan servers adds nothing. + let report = json!({ + "servers": {"events": connected_runtime(), "files": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 1} + }); + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + assert!(what_ran["mcp"].get("servers_truncated").is_none()); + } + + #[test] + fn a_malformed_helper_servers_marker_invalidates_the_servers_report() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + for marker in [ + json!({"reason": "byte_budget", "dropped": 0}), + json!({"reason": "byte_budget"}), + json!("three"), + Value::Null, + ] { + let report = json!({ + "servers": {"events": connected_runtime(), "plugin-docs": connected_runtime()}, + "servers_truncated": marker + }); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + let servers = &what_ran["mcp"]["servers"]; + assert_eq!(map_keys(servers), ["events", "files", "github"], "{marker}"); + assert_eq!( + servers["events"]["runtime"]["connected"], + report_state("collection_failed", "invalid_report"), + "{marker}" + ); + assert!( + what_ran["mcp"].get("servers_truncated").is_none(), + "{marker}" + ); + } + } } diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index e9a6bfcb9..3c1761b3e 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1089,15 +1089,18 @@ servers only the harness reports fill the places left. An entry whose key is longer than 128 characters is dropped. While the whole `what_ran` is larger than 128 KiB of compact JSON, MCP servers are dropped from the end of the sorted order, harness-reported servers before plan servers, and then model -roles. A map that -lost entries has a marker next to it, `models_truncated` or +roles. A map that lost entries has a marker next to it, `models_truncated` or `mcp.servers_truncated`, with the `reason` (`key_too_long`, `entry_cap`, or `byte_budget`) and the `dropped` count. Each map has one marker: when more than one rule dropped entries, `reason` names the last rule applied, in that order, -and `dropped` counts every dropped entry. `dropped` is a JSON integer. The Rust -core and the Python SDK reject `1.0`, although a JSON Schema validator may -accept it. If `what_ran` is still too large with no MCP server and no model -role left, it holds only `schema`, `version`, and `what_ran_truncated` with +and `dropped` counts every dropped entry. When the adapter's report drops +servers to fit its own size limit, `mcp.servers_truncated` also counts the ones +only the harness reported, because those disappear; a plan server the report +drops keeps its entry, and its runtime facts read `unavailable` with reason +`fact_omitted`. `dropped` is a JSON integer. The Rust core and the Python SDK +reject `1.0`, although a JSON Schema validator may accept it. If `what_ran` is +still too large with no MCP server and no model role left, it holds only +`schema`, `version`, and `what_ran_truncated` with reason `byte_budget` and `dropped` 1. Check for `what_ran_truncated` before you read a section. Dropping entries never changes the run's status or error. From 6980d82bcf8f100fe227277ab4057194436a4baa Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 07:53:51 -0700 Subject: [PATCH 096/159] test: check warm-runtime invocation CPU is end minus start A warm runtime's snapshot CPU time is cumulative since the adapter host started, and readers estimate each invocation's CPU time as the end line's value minus the start line's. Nothing checked that the start line reads the snapshot when its own invocation starts, so a record whose start value fell back to the host's start would report the running total as the second invocation's CPU time. The test runs two records against one fake cumulative CPU counter and asserts that end minus start is 2.0 s and 0.5 s, not 3.0 s and 3.5 s. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 38 +++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 01a946912..b5f68eb7d 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1407,4 +1407,42 @@ mod tests { ); let _ = std::fs::remove_dir_all(folder); } + + #[test] + fn warm_runtime_end_minus_start_cpu_is_each_invocation_own_cpu_time() { + use std::sync::atomic::{AtomicU64, Ordering}; + // The snapshot's CPU time is cumulative since the host started, so a + // warm runtime's second invocation starts with the first one's CPU + // time already counted. Whole milliseconds keep the subtraction exact. + let host_cpu_ms = Arc::new(AtomicU64::new(1_000)); + let read: ReadResources = { + let host_cpu_ms = Arc::clone(&host_cpu_ms); + Arc::new(move || { + let cpu_ms = host_cpu_ms.load(Ordering::SeqCst); + Ok(RuntimeResourceSnapshot { + cpu_time_seconds: Duration::from_millis(cpu_ms).as_secs_f64(), + ..snapshot() + }) + }) + }; + let ticker = idle_ticker(); + let (_, _, invocation) = record_test_fixture(); + let cpu = |line: &Value| line["resources"]["cpu_time_seconds"].as_f64().expect("cpu"); + + let mut own_cpu = Vec::new(); + for (name, burn_ms) in [("warm-cpu-1", 2_000), ("warm-cpu-2", 500)] { + let (folder, mut recorder) = start_recorder(name, &ticker, Arc::clone(&read)); + host_cpu_ms.fetch_add(burn_ms, Ordering::SeqCst); + recorder.exchange_ended(recorder.read_exchange_end(Instant::now(), &read())); + recorder + .finish(Ok(succeeded_result(&invocation))) + .expect("result"); + let lines = read_lines(&folder.join(RECORD_FILE_NAME)); + let _ = std::fs::remove_dir_all(folder); + own_cpu.push(cpu(lines.last().expect("end line")) - cpu(&lines[0])); + } + + // Not 3.0 and 3.5, the host's totals at each end. + assert_eq!(own_cpu, [2.0, 0.5]); + } } From 29cf0fd22502338baad902140043c98f66f2c6ce Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 07:54:56 -0700 Subject: [PATCH 097/159] test: show an older core accepts descriptors that declare the versions key The design requires that a core without report-key ingestion still accept the new descriptors. The test loads every bundled descriptor and runs the result-extension check that predates ingestion, with and without the versions report, and checks that the key stays an optional object property. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 113 +++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index a452c7f94..cfce6639e 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -6930,6 +6930,119 @@ mod tests { .expect("declared result extension"); } + /// Every bundled descriptor under `adapters/`. + fn bundled_descriptor_paths() -> Vec { + let mut paths = Vec::new(); + for language in ["python", "typescript"] { + let root = repository_root().join("adapters").join(language); + for adapter in fs::read_dir(&root).expect("adapter directory") { + let adapter = adapter.expect("adapter entry").path(); + let Ok(files) = fs::read_dir(&adapter) else { + continue; + }; + for file in files { + let file = file.expect("adapter file").path(); + if file + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".fabric-adapter.json")) + { + paths.push(file); + } + } + } + } + paths.sort(); + paths + } + + // The descriptor loader and the result-extension check below predate + // report-key ingestion and run unchanged in an older core, which treats + // `nemo_fabric.versions.v1` as an ordinary optional object property. + #[test] + fn a_descriptor_declaring_the_versions_key_passes_the_pre_ingestion_checks() { + let paths = bundled_descriptor_paths(); + assert_eq!(paths.len(), 16, "{paths:?}"); + let report = serde_json::json!({ + "adapter": {"package_version": {"value": "0.6.0", "source": "reported", + "collected_by": "adapter", "method": "package_metadata"}} + }); + for path in paths { + let descriptor = load_adapter_descriptor(&path).expect("descriptor loads"); + let run_result = &descriptor.extension_schemas[&AdapterExtensionPoint::RunResult]; + assert_eq!( + run_result["properties"]["nemo_fabric.versions.v1"], + serde_json::json!({"type": "object"}), + "{}", + path.display() + ); + // Core skips an empty extensions block, so check optionality on + // the schema too: a required key would reject an older adapter's + // result that carries other keys but not this one. + assert!( + !run_result + .get("required") + .and_then(serde_json::Value::as_array) + .is_some_and( + |required| required.contains(&serde_json::json!("nemo_fabric.versions.v1")) + ), + "{}", + path.display() + ); + let base = required_result_extensions(run_result); + let resolved = resolved_adapter(path.clone(), descriptor); + let mut with_report = base.clone(); + with_report.insert("nemo_fabric.versions.v1".to_string(), report.clone()); + for extensions in [base, with_report] { + let result: AgentRunResult = serde_json::from_value(serde_json::json!({ + "status": "succeeded", + "output": "done", + "extensions": extensions + })) + .expect("agent result"); + validate_agent_run_result_extensions(&result, Some(&resolved)).unwrap_or_else( + |error| { + panic!( + "{} rejected {:?}: {error}", + path.display(), + result.extensions + ) + }, + ); + } + } + } + + /// The smallest values for the keys other than the versions key that a + /// `run_result` schema requires, such as a session ID. + fn required_result_extensions( + schema: &serde_json::Map, + ) -> serde_json::Map { + let required = schema + .get("required") + .and_then(serde_json::Value::as_array) + .cloned() + .unwrap_or_default(); + required + .iter() + .map(|key| key.as_str().expect("required key")) + // A descriptor that required the versions key would break an + // older core's results that omit it, so the base never adds it. + .filter(|key| *key != "nemo_fabric.versions.v1") + .map(|key| { + let property = &schema["properties"][key]; + let value = match (property.get("enum"), property["type"].as_str()) { + (Some(values), _) => values[0].clone(), + (None, Some("string")) => serde_json::json!("value"), + (None, Some("integer")) => serde_json::json!(0), + (None, Some("boolean")) => serde_json::json!(false), + (None, other) => panic!("no sample for {key}: {other:?}"), + }; + (key.to_string(), value) + }) + .collect() + } + #[test] fn workflow_settings_are_fail_closed() { let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); From 5696458868ce73146e2eefa33d6dbb7a414d16d3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 07:56:48 -0700 Subject: [PATCH 098/159] test: read the string length vectors from one shared fixture Rust core, the Python helper and the TypeScript helper each kept their own copy of the three string length vectors, so a change to one copy could leave the others agreeing with a stale rule. All three now read tests/fixtures/what_ran/string-length.json, next to the role-order fixture. The lone-surrogate case stays in the helper tests because a JSON fixture can't carry it into Rust. Signed-off-by: mnajafian-nv --- .../common/test/provenance.test.mjs | 15 +++++---- crates/fabric-core/src/what_ran.rs | 28 ++++++++-------- .../test_adapters_common_provenance.py | 33 ++++++++++--------- tests/fixtures/what_ran/string-length.json | 17 ++++++++++ 4 files changed, 58 insertions(+), 35 deletions(-) create mode 100644 tests/fixtures/what_ran/string-length.json diff --git a/adapters/typescript/common/test/provenance.test.mjs b/adapters/typescript/common/test/provenance.test.mjs index 187886cd8..bb38d8cd5 100644 --- a/adapters/typescript/common/test/provenance.test.mjs +++ b/adapters/typescript/common/test/provenance.test.mjs @@ -19,11 +19,14 @@ import { const noSource = {state: "unsupported", reason: "no_source", source: "reported", collected_by: "adapter", method: "adapter_report"}; -// The same vectors as core's string length test in `what_ran.rs`. -test("bounded text matches core's string length rule", () => { - assert.equal(boundedText("é".repeat(256)), "é".repeat(256)); - assert.equal(boundedText(`${"m".repeat(300)}a`), `${"m".repeat(240)}~96858eedeb3c8e6`); - assert.equal(boundedText("é".repeat(300)), `${"é".repeat(240)}~7250b66610f8b7d`); +const fixtures = new URL("../../../../tests/fixtures/what_ran/", import.meta.url); + +test("bounded text matches core's string length rule", async () => { + // The vectors Rust core and the Python helper also read. + const {cases} = JSON.parse(await readFile(new URL("string-length.json", fixtures), "utf8")); + assert.equal(cases.length, 3); + for (const {text, expected} of cases) assert.equal(boundedText(text), expected); + // JSON fixtures can't carry a lone surrogate into Rust, so this case stays here. assert.equal(boundedText("v\ud800"), "v?"); }); @@ -110,7 +113,7 @@ test("the wrapper picks the served-model role from the start input", async () => test("the report keeps the roles core keeps, in code point order", async () => { // The vectors Rust core and the Python helper also read. - const vectors = JSON.parse(await readFile(new URL("../../../../tests/fixtures/what_ran/role-order.json", import.meta.url), "utf8")); + const vectors = JSON.parse(await readFile(new URL("role-order.json", fixtures), "utf8")); const packageVersion = {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}; const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 2724f85d1..a8ec1873f 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1937,19 +1937,21 @@ mod tests { #[test] fn string_values_longer_than_256_characters_end_with_a_digest_prefix() { - let exact = "é".repeat(256); - assert_eq!(bounded_text(&exact), exact); - - // Expected digests: the first 15 hex digits of `sha256sum` over the - // whole value's UTF-8 bytes. - for (long, prefix, digest) in [ - (format!("{}a", "m".repeat(300)), "m", "96858eedeb3c8e6"), - ("é".repeat(300), "é", "7250b66610f8b7d"), - ] { - let bounded = bounded_text(&long); - - assert_eq!(bounded, format!("{}~{digest}", prefix.repeat(240))); - assert_eq!(bounded.chars().count(), 256); + // The vectors the Python and TypeScript helpers also read. Expected + // digests are the first 15 hex digits of `sha256sum` over the whole + // value's UTF-8 bytes. + let vectors: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/what_ran/string-length.json" + )) + .expect("string length JSON"); + let cases = vectors["cases"].as_array().expect("cases"); + assert_eq!(cases.len(), 3); + for case in cases { + let text = case["text"].as_str().expect("text"); + let bounded = bounded_text(text); + + assert_eq!(bounded, case["expected"].as_str().expect("expected")); + assert_eq!(bounded.chars().count(), text.chars().count().min(256)); } } diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index b649ee10f..4f0e899f7 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -37,17 +37,22 @@ def missing(state: str, reason: str, method: str = "package_metadata") -> dict[s } -# The same vectors as core's string length test in `what_ran.rs`. -@pytest.mark.parametrize( - ("text", "expected"), - [ - ("é" * 256, "é" * 256), - ("m" * 300 + "a", "m" * 240 + "~96858eedeb3c8e6"), - ("é" * 300, "é" * 240 + "~7250b66610f8b7d"), - ], -) -def test_bounded_text_matches_core_string_length_rule(text, expected): - assert provenance.bounded_text(text) == expected +FIXTURES = Path(__file__).resolve().parents[1] / "fixtures/what_ran" + + +def string_length_cases() -> list[dict[str, str]]: + """The string length vectors that core and the TypeScript helper also read.""" + + vectors = json.loads((FIXTURES / "string-length.json").read_text(encoding="utf-8")) + return vectors["cases"] + + +def test_bounded_text_matches_core_string_length_rule(): + cases = string_length_cases() + + assert len(cases) == 3 + for case in cases: + assert provenance.bounded_text(case["text"]) == case["expected"] def test_known_fact_replaces_a_lone_surrogate_so_the_result_stays_valid_json(): @@ -116,11 +121,7 @@ def test_versions_report_keeps_16_roles_in_sorted_order_and_bounds_values(): def test_versions_report_keeps_the_roles_core_keeps_in_code_point_order(): - vectors = json.loads( - ( - Path(__file__).resolve().parents[1] / "fixtures/what_ran/role-order.json" - ).read_text(encoding="utf-8") - ) + vectors = json.loads((FIXTURES / "role-order.json").read_text(encoding="utf-8")) override = missing("unavailable", "not_attributed", "harness_api") served = {role: provenance.no_source_fact() for role in vectors["roles"]} served["\ue000"] = override diff --git a/tests/fixtures/what_ran/string-length.json b/tests/fixtures/what_ran/string-length.json new file mode 100644 index 000000000..893fc1ae1 --- /dev/null +++ b/tests/fixtures/what_ran/string-length.json @@ -0,0 +1,17 @@ +{ + "description": "String length rule vectors that Rust core, the Python helper and the TypeScript helper share. A value of at most 256 Unicode scalar values is kept whole. A longer one keeps its first 240, then ~, then the first 15 hex digits of the SHA-256 of its UTF-8 bytes.", + "cases": [ + { + "text": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", + "expected": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé" + }, + { + "text": "mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmma", + "expected": "mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm~96858eedeb3c8e6" + }, + { + "text": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé", + "expected": "éééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééééé~7250b66610f8b7d" + } + ] +} From 64feb12af801e3b208d486af5ae3c1fdd803872e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:01:29 -0700 Subject: [PATCH 099/159] test: check versions report validation under three run_result schemas Core keeps its strict result-extension check ahead of ingestion. The runtime test now runs the same emitted versions report with no run_result schema and with a schema that rejects the key, both of which fail the invocation with the exact existing errors, and with a schema that accepts the key without naming it, which succeeds, keeps the key in metadata.adapter and leaves the facts key_not_declared. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 82 ++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 8 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 17c4fee03..8c017798e 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -5620,22 +5620,41 @@ for line in sys.stdin: } /// An adapter that declares the versions key and reports it gets its - /// facts into `what_ran`, and the key leaves `metadata.adapter`. + /// facts into `what_ran`, and the key leaves `metadata.adapter`. The + /// existing result-extension check runs first: a descriptor with no + /// `run_result` schema, or one that rejects the key, fails the + /// invocation, and one that accepts the key without naming it leaves + /// the facts `key_not_declared`. #[test] fn local_host_what_ran_ingests_a_declared_versions_report() { use serde_json::json; let (root, plan) = local_host_plan("versions_report"); let descriptor_path = root.join("adapters/local-host/local-host.fabric-adapter.json"); - let mut descriptor: Value = + let original: Value = serde_json::from_str(&fs::read_to_string(&descriptor_path).expect("read descriptor")) .expect("descriptor JSON"); - descriptor["extension_schemas"]["run_result"]["properties"]["nemo_fabric.versions.v1"] = - json!({"type": "object"}); - fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); - let plan = resolve_run_plan_from_config(plan.config, ResolveContext::new(&root)) - .expect("resolve plan"); + // Run the same emitted report under the given `run_result` schema. + let run_with = |run_result: Option| { + let mut descriptor = original.clone(); + let schemas = descriptor["extension_schemas"] + .as_object_mut() + .expect("extension schemas"); + match run_result { + Some(schema) => schemas.insert("run_result".to_string(), schema), + None => schemas.remove("run_result"), + }; + fs::write(&descriptor_path, descriptor.to_string()).expect("write descriptor"); + let plan = + resolve_run_plan_from_config(plan.config.clone(), ResolveContext::new(&root)) + .expect("resolve plan"); + run_plan(&plan, RunRequest::text("provenance")) + }; + let declared = json!({"type": "object", "properties": { + "trace_id": {"type": "string"}, + "nemo_fabric.versions.v1": {"type": "object"} + }, "additionalProperties": false}); - let result = run_plan(&plan, RunRequest::text("provenance")).expect("result"); + let result = run_with(Some(declared)).expect("result"); assert_eq!(result.status, RunStatus::Succeeded); assert_eq!(result.metadata["adapter"], json!({"trace_id": "trace-1"})); @@ -5653,6 +5672,53 @@ for line in sys.stdin: json!({"name": reported("fake-harness", "harness_api"), "version": reported("4.5.6", "harness_api")}) ); + + // No `run_result` schema, and one that rejects the undeclared key, + // both fail the invocation before ingestion. + let error = run_with(None).expect_err("no run_result schema"); + assert!( + matches!( + &error, + FabricError::AdapterCompatibility { field, reason, .. } + if field == "result.extensions" + && reason + == "the adapter descriptor does not declare an extension schema for run_result" + ), + "{error:?}" + ); + let rejecting = json!({"type": "object", "properties": {"trace_id": {"type": "string"}}, + "additionalProperties": false}); + let error = run_with(Some(rejecting)).expect_err("rejecting run_result schema"); + assert!( + matches!( + &error, + FabricError::InvalidAdapterExtension { extension_path, reason, .. } + if extension_path == "result.extensions.nemo_fabric.versions.v1" + && reason == "is not declared by the adapter extension schema" + ), + "{error:?}" + ); + + // A schema that accepts the key without naming it keeps the run and + // the key in `metadata.adapter`, and ingestion leaves the facts + // `key_not_declared`. + let permissive = json!({"type": "object", "properties": {"trace_id": {"type": "string"}}}); + + let result = run_with(Some(permissive)).expect("result"); + + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!( + result.metadata["adapter"]["nemo_fabric.versions.v1"]["harness"]["version"], + reported("4.5.6", "harness_api") + ); + let what_ran = serde_json::to_value(result.what_ran.expect("what_ran")).expect("JSON"); + let not_declared = json!({"state": "unsupported", "reason": "key_not_declared", + "source": "reported", "collected_by": "adapter", "method": "adapter_report"}); + assert_eq!(what_ran["adapter"]["package_version"], not_declared); + assert_eq!( + what_ran["harness"], + json!({"name": not_declared, "version": not_declared}) + ); let _ = fs::remove_dir_all(root); } From e16b42b41a099dd0201f4174016d76986ad0b837 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:01:55 -0700 Subject: [PATCH 100/159] docs: explain the descriptor-copy migration for the versions report Core validates result extensions before it ingests report keys, so an external copy of a bundled descriptor whose run_result schema rejects nemo_fabric.versions.v1 fails every invocation once the adapter implementation is upgraded. The SDK guide and the integrate skill now say how to update such a copy, and the adapter tutorial and the adapter authoring skill say to declare the key before emitting it. Signed-off-by: mnajafian-nv --- docs/adapter-contract/tutorials/results.md | 2 +- docs/sdk/python.mdx | 8 ++++++++ skills/nemo-fabric-build-adapter/SKILL.md | 2 +- .../references/results-and-errors.md | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index cfa6d58d7..ea89dce17 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -79,7 +79,7 @@ Report only observed invocation usage. For cumulative target counters, compute t `fabric.adapter/v1alpha2` usage is a closed object. Do not add cache-write, reasoning, peak request, or per-model fields at its root. Put that detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` and declare the extension with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized input, cached-input, output, and total token fields and can add `cache_write_input_tokens`, `reasoning_tokens`, `output_tokens_include_reasoning`, `peak_request_input_tokens`, and `models`. Within this object, `input_tokens_include_cache` applies to both cached reads and cache writes. Normalize `input_tokens` first if the provider includes only one component. Each model entry uses the same counters and requires a non-blank `model`; include `provider` when known. -To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. +To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. Result-extension validation runs first: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat the detailed object as authoritative independently for each token dimension. Any of `input_tokens`, `cached_input_tokens`, `cache_write_input_tokens`, or `input_tokens_include_cache` selects the detailed input counters. Any of `output_tokens`, `reasoning_tokens`, or `output_tokens_include_reasoning` selects the detailed output counters. Consumers do not fall back to the top-level counters for a selected dimension, so the detailed and top-level views can diverge. If the selected counters do not contain enough information to determine an inclusive total, that total remains unknown even when the top-level view contains a value. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 3c1761b3e..444e2abac 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1046,6 +1046,14 @@ When the descriptor does not declare a report key, ingestion leaves the corresponding facts `unsupported` with reason `key_not_declared`. Result-extension validation runs first: if the schema rejects an emitted undeclared key, the run fails before ingestion and no `RunResult` is returned. + +If you maintain an external descriptor copy for a bundled adapter, update its +`run_result` extension schema before upgrading the adapter implementation. Add +`"nemo_fabric.versions.v1": {"type": "object"}` under `properties`, preserving +existing properties and requirements, and do not add the key to `required`. +Without a schema that accepts the report, invocation fails during +result-extension validation. + A fact the adapter has no source for has state `unsupported` with reason `no_source`. A declared key that the result omits gives `unavailable` with reason `key_omitted`, and a fact the key omits gives `unavailable` with reason diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 15705843c..a8541ec8a 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -230,7 +230,7 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. -Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. +Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. Result-extension validation runs before that: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index 6abc28ea5..9af5d92d2 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -55,7 +55,7 @@ harness version and the served model, read state `unsupported` with reason `key_not_declared` when the adapter descriptor does not declare their report key. When the descriptor declares the key, a result that omits the key reads `unavailable` with reason `key_omitted`, and a key that omits the fact reads -`unavailable` with reason `fact_omitted`. Every bundled +`unavailable` with reason `fact_omitted`. If you maintain an external descriptor copy for a bundled adapter, update its `run_result` extension schema before upgrading the adapter implementation. Add `"nemo_fabric.versions.v1": {"type": "object"}` under `properties`, preserving existing properties and requirements, and do not add the key to `required`. Without a schema that accepts the report, invocation fails during result-extension validation. Every bundled adapter reports its package version. The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters report the harness version, and the Claude Code adapter reports the served model. The Cline adapter reports From a0902a1706f5b2593870a3bed6e890d63d485801 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:08:49 -0700 Subject: [PATCH 101/159] test: check the end line keeps the exchange-end CPU time The warm-runtime CPU test left the fake counter unchanged between the exchange end and finish, so an end line that read its snapshot at finish would still have passed. The host can keep using CPU after the exchange ends, and that time belongs to neither invocation's end line. The test now adds 0.25 s after each exchange end. It asserts that the end lines keep the exchange-end totals of 3.0 s and 3.75 s, and that end minus start stays 2.0 s and 0.5 s although the second start line includes the first invocation's extra 0.25 s. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index b5f68eb7d..fc19855a7 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1409,11 +1409,13 @@ mod tests { } #[test] - fn warm_runtime_end_minus_start_cpu_is_each_invocation_own_cpu_time() { + fn warm_runtime_end_minus_start_cpu_is_each_exchange_own_cpu_time() { use std::sync::atomic::{AtomicU64, Ordering}; // The snapshot's CPU time is cumulative since the host started, so a // warm runtime's second invocation starts with the first one's CPU - // time already counted. Whole milliseconds keep the subtraction exact. + // time already counted. CPU time the host uses after the exchange + // ends and before the result is finished belongs to neither end + // line. Whole and quarter seconds keep the subtraction exact. let host_cpu_ms = Arc::new(AtomicU64::new(1_000)); let read: ReadResources = { let host_cpu_ms = Arc::clone(&host_cpu_ms); @@ -1429,20 +1431,29 @@ mod tests { let (_, _, invocation) = record_test_fixture(); let cpu = |line: &Value| line["resources"]["cpu_time_seconds"].as_f64().expect("cpu"); - let mut own_cpu = Vec::new(); + let (mut end_cpu, mut own_cpu) = (Vec::new(), Vec::new()); for (name, burn_ms) in [("warm-cpu-1", 2_000), ("warm-cpu-2", 500)] { let (folder, mut recorder) = start_recorder(name, &ticker, Arc::clone(&read)); host_cpu_ms.fetch_add(burn_ms, Ordering::SeqCst); recorder.exchange_ended(recorder.read_exchange_end(Instant::now(), &read())); + // The host keeps running between the exchange end and finish. + host_cpu_ms.fetch_add(250, Ordering::SeqCst); recorder .finish(Ok(succeeded_result(&invocation))) .expect("result"); let lines = read_lines(&folder.join(RECORD_FILE_NAME)); let _ = std::fs::remove_dir_all(folder); - own_cpu.push(cpu(lines.last().expect("end line")) - cpu(&lines[0])); + let end = cpu(lines.last().expect("end line")); + end_cpu.push(end); + own_cpu.push(end - cpu(&lines[0])); } - // Not 3.0 and 3.5, the host's totals at each end. + // The end lines keep the exchange-end totals, not 3.25 and 4.0 read + // at finish. The second start line's 3.25 includes the first + // invocation's 0.25 after its exchange, which neither difference + // counts. + assert_eq!(end_cpu, [3.0, 3.75]); + // Not 3.0 and 3.75, the host's totals at each exchange end. assert_eq!(own_cpu, [2.0, 0.5]); } } From 4585ac7ee03f0d055f71fc8dd46eef91778fbd97 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:51:15 -0700 Subject: [PATCH 102/159] fix: report Deep Agents cached input tokens LangChain carries cache reads per model call in usage_metadata.input_token_details.cache_read, and OpenAI raw usage carries them in prompt_tokens_details.cached_tokens, but the adapter never reported cached_input_tokens. Sum cached input per call with the same rule as the other counts: one call without a cache count leaves the sum unknown, and a reported 0 stays 0. Raw OpenAI usage is read first because ChatOpenAI renames the cache key for priority and flex service tiers. Report input_tokens_include_cache=True whenever input tokens are known. LangChain's UsageMetadata.input_tokens counts every input token type; langchain-anthropic adds cache reads and writes back into it, and OpenAI's prompt_tokens already includes cached tokens. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 7 ++ .../deepagents/adapter.py | 30 +++++ tests/adapters/test_deepagents.py | 113 ++++++++++++++++-- 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 66a44eece..9e41e8f53 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -252,6 +252,13 @@ and the provider's total is used when present. Other providers and streamed responses use LangChain's `usage_metadata`, which some integrations fill with 0 for an omitted count. +Cached input tokens come from `prompt_tokens_details.cached_tokens` in +OpenAI-shaped raw usage and from `usage_metadata.input_token_details.cache_read` +otherwise. They follow the same rule: one call without a cache count leaves the +invocation's cached count unknown, and a reported 0 stays 0. When input tokens +are known, the adapter reports that they include cached tokens, because +LangChain's `UsageMetadata.input_tokens` counts every input token type. + ## Runtime Lifecycle NeMo Fabric starts one local adapter host for every runtime. During runtime start, diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 37771f419..ba304d6b3 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -770,6 +770,7 @@ async def invoke( "input_tokens": usage.get("prompt_tokens"), "output_tokens": usage.get("completion_tokens"), "total_tokens": usage.get("total_tokens"), + "cached_input_tokens": usage.get("cached_tokens"), } tokens = { name: value @@ -778,6 +779,11 @@ async def invoke( and not isinstance(value, bool) and 0 <= value <= (1 << 64) - 1 } + if "input_tokens" in tokens: + # LangChain's UsageMetadata defines input_tokens as the sum of every + # input token type, cache reads included; OpenAI's raw prompt_tokens + # includes cached tokens too. + tokens["input_tokens_include_cache"] = True cost = usage.get("cost") if ( not isinstance(cost, (int, float)) @@ -1300,6 +1306,9 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: values = [_reported_tokens(call, source) for call in calls] if all(value is not None for value in values): totals[target] = sum(values) + cached = [_reported_cached_tokens(call) for call in calls] + if all(value is not None for value in cached): + totals["cached_tokens"] = sum(cached) costs = [_reported_cost(call) for call in calls] if all(cost is not None for cost in costs): totals["cost"] = math.fsum(costs) @@ -1342,6 +1351,27 @@ def _reported_tokens(call: dict[str, Any], name: str) -> int | None: return None +def _reported_cached_tokens(call: dict[str, Any]) -> int | None: + """Return one call's cache-read input tokens, or ``None`` when not reported. + + OpenAI-shaped raw usage reports them as ``prompt_tokens_details.cached_tokens``; + ``ChatOpenAI`` renames that key for priority and flex service tiers, so the raw + count is read when present. Other responses use LangChain's + ``usage_metadata.input_token_details.cache_read``. + """ + + metadata = call.get("response_metadata") + raw = metadata.get("token_usage") if isinstance(metadata, dict) else None + if isinstance(raw, dict): + details, key = raw.get("prompt_tokens_details"), "cached_tokens" + else: + usage = call.get("usage") + details = usage.get("input_token_details") if isinstance(usage, dict) else None + key = "cache_read" + value = details.get(key) if isinstance(details, dict) else None + return value if _is_token_count(value) else None + + def _is_token_count(value: Any) -> bool: return ( isinstance(value, int) diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index c916cb2a3..5268f3537 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2604,11 +2604,17 @@ async def test_summarization_model_call_usage_is_counted( assert "tools" not in summary_request assert "tools" in agent_request assert first.usage == AgentUsage( - input_tokens=1500, output_tokens=10, total_tokens=1510 + input_tokens=1500, + output_tokens=10, + total_tokens=1510, + input_tokens_include_cache=True, ) assert second.status is AgentRunStatus.SUCCEEDED assert second.usage == AgentUsage( - input_tokens=1700, output_tokens=65, total_tokens=1765 + input_tokens=1700, + output_tokens=65, + total_tokens=1765, + input_tokens_include_cache=True, ) @@ -2658,7 +2664,10 @@ async def test_subagent_model_calls_are_counted( } assert len(subgraphs) == 2 assert result.usage == AgentUsage( - input_tokens=1111, output_tokens=1111, total_tokens=2222 + input_tokens=1111, + output_tokens=1111, + total_tokens=2222, + input_tokens_include_cache=True, ) @@ -2680,7 +2689,10 @@ async def test_recursion_limit_failure_keeps_usage_of_completed_model_calls( assert result.status is AgentRunStatus.FAILED assert result.error.code == "deepagents_recursion_limit_reached" assert result.usage == AgentUsage( - input_tokens=10 * calls, output_tokens=calls, total_tokens=11 * calls + input_tokens=10 * calls, + output_tokens=calls, + total_tokens=11 * calls, + input_tokens_include_cache=True, ) @@ -2701,7 +2713,12 @@ def provider_failure(kind: str) -> Any: # usage stays known. pytest.param( "400", - AgentUsage(input_tokens=120, output_tokens=8, total_tokens=128), + AgentUsage( + input_tokens=120, + output_tokens=8, + total_tokens=128, + input_tokens_include_cache=True, + ), id="400-rejection", ), # The provider may have done work without reporting usage. @@ -2785,12 +2802,17 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): [ pytest.param( [{"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}], - AgentUsage(input_tokens=0, output_tokens=0, total_tokens=0), + AgentUsage( + input_tokens=0, + output_tokens=0, + total_tokens=0, + input_tokens_include_cache=True, + ), id="explicit-zero", ), pytest.param( [{"prompt_tokens": 100}], - AgentUsage(input_tokens=100), + AgentUsage(input_tokens=100, input_tokens_include_cache=True), id="input-only", ), pytest.param( @@ -2800,7 +2822,12 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): ), pytest.param( [{"prompt_tokens": 100, "completion_tokens": 5}], - AgentUsage(input_tokens=100, output_tokens=5, total_tokens=105), + AgentUsage( + input_tokens=100, + output_tokens=5, + total_tokens=105, + input_tokens_include_cache=True, + ), id="total-omitted", ), pytest.param( @@ -2810,7 +2837,12 @@ async def test_recorder_keeps_a_cancelled_call_unreported(): ), pytest.param( [{"prompt_tokens": 7, "completion_tokens": 5, "total_tokens": 0}], - AgentUsage(input_tokens=7, output_tokens=5, total_tokens=0), + AgentUsage( + input_tokens=7, + output_tokens=5, + total_tokens=0, + input_tokens_include_cache=True, + ), id="provider-total-zero-kept", ), pytest.param( @@ -2839,6 +2871,69 @@ async def test_partial_usage_reports_keep_missing_counts_unknown( assert result.usage == expected +@pytest.mark.parametrize( + ("cached", "expected"), + [ + pytest.param([40, 60], 100, id="every-call-reports-cache"), + pytest.param([40, None], None, id="one-call-omits-cache"), + pytest.param([0, 0], 0, id="explicit-zero"), + ], +) +async def test_cached_input_tokens_are_known_only_when_every_call_reported_them( + tmp_path, make_payload, openai_endpoint, cached, expected +): + def usage(cached_tokens: int | None) -> dict[str, Any]: + reported: dict[str, Any] = { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + } + if cached_tokens is not None: + reported["prompt_tokens_details"] = {"cached_tokens": cached_tokens} + return reported + + openai_endpoint["replies"] += [ + chat_completion(usage(cached[0]), tool_calls=[LS_TOOL_CALL]), + chat_completion(usage(cached[1])), + ] + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage == AgentUsage( + input_tokens=200, + cached_input_tokens=expected, + input_tokens_include_cache=True, + output_tokens=10, + total_tokens=210, + ) + + +@pytest.mark.parametrize( + ("cache_read", "expected"), + [ + pytest.param(30, 30, id="reported"), + pytest.param(0, 0, id="explicit-zero"), + pytest.param(None, None, id="omitted"), + ], +) +def test_anthropic_cache_reads_are_counted_as_included_input(cache_read, expected): + # langchain-anthropic adds cache reads and writes to input_tokens and keeps an + # omitted cache count out of input_token_details. + from anthropic.types import Usage + from langchain_anthropic.chat_models import _create_usage_metadata + + usage_metadata = _create_usage_metadata( + Usage(input_tokens=10, output_tokens=5, cache_read_input_tokens=cache_read) + ) + totals = adapter._aggregate_usage( + [adapter._message_to_dict(AIMessage("done", usage_metadata=usage_metadata))] + ) + + assert totals.get("cached_tokens") == expected + assert totals["prompt_tokens"] == 10 + (cache_read or 0) + + async def test_bad_mcp_transport_fails_runtime_start(tmp_path, make_payload): # A misconfigured MCP server must fail loudly, not be silently dropped. payload = make_payload(tmp_path) From 1ab2640f4f30fa870b62779bd716971a19c88553 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 08:52:15 -0700 Subject: [PATCH 103/159] fix: count only the report's harness-only server drops in core's marker Core subtracted every plan server missing from an MCP report from the report's servers_truncated count, assuming the helper had dropped them. A plan server can be missing for other reasons, so a harness-only drop could vanish from the count. The report marker now counts only servers outside the adapter's start configuration by definition, since a helper knows its configured servers, and core adds the count directly. A plan server the helper drops keeps its entry with fact_omitted facts. No bundled adapter writes the MCP report yet. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 106 +++++++++++++++------ docs/adapter-contract/tutorials/results.md | 2 +- docs/sdk/python.mdx | 9 +- skills/nemo-fabric-build-adapter/SKILL.md | 2 +- 4 files changed, 82 insertions(+), 37 deletions(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index a8ec1873f..6377252ec 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -979,7 +979,7 @@ const REASON_TOO_LARGE: &str = "too_large"; /// the harness name and version, each plan role's served model, each MCP /// server's `runtime` block with its `tool_names_truncated` marker, a /// `harness` server's transport, and the `servers_truncated` marker beside -/// `servers`, whose count core adds to its own only for `harness` entries. A malformed or forged fact reads +/// `servers`, which counts only harness-only servers the helper dropped. A malformed or forged fact reads /// `collection_failed` with reason `invalid_report`, and an omitted one /// reads `unavailable` with reason `fact_omitted`. Core ignores every other /// property, so a report never changes a fact core owns. The string length @@ -1116,11 +1116,12 @@ fn apply_versions_report(what_ran: &mut WhatRan, report: &Report<'_>) { /// Fill each plan server's `runtime` block, and add a `harness` entry for /// each reported server the plan does not configure. /// -/// The report's `servers_truncated` marker counts servers the adapter's -/// helper dropped from the report. A plan server the helper dropped keeps -/// its entry, with facts that read `fact_omitted`, so core adds to its own -/// marker only the count left after the plan servers the report omits: -/// the `harness` entries that disappear from `what_ran`. A malformed marker +/// The report's `servers_truncated` marker counts only the harness-only +/// servers the adapter's helper dropped from the report: servers outside +/// the `mcp.servers` of the adapter's start configuration, which would +/// otherwise disappear from `what_ran`. A plan server the helper dropped +/// keeps its entry, with runtime facts that read `fact_omitted`, and is not +/// counted. Core adds the marker's count to its own. A malformed marker /// makes the whole `servers` report read `invalid_report`, as a malformed /// `tool_names_truncated` does for `tool_names`. fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) { @@ -1139,23 +1140,10 @@ fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) } } } - let plan_servers = &plan.capability_plan.mcp_servers; - if let (Some(marker), Report::Object(reported)) = (helper_marker, &servers) { - let plan_servers_omitted = plan_servers - .keys() - .filter(|name| !reported.contains_key(*name)) - .count() as u64; - let harness_dropped = marker.dropped.saturating_sub(plan_servers_omitted); - if harness_dropped > 0 { - mcp.servers_truncated = merged_marker( - mcp.servers_truncated.take(), - Some(TruncatedEntries { - reason: marker.reason, - dropped: harness_dropped, - }), - ); - } + if let Report::Object(_) = servers { + mcp.servers_truncated = merged_marker(mcp.servers_truncated.take(), helper_marker); } + let plan_servers = &plan.capability_plan.mcp_servers; for (name, server) in &mut mcp.servers { server.runtime = runtime_facts(&servers.child(name).child("runtime")); } @@ -2667,33 +2655,89 @@ mod tests { } #[test] - fn a_helper_servers_marker_never_counts_plan_servers_the_report_omits() { + fn a_helper_servers_marker_counts_harness_drops_whatever_plan_servers_the_report_omits() { let plan = plan_declaring(&[MCP_REPORT_KEY]); - // The helper dropped `github`, a plan server that keeps its entry, - // and two harness entries. + // `github` was never reported, and the helper dropped the harness-only + // `plugin-docs` for its byte budget. let report = json!({ "servers": {"events": connected_runtime(), "files": connected_runtime()}, - "servers_truncated": {"reason": "byte_budget", "dropped": 3} + "servers_truncated": {"reason": "byte_budget", "dropped": 1} }); let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + assert_eq!( + map_keys(&what_ran["mcp"]["servers"]), + ["events", "files", "github"] + ); assert_eq!( what_ran["mcp"]["servers"]["github"]["runtime"]["connected"], report_state("unavailable", "fact_omitted") ); assert_eq!( what_ran["mcp"]["servers_truncated"], - json!({"reason": "byte_budget", "dropped": 2}) + json!({"reason": "byte_budget", "dropped": 1}) ); + } - // A marker that counts only omitted plan servers adds nothing. + #[test] + fn a_helper_servers_marker_adds_to_the_marker_core_already_holds() { + let mut plan = plan_declaring(&[MCP_REPORT_KEY]); + // Core drops this plan server for its long key when it collects. + let server = plan.capability_plan.mcp_servers["files"].clone(); + plan.capability_plan + .mcp_servers + .insert("s".repeat(129), server); let report = json!({ - "servers": {"events": connected_runtime(), "files": connected_runtime()}, - "servers_truncated": {"reason": "byte_budget", "dropped": 1} + "servers": {"events": connected_runtime(), "files": connected_runtime(), + "github": connected_runtime()}, + "servers_truncated": {"reason": "byte_budget", "dropped": 2} }); + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); - assert!(what_ran["mcp"].get("servers_truncated").is_none()); + + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", "dropped": 3}) + ); + } + + #[test] + fn core_entry_cap_and_byte_budget_drops_follow_a_helper_marker() { + // Entry cap: 30 plan servers leave two places for five harness entries. + let plan_names = numbered_keys("server", 30); + let mut report = mcp_report(&numbered_keys("aa", 5), 0); + report[MCP_REPORT_KEY]["servers_truncated"] = + json!({"reason": "byte_budget", "dropped": 2}); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, false), report); + + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "entry_cap", "dropped": 5}) + ); + + // Byte budget: harness entries that do not fit add to the helper's + // count. + let collected = collect(&plan_near_the_byte_budget(&numbered_keys("server", 32))); + let plan_names = map_keys(&collected["mcp"]["servers"]); + let harness_names = numbered_keys("aa", 5); + let mut report = mcp_report(&harness_names, 40); + report[MCP_REPORT_KEY]["servers_truncated"] = + json!({"reason": "byte_budget", "dropped": 4}); + + let (what_ran, _) = ingest(&mcp_plan(&plan_names, true), report); + + let harness_kept = origins(&what_ran) + .iter() + .filter(|(_, origin)| origin == "harness") + .count(); + assert!(harness_kept < harness_names.len()); + assert_eq!( + what_ran["mcp"]["servers_truncated"], + json!({"reason": "byte_budget", + "dropped": 4 + harness_names.len() - harness_kept}) + ); } #[test] diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index ea89dce17..3da1c7241 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -79,7 +79,7 @@ Report only observed invocation usage. For cumulative target counters, compute t `fabric.adapter/v1alpha2` usage is a closed object. Do not add cache-write, reasoning, peak request, or per-model fields at its root. Put that detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` and declare the extension with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized input, cached-input, output, and total token fields and can add `cache_write_input_tokens`, `reasoning_tokens`, `output_tokens_include_reasoning`, `peak_request_input_tokens`, and `models`. Within this object, `input_tokens_include_cache` applies to both cached reads and cache writes. Normalize `input_tokens` first if the provider includes only one component. Each model entry uses the same counters and requires a non-blank `model`; include `provider` when known. -To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. Result-extension validation runs first: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. +To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. Result-extension validation runs first: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. If you drop servers from that report to keep it under 64 KiB, count in its `servers_truncated` marker only the servers missing from your start configuration's `mcp.servers`, which NeMo Fabric adds to `what_ran`'s marker. A plan server you drop keeps its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat the detailed object as authoritative independently for each token dimension. Any of `input_tokens`, `cached_input_tokens`, `cache_write_input_tokens`, or `input_tokens_include_cache` selects the detailed input counters. Any of `output_tokens`, `reasoning_tokens`, or `output_tokens_include_reasoning` selects the detailed output counters. Consumers do not fall back to the top-level counters for a selected dimension, so the detailed and top-level views can diverge. If the selected counters do not contain enough information to determine an inclusive total, that total remains unknown even when the top-level view contains a value. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 444e2abac..6f572a084 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1102,10 +1102,11 @@ roles. A map that lost entries has a marker next to it, `models_truncated` or `byte_budget`) and the `dropped` count. Each map has one marker: when more than one rule dropped entries, `reason` names the last rule applied, in that order, and `dropped` counts every dropped entry. When the adapter's report drops -servers to fit its own size limit, `mcp.servers_truncated` also counts the ones -only the harness reported, because those disappear; a plan server the report -drops keeps its entry, and its runtime facts read `unavailable` with reason -`fact_omitted`. `dropped` is a JSON integer. The Rust core and the Python SDK +servers to fit its own size limit, its `servers_truncated` marker counts only +the servers the harness reported outside the plan, because those disappear, and +`mcp.servers_truncated` adds that count. A plan server the report drops keeps +its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. +`dropped` is a JSON integer. The Rust core and the Python SDK reject `1.0`, although a JSON Schema validator may accept it. If `what_ran` is still too large with no MCP server and no model role left, it holds only `schema`, `version`, and `what_ran_truncated` with diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index a8541ec8a..409393ae0 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -230,7 +230,7 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. -Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. Result-extension validation runs before that: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. +Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. Result-extension validation runs before that: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`. If you drop servers from that report to keep it under 64 KiB, count in its `servers_truncated` marker only the servers missing from your start configuration's `mcp.servers`, which NeMo Fabric adds to `what_ran`'s marker. A plan server you drop keeps its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then From 7501f47c1fecb87525c69202f9d747ded1304dd6 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:10:02 -0700 Subject: [PATCH 104/159] docs: define report server drops against the full capability plan The helper rule counted harness-only drops against the adapter's start configuration, which holds only native MCP servers, while core classifies servers against the full capability plan. Every lifecycle start payload already carries that plan, so the rule now uses it, and the rustdoc, SDK guide, adapter tutorial and adapter authoring skill share one wording. Core behavior is unchanged. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 12 ++++++------ docs/adapter-contract/tutorials/results.md | 2 +- docs/sdk/python.mdx | 11 ++++++----- skills/nemo-fabric-build-adapter/SKILL.md | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 6377252ec..29dd2a3c8 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1116,12 +1116,12 @@ fn apply_versions_report(what_ran: &mut WhatRan, report: &Report<'_>) { /// Fill each plan server's `runtime` block, and add a `harness` entry for /// each reported server the plan does not configure. /// -/// The report's `servers_truncated` marker counts only the harness-only -/// servers the adapter's helper dropped from the report: servers outside -/// the `mcp.servers` of the adapter's start configuration, which would -/// otherwise disappear from `what_ran`. A plan server the helper dropped -/// keeps its entry, with runtime facts that read `fact_omitted`, and is not -/// counted. Core adds the marker's count to its own. A malformed marker +/// When a helper drops server entries to keep `nemo_fabric.mcp.v1` under +/// 64 KiB, the report's `servers_truncated` counts only dropped names absent +/// from the full capability plan supplied at lifecycle start. Core adds that +/// count to `what_ran.mcp.servers_truncated`. A dropped plan server keeps its +/// entry, with runtime facts marked `unavailable` for reason `fact_omitted`. +/// A malformed marker /// makes the whole `servers` report read `invalid_report`, as a malformed /// `tool_names_truncated` does for `tool_names`. fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) { diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 3da1c7241..47e6895c4 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -79,7 +79,7 @@ Report only observed invocation usage. For cumulative target counters, compute t `fabric.adapter/v1alpha2` usage is a closed object. Do not add cache-write, reasoning, peak request, or per-model fields at its root. Put that detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` and declare the extension with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized input, cached-input, output, and total token fields and can add `cache_write_input_tokens`, `reasoning_tokens`, `output_tokens_include_reasoning`, `peak_request_input_tokens`, and `models`. Within this object, `input_tokens_include_cache` applies to both cached reads and cache writes. Normalize `input_tokens` first if the provider includes only one component. Each model entry uses the same counters and requires a non-blank `model`; include `provider` when known. -To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. Result-extension validation runs first: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. If you drop servers from that report to keep it under 64 KiB, count in its `servers_truncated` marker only the servers missing from your start configuration's `mcp.servers`, which NeMo Fabric adds to `what_ran`'s marker. A plan server you drop keeps its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. +To report what ran, put a `nemo_fabric.versions.v1` object in `AgentRunResult.extensions` and name the key, as `{"type": "object"}`, under `properties` in `extension_schemas.run_result`. NeMo Fabric reads the key only when that schema names it literally, then moves it from `RunResult.metadata.adapter` into `RunResult.what_ran`. Result-extension validation runs first: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. The object holds `adapter.package_version`, `harness.name`, `harness.version`, and `models..served`. Each is a fact with `source` `reported`, `collected_by` `adapter`, a `method`, and either a `value` or a `state` and `reason`. Report the version of the harness program that ran, never the version of an SDK that drives it, and report a fact you have no source for as `unsupported` with reason `no_source`. NeMo Fabric marks a fact with another source or collector `collection_failed` with reason `invalid_report`, ignores every other property, and marks every fact of a key larger than 64 KiB `collection_failed` with reason `too_large`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`; its `tool_names` keeps 64 sorted names. If you drop server entries to keep `nemo_fabric.mcp.v1` under 64 KiB, `servers_truncated` counts only dropped names absent from the full capability plan supplied at lifecycle start. Core adds that count to `what_ran.mcp.servers_truncated`. A dropped plan server keeps its entry, with runtime facts marked `unavailable` for reason `fact_omitted`. NeMo Fabric preserves the extension at `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat the detailed object as authoritative independently for each token dimension. Any of `input_tokens`, `cached_input_tokens`, `cache_write_input_tokens`, or `input_tokens_include_cache` selects the detailed input counters. Any of `output_tokens`, `reasoning_tokens`, or `output_tokens_include_reasoning` selects the detailed output counters. Consumers do not fall back to the top-level counters for a selected dimension, so the detailed and top-level views can diverge. If the selected counters do not contain enough information to determine an inclusive total, that total remains unknown even when the top-level view contains a value. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 6f572a084..def66e8d3 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1101,11 +1101,12 @@ roles. A map that lost entries has a marker next to it, `models_truncated` or `mcp.servers_truncated`, with the `reason` (`key_too_long`, `entry_cap`, or `byte_budget`) and the `dropped` count. Each map has one marker: when more than one rule dropped entries, `reason` names the last rule applied, in that order, -and `dropped` counts every dropped entry. When the adapter's report drops -servers to fit its own size limit, its `servers_truncated` marker counts only -the servers the harness reported outside the plan, because those disappear, and -`mcp.servers_truncated` adds that count. A plan server the report drops keeps -its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. +and `dropped` counts every dropped entry. When an adapter drops server entries +to keep `nemo_fabric.mcp.v1` under 64 KiB, the report's `servers_truncated` +counts only dropped names absent from the full capability plan supplied at +lifecycle start. NeMo Fabric adds that count to +`what_ran.mcp.servers_truncated`. A dropped plan server keeps its entry, with +runtime facts marked `unavailable` for reason `fact_omitted`. `dropped` is a JSON integer. The Rust core and the Python SDK reject `1.0`, although a JSON Schema validator may accept it. If `what_ran` is still too large with no MCP server and no model role left, it holds only diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 409393ae0..d1b9cbcb8 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -230,7 +230,7 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. -Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. Result-extension validation runs before that: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`. If you drop servers from that report to keep it under 64 KiB, count in its `servers_truncated` marker only the servers missing from your start configuration's `mcp.servers`, which NeMo Fabric adds to `what_ran`'s marker. A plan server you drop keeps its entry, and its runtime facts read `unavailable` with reason `fact_omitted`. +Report the adapter package version, the harness name and version, and served models in `AgentRunResult.extensions["nemo_fabric.versions.v1"]`, and name that key as `{"type": "object"}` under `properties` in `extension_schemas.run_result`. Each fact carries `source` `reported`, `collected_by` `adapter`, a `method`, and a `value` or a `state` and `reason`. Report the harness program's version, never the driving SDK's, and use `unsupported` with reason `no_source` when no source exists. NeMo Fabric moves the key into `RunResult.what_ran`. Result-extension validation runs before that: a result that carries the key fails the invocation when `run_result` is missing or its schema rejects the key, so declare the key before you emit it, and keep it out of `required`. `nemo_fabric.mcp.v1` follows the same rules for `servers..runtime`. If you drop server entries to keep `nemo_fabric.mcp.v1` under 64 KiB, `servers_truncated` counts only dropped names absent from the full capability plan supplied at lifecycle start. Core adds that count to `what_ran.mcp.servers_truncated`. A dropped plan server keeps its entry, with runtime facts marked `unavailable` for reason `fact_omitted`. Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then From fee4c9287832d1fd24664bfa1848ac784af66bad Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:30:54 -0700 Subject: [PATCH 105/159] fix: report zero Deep Agents usage when no model call ran The adapter drops a model call that the provider rejects with a 4xx status other than 408, because the provider spent no tokens on it. When every call was rejected, or no call started, the call list was empty and the usage was reported as unknown, although the invocation provably spent no tokens. The adapter now reports 0 for every token count and the cost in that case. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 15 +++++----- .../deepagents/adapter.py | 6 ++-- tests/adapters/test_deepagents.py | 29 +++++++++++++++++++ 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 9e41e8f53..e88b41c04 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -237,13 +237,14 @@ summarization. Agent Protocol subagents (entries with `graph_id`) run in their remote service, and their tokens are not included. A token count or cost is reported only when every counted call reported it; otherwise it stays unknown. This also holds for a failed invocation. A request the provider rejects with a -4xx status (other than 408) contributes no usage. A 5xx response, timeout, -cancellation or unfinished call leaves the invocation's counts and cost -unknown, because the provider may have done work without reporting usage. -OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries -to success is not visible to the adapter, and only the final response's -reported usage counts. Usage replayed from a configured LangChain model cache -would be counted as reported; NeMo Fabric configures no such cache. +4xx status (other than 408) contributes no usage, so an invocation that made no +model call, or whose every call was rejected, reports 0 tokens and 0 cost. A +5xx response, timeout, cancellation or unfinished call leaves the invocation's +counts and cost unknown, because the provider may have done work without +reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx +the SDK retries to success is not visible to the adapter, and only the final +response's reported usage counts. Usage replayed from a configured LangChain +model cache would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index ba304d6b3..e5561f215 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1292,11 +1292,11 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: """Sum model-call usage; a count is known only when every call reported it. One call without a count makes the invocation's sum for that count a - partial value, so it stays unknown rather than under-reported. + partial value, so it stays unknown rather than under-reported. Without a + call, because none started or the provider rejected every one, the + invocation spent nothing, so every count and the cost are 0. """ - if not calls: - return None totals: dict[str, Any] = {} for source, target in ( ("input_tokens", "prompt_tokens"), diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 5268f3537..6cdb0f466 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2744,6 +2744,35 @@ async def test_failed_model_call_usage_follows_provider_response( assert result.usage == expected +async def test_rejected_only_model_call_reports_zero_usage( + tmp_path, make_payload, openai_endpoint +): + # The provider rejected the only model call, so the invocation spent nothing. + openai_endpoint["replies"].append(provider_failure("400")) + + (result,) = await run_invocations(make_payload(tmp_path), 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage == AgentUsage( + input_tokens=0, + cached_input_tokens=0, + output_tokens=0, + total_tokens=0, + input_tokens_include_cache=True, + cost_usd=0.0, + ) + + +def test_aggregate_usage_without_model_calls_is_zero(): + assert adapter._aggregate_usage([]) == { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "cached_tokens": 0, + "cost": 0.0, + } + + @pytest.mark.parametrize( ("failure", "expected"), [ From 2ee128d7485f15db1a5b0c66f98cbfdb04b30277 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:31:49 -0700 Subject: [PATCH 106/159] fix: report unknown Deep Agents usage with an Agent Protocol subagent Deep Agents routes a subagent spec with graph_id to its async subagent middleware, which starts and updates tasks on a remote Agent Protocol server. Those model calls never reach the adapter's callbacks, so the reported counts left them out while appearing complete. The adapter blocked such subagents only when a tools policy was configured. When the configured subagents include one with graph_id, the adapter now reports the invocation's token counts and cost as unknown. The local sums stay in output.usage. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 24 ++++++++-------- .../deepagents/adapter.py | 25 ++++++++++++++++- tests/adapters/test_deepagents.py | 28 +++++++++++++++++++ 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index e88b41c04..56f93989d 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -234,17 +234,19 @@ runtime start before an invocation is accepted. Usage sums every chat-model call that runs in the adapter process for the invocation: the main agent, declarative subagents, and Deep Agents summarization. Agent Protocol subagents (entries with `graph_id`) run in their -remote service, and their tokens are not included. A token count or cost is -reported only when every counted call reported it; otherwise it stays unknown. -This also holds for a failed invocation. A request the provider rejects with a -4xx status (other than 408) contributes no usage, so an invocation that made no -model call, or whose every call was rejected, reports 0 tokens and 0 cost. A -5xx response, timeout, cancellation or unfinished call leaves the invocation's -counts and cost unknown, because the provider may have done work without -reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx -the SDK retries to success is not visible to the adapter, and only the final -response's reported usage counts. Usage replayed from a configured LangChain -model cache would be counted as reported; NeMo Fabric configures no such cache. +remote service, where the adapter cannot count their calls, so a runtime +configured with one reports unknown token counts and cost. A token count or +cost is reported only when every counted call reported it; otherwise it stays +unknown. This also holds for a failed invocation. A request the provider +rejects with a 4xx status (other than 408) contributes no usage, so an +invocation that made no model call, or whose every call was rejected, reports 0 +tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call +leaves the invocation's counts and cost unknown, because the provider may have +done work without reporting usage. OpenAI SDK retries happen inside one +LangChain call, so a 5xx the SDK retries to success is not visible to the +adapter, and only the final response's reported usage counts. Usage replayed +from a configured LangChain model cache would be counted as reported; NeMo +Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index e5561f215..38a0eacbf 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -541,6 +541,18 @@ def _validated_deepagents_settings(extra: Any) -> dict[str, Any]: return dict(extra) +def _has_agent_protocol_subagent(subagents: Any) -> bool: + """Whether ``create_deep_agent`` runs a subagent on an Agent Protocol server. + + Deep Agents routes a subagent spec with ``graph_id`` to its async subagent + middleware, which starts and updates tasks on a remote server. + """ + return isinstance(subagents, list) and any( + isinstance(subagent, dict) and "graph_id" in subagent + for subagent in subagents + ) + + def _gate_subagent( subagent: dict[str, Any], enabled: set[str] | None, blocked: set[str] ) -> dict[str, Any]: @@ -612,6 +624,7 @@ def __init__(self) -> None: self._callback_handler_type: Any = None self._telemetry_quarantine: str | None = None self._telemetry_quarantine_cause: str | None = None + self._has_remote_subagent = False async def start(self, payload: dict[str, Any]) -> None: if self._started: @@ -662,6 +675,9 @@ async def start(self, payload: dict[str, Any]) -> None: model, settings, ) + self._has_remote_subagent = _has_agent_protocol_subagent( + agent_kwargs.get("subagents") + ) self._checkpointer = await open_checkpointer( checkpointer_path(runtime_context, base_dir, runtime_id) ) @@ -765,7 +781,14 @@ async def invoke( failed = bool(output.pop("failed", False)) reported_error = output.pop("error", None) raw_usage = output.get("usage") - usage = raw_usage if isinstance(raw_usage, dict) else {} + # An Agent Protocol subagent's model calls run on its remote server and + # never reach this process, so the local sums may undercount the + # invocation and its counts and cost stay unknown. + usage = ( + raw_usage + if isinstance(raw_usage, dict) and not self._has_remote_subagent + else {} + ) token_fields = { "input_tokens": usage.get("prompt_tokens"), "output_tokens": usage.get("completion_tokens"), diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 6cdb0f466..873126845 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2763,6 +2763,34 @@ async def test_rejected_only_model_call_reports_zero_usage( ) +async def test_agent_protocol_subagent_makes_usage_unknown( + tmp_path, make_payload, openai_endpoint +): + # An Agent Protocol subagent runs its model calls on a remote server, which + # the adapter cannot count, so local sums would undercount the invocation. + payload = make_payload(tmp_path) + payload["config"]["harness"]["settings"]["deepagents"] = { + "subagents": [ + { + "name": "researcher", + "description": "Researches on a remote server.", + "graph_id": "research_agent", + "url": "https://agents.example.test", + } + ] + } + openai_endpoint["replies"].append( + chat_completion( + {"prompt_tokens": 120, "completion_tokens": 8, "total_tokens": 128} + ) + ) + + (result,) = await run_invocations(payload, 1) + + assert result.status is AgentRunStatus.SUCCEEDED + assert result.usage is None + + def test_aggregate_usage_without_model_calls_is_zero(): assert adapter._aggregate_usage([]) == { "prompt_tokens": 0, From 06de4c0281439b64a9d05340ebca771278fe5743 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:32:07 -0700 Subject: [PATCH 107/159] docs: say a failed Deep Agents call may have been billed The usage notes said that after a 5xx response, timeout, or cancelled call the provider may have done work. The reason the counts stay unknown is narrower: the provider may have billed the call without reporting usage. The README, the recorder docstring, and a test comment now say that. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 2 +- .../deepagents/src/nemo_fabric_adapters/deepagents/adapter.py | 4 ++-- tests/adapters/test_deepagents.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index 56f93989d..c1a5a8c98 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -242,7 +242,7 @@ rejects with a 4xx status (other than 408) contributes no usage, so an invocation that made no model call, or whose every call was rejected, reports 0 tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call leaves the invocation's counts and cost unknown, because the provider may have -done work without reporting usage. OpenAI SDK retries happen inside one +billed the call without reporting usage. OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries to success is not visible to the adapter, and only the final response's reported usage counts. Usage replayed from a configured LangChain model cache would be counted as reported; NeMo diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 38a0eacbf..f886155c9 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1012,8 +1012,8 @@ class ModelCallUsage(BaseCallbackHandler): record when it ends. A request the provider rejects with a 4xx status (other than 408) contributes no usage and is dropped. A 5xx response, timeout, cancellation or unfinished call stays unreported and makes the invocation's - counts and cost unknown, because the provider may have done work without - reporting usage. + counts and cost unknown, because the provider may have billed the call + without reporting usage. """ run_inline = True diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index 873126845..eec895d6a 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2721,7 +2721,7 @@ def provider_failure(kind: str) -> Any: ), id="400-rejection", ), - # The provider may have done work without reporting usage. + # The provider may have billed the call without reporting usage. pytest.param("408", None, id="408-request-timeout"), pytest.param("503", None, id="503-server-error"), pytest.param("timeout", None, id="read-timeout"), From e535051318613392a86fcb75cd4a469cedd21b18 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:38:36 -0700 Subject: [PATCH 108/159] style: format the Deep Agents remote subagent check Apply ruff format to the check added in the previous commit. Signed-off-by: mnajafian-nv --- .../deepagents/src/nemo_fabric_adapters/deepagents/adapter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index f886155c9..9a51bd817 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -548,8 +548,7 @@ def _has_agent_protocol_subagent(subagents: Any) -> bool: middleware, which starts and updates tasks on a remote server. """ return isinstance(subagents, list) and any( - isinstance(subagent, dict) and "graph_id" in subagent - for subagent in subagents + isinstance(subagent, dict) and "graph_id" in subagent for subagent in subagents ) From cb42bfe7d37d76beb93d1c0d8aa7f6bf61edda53 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Tue, 20 Oct 2026 09:45:54 -0700 Subject: [PATCH 109/159] test: keep zero observed Deep Agents calls unknown with a remote subagent The adapter reports 0 when its callbacks observed no billed model call. That 0 holds for the invocation only when every model call runs through those callbacks. Deep Agents runs declarative subagents and summarization inside the graph, where the run config's callbacks reach them, but an Agent Protocol subagent (graph_id) runs on a remote server that the callbacks never see. The adapter already reports unknown usage for a runtime with such a subagent; this test pins that for an invocation whose only observed call was rejected, and the docstring and README state the condition. Signed-off-by: mnajafian-nv --- adapters/python/deepagents/README.md | 18 +++++++------- .../deepagents/adapter.py | 5 +++- tests/adapters/test_deepagents.py | 24 +++++++++++++++++++ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index c1a5a8c98..6340b4ce3 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -238,15 +238,15 @@ remote service, where the adapter cannot count their calls, so a runtime configured with one reports unknown token counts and cost. A token count or cost is reported only when every counted call reported it; otherwise it stays unknown. This also holds for a failed invocation. A request the provider -rejects with a 4xx status (other than 408) contributes no usage, so an -invocation that made no model call, or whose every call was rejected, reports 0 -tokens and 0 cost. A 5xx response, timeout, cancellation or unfinished call -leaves the invocation's counts and cost unknown, because the provider may have -billed the call without reporting usage. OpenAI SDK retries happen inside one -LangChain call, so a 5xx the SDK retries to success is not visible to the -adapter, and only the final response's reported usage counts. Usage replayed -from a configured LangChain model cache would be counted as reported; NeMo -Fabric configures no such cache. +rejects with a 4xx status (other than 408) contributes no usage, so without an +Agent Protocol subagent an invocation that made no model call, or whose every +call was rejected, reports 0 tokens and 0 cost. A 5xx response, timeout, +cancellation or unfinished call leaves the invocation's counts and cost +unknown, because the provider may have billed the call without reporting usage. +OpenAI SDK retries happen inside one LangChain call, so a 5xx the SDK retries +to success is not visible to the adapter, and only the final response's +reported usage counts. Usage replayed from a configured LangChain model cache +would be counted as reported; NeMo Fabric configures no such cache. When a response carries OpenAI-shaped usage in `response_metadata.token_usage`, as `ChatOpenAI` responses without streaming diff --git a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py index 9a51bd817..bfda396dc 100644 --- a/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py +++ b/adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py @@ -1316,7 +1316,10 @@ def _aggregate_usage(calls: list[dict[str, Any]]) -> dict[str, Any] | None: One call without a count makes the invocation's sum for that count a partial value, so it stays unknown rather than under-reported. Without a call, because none started or the provider rejected every one, the - invocation spent nothing, so every count and the cost are 0. + observed calls spent nothing, so every count and the cost are 0. That 0 + covers the invocation only when every model call runs through these + callbacks; the caller makes usage unknown when an Agent Protocol subagent + can run models on a remote server. """ totals: dict[str, Any] = {} diff --git a/tests/adapters/test_deepagents.py b/tests/adapters/test_deepagents.py index eec895d6a..816ccb1ec 100644 --- a/tests/adapters/test_deepagents.py +++ b/tests/adapters/test_deepagents.py @@ -2791,6 +2791,30 @@ async def test_agent_protocol_subagent_makes_usage_unknown( assert result.usage is None +async def test_rejected_only_call_with_agent_protocol_subagent_is_unknown( + tmp_path, make_payload, openai_endpoint +): + # Zero observed calls prove nothing when a remote subagent can run models + # that the adapter's callbacks never see. + payload = make_payload(tmp_path) + payload["config"]["harness"]["settings"]["deepagents"] = { + "subagents": [ + { + "name": "researcher", + "description": "Researches on a remote server.", + "graph_id": "research_agent", + "url": "https://agents.example.test", + } + ] + } + openai_endpoint["replies"].append(provider_failure("400")) + + (result,) = await run_invocations(payload, 1) + + assert result.status is AgentRunStatus.FAILED + assert result.usage is None + + def test_aggregate_usage_without_model_calls_is_zero(): assert adapter._aggregate_usage([]) == { "prompt_tokens": 0, From ce744150ea1e26be5b6b8a42a26d95b8a533d77f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:04:44 -0700 Subject: [PATCH 110/159] fix: preserve empty OpenShell environment arguments Signed-off-by: mnajafian-nv --- examples/openshell/run-demo.sh | 4 ++- tests/scripts/test_openshell_launcher.py | 44 +++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/examples/openshell/run-demo.sh b/examples/openshell/run-demo.sh index 56d2b534f..4fe36156e 100644 --- a/examples/openshell/run-demo.sh +++ b/examples/openshell/run-demo.sh @@ -272,9 +272,11 @@ run_consumer() { if [[ "${EXAMPLE_MODE}" == caller_owned || "${EXAMPLE_MODE}" == both ]]; then CALLER_SANDBOX_ATTEMPTED=1 + # Guard empty arrays for Bash 3.2's set -u behavior without adding empty args. "${OPENSHELL_CLI}" --gateway-endpoint "http://127.0.0.1:${GATEWAY_PORT}" \ --workspace default sandbox create --name "${SANDBOX_NAME}" --from "${RUNTIME_IMAGE}" \ - --policy "${POLICY_FILE}" "${MODEL_ENV_ARGS[@]}" "${SERVER_ENV_ARGS[@]}" \ + --policy "${POLICY_FILE}" ${MODEL_ENV_ARGS[@]+"${MODEL_ENV_ARGS[@]}"} \ + ${SERVER_ENV_ARGS[@]+"${SERVER_ENV_ARGS[@]}"} \ --detach --no-tty -- fabric-agent-ctl serve SANDBOX_JSON="$("${OPENSHELL_CLI}" --gateway-endpoint "http://127.0.0.1:${GATEWAY_PORT}" \ --workspace default sandbox get "${SANDBOX_NAME}" --output json)" diff --git a/tests/scripts/test_openshell_launcher.py b/tests/scripts/test_openshell_launcher.py index 317b8052d..15ab68caa 100644 --- a/tests/scripts/test_openshell_launcher.py +++ b/tests/scripts/test_openshell_launcher.py @@ -12,6 +12,7 @@ import json import os from pathlib import Path +import shlex import shutil import socket import subprocess @@ -172,7 +173,22 @@ def _run(launcher, workload, env): ) -def test_generic_workload_gets_ownership_and_consumer_args(launcher): +@pytest.mark.parametrize( + ("model_env_args", "server_env_args"), + [ + pytest.param([], None, id="default-server-env"), + pytest.param([], [], id="no-env"), + pytest.param(["--env", "MODEL_HINT=words with spaces"], [], id="no-server-env"), + pytest.param( + ["--env", "MODEL_HINT=words with spaces"], + ["--env", "SERVER_HINT=words with spaces"], + id="quoted-env", + ), + ], +) +def test_generic_workload_gets_ownership_and_consumer_args( + launcher, model_env_args, server_env_args +): _, root, _ = launcher workload = root / "third-workload" workload.mkdir() @@ -185,6 +201,12 @@ def test_generic_workload_gets_ownership_and_consumer_args(launcher): stage_workload_image() { :; } run_workload_consumer() { local mode="$1"; shift; uv run --no-sync consumer "$mode" "$@"; } """ + + f"MODEL_ENV_ARGS=({shlex.join(model_env_args)})\n" + + ( + f"SERVER_ENV_ARGS=({shlex.join(server_env_args)})\n" + if server_env_args is not None + else "" + ) ) completed = _run(launcher, hook, {"OPENSHELL_EXAMPLE_MODE": "both"}) @@ -195,6 +217,26 @@ def test_generic_workload_gets_ownership_and_consumer_args(launcher): assert all(call["args"][-2:] == ["--input", "words with spaces"] for call in calls) assert all(call["provider"].endswith("/fabric-environment-openshell") for call in calls) assert not (root / "sandbox-state.json").exists() + create = next( + operation["args"] + for operation in _operations(root) + if operation["command"] == "openshell" + and "sandbox" in operation["args"] + and "create" in operation["args"] + ) + expected_server_env = ( + server_env_args + if server_env_args is not None + else [ + "--env", + "ADAPTER_PYTHON=/opt/fabric/bin/python", + "--env", + "TMPDIR=/sandbox/tmp", + ] + ) + assert create[create.index("--policy") + 2 : create.index("--detach")] == ( + model_env_args + expected_server_env + ) @pytest.mark.parametrize("mode", ["caller_owned", "fabric_owned"]) From 790f3693aaebf9465bc224bd76da41fb3d81d420 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:10:57 -0700 Subject: [PATCH 111/159] fix: avoid premature Relay gateway startup failures Signed-off-by: mnajafian-nv --- adapters/python/claude/README.md | 2 +- adapters/python/codex/README.md | 4 +- adapters/python/common/README.md | 14 +++++ .../common/relay_gateway.py | 21 +++++-- tests/_utils/utils.py | 22 +++++-- tests/adapters/test_adapaters_common_utils.py | 4 +- .../test_adapters_common_relay_gateway.py | 63 +++++++++++++++++++ 7 files changed, 117 insertions(+), 13 deletions(-) diff --git a/adapters/python/claude/README.md b/adapters/python/claude/README.md index a6b8ac271..bfa13847c 100644 --- a/adapters/python/claude/README.md +++ b/adapters/python/claude/README.md @@ -169,7 +169,7 @@ config.enable_relay( ``` For each Relay-enabled Claude runtime, NeMo Fabric starts one `nemo-relay` gateway, -waits for its health endpoint, and stops it with the runtime. NeMo Fabric passes the +waits up to 30 seconds for its health endpoint, and stops it with the runtime. NeMo Fabric passes the gateway URL to the connected Claude Code process through `ANTHROPIC_BASE_URL` and `NEMO_RELAY_GATEWAY_URL`, and passes the selected explicit model endpoint to the gateway as its Anthropic upstream. It also stages a runtime-scoped Claude diff --git a/adapters/python/codex/README.md b/adapters/python/codex/README.md index 3cf1de779..ba7eab1b7 100644 --- a/adapters/python/codex/README.md +++ b/adapters/python/codex/README.md @@ -154,7 +154,9 @@ Codex adapter does not provide a separate `relay` extra; its `harness` and Enable Relay with `FabricConfig.enable_relay(...)`. The adapter starts the installed `nemo-relay` CLI as a supervised sidecar; do not start the gateway -separately. +separately. Gateway readiness has a 30-second budget covering configuration +validation and required worker initialization. Startup fails on early process exit +or an expired budget, and the gateway log is retained for diagnosis. NeMo Fabric routes the selected Responses-compatible provider through the gateway and passes its explicit `base_url` to Relay as the upstream endpoint. diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index 17e537cd5..fb8400337 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -122,6 +122,20 @@ state during `start`. Adapter stdout is reserved for the protocol; diagnostics are redirected to stderr. A host crash or protocol timeout terminates that runtime. +## Relay Gateway Readiness + +The Claude and Codex adapters wait up to 30 seconds for the supervised Relay +gateway's HTTP health endpoint. This budget includes configuration validation +and required worker initialization. Early process exit fails startup immediately; +a gateway that never becomes healthy fails when the budget expires and is stopped. +The adapters retain its log for diagnosis. Local Python hosts separately enforce +a 90-second adapter startup limit. + +Relay 0.10.3 includes an +[upstream startup-memory fix](https://github.com/NVIDIA/NeMo-Relay/pull/1375). +Use that release to obtain the fix on 0.10 installations. Fabric continues to +accept the supported `>=0.9,<0.11` CLI range. + ## Relay Request Correlation In-process Relay SDK adapters that own their Agent scope can use diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 45cf04faa..959f54fa7 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -20,7 +20,10 @@ from nemo_fabric_adapters.common.relay_version import is_supported_relay_version -RELAY_HEALTH_TIMEOUT_SECONDS = 10.0 +# Readiness includes plugin preflight and worker startup before health is served. +# This operational budget is separate from Relay's inner request timeouts and +# Fabric's outer adapter-start limit. +RELAY_STARTUP_TIMEOUT_SECONDS = 30.0 RELAY_STOP_TIMEOUT_SECONDS = 5.0 RELAY_VERSION_TIMEOUT_SECONDS = 5.0 @@ -110,24 +113,32 @@ def wait_for_relay_gateway( process: subprocess.Popen[Any], health_url: str, *, - timeout: float = RELAY_HEALTH_TIMEOUT_SECONDS, + timeout: float = RELAY_STARTUP_TIMEOUT_SECONDS, ) -> None: """Wait until the Relay health endpoint succeeds or startup fails.""" deadline = time.monotonic() + timeout - while time.monotonic() < deadline: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break returncode = process.poll() if returncode is not None: raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) try: - with urllib.request.urlopen(health_url, timeout=1) as response: + with urllib.request.urlopen( + health_url, timeout=min(1.0, remaining) + ) as response: if 200 <= response.status < 300: return except (OSError, urllib.error.URLError): pass - time.sleep(0.1) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(0.1, remaining)) raise RelayGatewayError(f"NeMo Relay gateway did not become ready at {health_url}") diff --git a/tests/_utils/utils.py b/tests/_utils/utils.py index 9b7ca63be..aa5521ae8 100644 --- a/tests/_utils/utils.py +++ b/tests/_utils/utils.py @@ -22,11 +22,17 @@ def assert_atof_skill_selection( output: Mapping[str, Any], expected_skill: str | None ) -> None: records = atof_records(output) - tool_records = [record for record in records if record["category"] == "tool"] + tool_records = [ + record + for record in records + if record["kind"] == "scope" and record["category"] == "tool" + ] llm_end_records = [ record for record in records - if record["category"] == "llm" and record["scope_category"] == "end" + if record["kind"] == "scope" + and record["category"] == "llm" + and record["scope_category"] == "end" ] serialized_tool_records = json.dumps(tool_records).replace("\\\\", "/") serialized_skill_calls = json.dumps(tool_records + llm_end_records).replace( @@ -55,7 +61,9 @@ def assert_atof_model(output: Mapping[str, Any], expected_model: str) -> None: llm_starts = [ record for record in events - if record["category"] == "llm" and record["scope_category"] == "start" + if record["kind"] == "scope" + and record["category"] == "llm" + and record["scope_category"] == "start" ] assert llm_starts, events assert {record["data"]["content"]["model"] for record in llm_starts} == { @@ -81,7 +89,9 @@ def assert_semantic_relay_artifacts( llm_starts = [ event for event in events - if event.get("category") == "llm" and event.get("scope_category") == "start" + if event["kind"] == "scope" + and event["category"] == "llm" + and event["scope_category"] == "start" ] assert llm_starts, events assert all( @@ -92,7 +102,9 @@ def assert_semantic_relay_artifacts( llm_ends = [ event for event in events - if event.get("category") == "llm" and event.get("scope_category") == "end" + if event["kind"] == "scope" + and event["category"] == "llm" + and event["scope_category"] == "end" ] assert llm_ends, events assert any(_relay_event_total_tokens(event) > 0 for event in llm_ends) diff --git a/tests/adapters/test_adapaters_common_utils.py b/tests/adapters/test_adapaters_common_utils.py index d1805f12b..0be298052 100644 --- a/tests/adapters/test_adapaters_common_utils.py +++ b/tests/adapters/test_adapaters_common_utils.py @@ -1332,8 +1332,10 @@ async def test_relay_initializes_v4_atof_atif_config( } common_utils.validate_relay_observability(plugin_config) assert plugin.validate_exact(plugin_config)["config"]["diagnostics"] == [] + # Activation includes discovered host plugins, so compare the same layers. + expected_diagnostics = plugin.validate(plugin_config)["config"]["diagnostics"] async with plugin.activate(plugin_config) as activation: - assert activation.report["config"]["diagnostics"] == [] + assert activation.report["config"]["diagnostics"] == expected_diagnostics def test_relay_validates_all_v4_otlp_fields(): diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index c307254ae..e478f85de 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -271,3 +271,66 @@ def test_stop_relay_gateway_is_idempotent_for_exited_process(): process.terminate.assert_not_called() process.kill.assert_not_called() + + +@pytest.fixture(name="gateway_clock") +def gateway_clock_fixture(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + mock_clock = MagicMock(spec=relay_gateway.time) + mock_clock.monotonic.return_value = 0.0 + + def advance(seconds: float) -> None: + mock_clock.monotonic.return_value += seconds + + mock_clock.sleep.side_effect = advance + monkeypatch.setattr(relay_gateway, "time", mock_clock) + return mock_clock + + +def test_wait_for_relay_gateway_allows_preflight_before_worker_readiness( + monkeypatch, gateway_clock +): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += timeout + if gateway_clock.monotonic() < 14.0: + raise relay_gateway.urllib.error.URLError("still initializing") + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") + + assert 14.0 <= gateway_clock.monotonic() < 30.0 + + +@pytest.mark.parametrize( + ("probe_elapsed", "expected_sleep"), [(0.25, None), (0.20, 0.05)] +) +def test_wait_for_relay_gateway_limits_each_probe_to_remaining_startup_budget( + monkeypatch, gateway_clock, probe_elapsed, expected_sleep +): + process = MagicMock() + process.poll.return_value = None + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += probe_elapsed + raise relay_gateway.urllib.error.URLError("not ready") + + mock_probe = MagicMock(side_effect=probe_health) + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", mock_probe) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + mock_probe.assert_called_once_with("http://127.0.0.1:43210/healthz", timeout=0.25) + if expected_sleep is None: + gateway_clock.sleep.assert_not_called() + else: + gateway_clock.sleep.assert_called_once_with(pytest.approx(expected_sleep)) + assert gateway_clock.monotonic() == pytest.approx(0.25) From 5db49175c27b37f704350cfbfa6fc51bdba1fd0e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 13:41:46 -0700 Subject: [PATCH 112/159] docs: use the NeMo Fabric product name Signed-off-by: mnajafian-nv --- adapters/python/common/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index fb8400337..919bdd233 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -133,7 +133,7 @@ a 90-second adapter startup limit. Relay 0.10.3 includes an [upstream startup-memory fix](https://github.com/NVIDIA/NeMo-Relay/pull/1375). -Use that release to obtain the fix on 0.10 installations. Fabric continues to +Use that release to obtain the fix on 0.10 installations. NeMo Fabric continues to accept the supported `>=0.9,<0.11` CLI range. ## Relay Request Correlation From 06369c42bf3451c2179c920ee1dde9ace0dbdc7c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 14:09:20 -0700 Subject: [PATCH 113/159] feat: estimate model cost with versioned pricing Signed-off-by: mnajafian-nv --- Cargo.lock | 1 + Cargo.toml | 2 + adapters/python/deepagents/README.md | 4 +- crates/fabric-cli/src/presets.rs | 1 + crates/fabric-cli/templates/rust/main.rs.tmpl | 1 + crates/fabric-core/Cargo.toml | 1 + crates/fabric-core/data/model-pricing.json | 71 ++ crates/fabric-core/src/config.rs | 48 + crates/fabric-core/src/lib.rs | 10 +- crates/fabric-core/src/model_usage.rs | 390 +----- crates/fabric-core/src/price_catalog.rs | 1128 +++++++++++++++++ crates/fabric-core/src/pricing.rs | 683 +++++++++- crates/fabric-core/src/runtime.rs | 89 +- crates/fabric-core/src/usage_normalization.rs | 6 +- crates/fabric-core/tests/model_usage.rs | 521 ++++++++ docs/adapter-contract/tutorials/results.md | 2 +- .../api/python-library-reference/index.md | 1 + .../nemo_fabric.models.md | 185 +-- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../config/enum-capabilitykind.mdx | 34 +- .../config/enum-capabilitytarget.mdx | 34 +- .../config/enum-relayatifstorageconfig.mdx | 32 +- .../config/enum-relayatofmode.mdx | 38 +- .../config/enum-relayatofsinkconfig.mdx | 32 +- .../enum-relayatofstreamfieldnamepolicy.mdx | 38 +- .../config/enum-relayatofstreamtransport.mdx | 38 +- .../enum-relayopentelemetryendpointtype.mdx | 34 +- .../enum-relayopentelemetrymarkprojection.mdx | 38 +- .../config/enum-relayotlptransport.mdx | 38 +- .../config/enum-relayunsupportedbehavior.mdx | 38 +- .../config/enum-resolutionstrategy.mdx | 34 +- .../config/enum-telemetryprovider.mdx | 62 +- .../config/fn-load-adapter-descriptor.mdx | 4 +- .../fn-load-adapter-target-descriptor.mdx | 4 +- .../fn-resolve-run-plan-from-config.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 3 +- .../config/struct-fabricconfig.mdx | 34 +- .../config/struct-pricingconfig.mdx | 106 ++ .../config/struct-relayatifconfig.mdx | 36 +- .../config/struct-relayatofconfig.mdx | 36 +- .../config/struct-relaycomponentconfig.mdx | 32 +- .../config/struct-relayconfig.mdx | 36 +- .../config/struct-relayconfigpolicy.mdx | 34 +- .../struct-relayobservabilityconfig.mdx | 36 +- .../struct-relayopentelemetryconfig.mdx | 36 +- ...truct-relayopentelemetryendpointconfig.mdx | 32 +- .../config/struct-resolvecontext.mdx | 24 +- .../struct-resolvedadapterdescriptor.mdx | 32 +- ...struct-resolvedadaptertargetdescriptor.mdx | 32 +- .../config/struct-runplan.mdx | 36 +- .../config/struct-runtimecapabilities.mdx | 36 +- .../config/struct-runtimeconfig.mdx | 32 +- .../config/struct-skillconfig.mdx | 36 +- .../config/struct-telemetryconfig.mdx | 32 +- .../config/struct-telemetryplan.mdx | 32 +- .../config/struct-telemetryproviderconfig.mdx | 36 +- .../config/struct-tooldefinitionconfig.mdx | 32 +- .../config/struct-toolsconfig.mdx | 36 +- .../config/struct-toolsplan.mdx | 36 +- .../config/struct-workflowconfig.mdx | 32 +- .../struct-workflowentrypointconfig.mdx | 32 +- .../config/struct-workflowtargetspec.mdx | 32 +- .../doctor/enum-doctorstatus.mdx | 34 +- .../doctor/fn-doctor-plan.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../doctor/struct-doctorcheck.mdx | 32 +- .../doctor/struct-doctorreport.mdx | 32 +- .../error/enum-fabricerror.mdx | 24 +- .../nemo-fabric-core/error/index.mdx | 2 +- .../nemo-fabric-core/error/type-result.mdx | 4 +- .../nemo-fabric-core/fn-version.mdx | 4 +- .../nemo-fabric-core/index.mdx | 4 +- .../model-usage/fn-normalize-model-usage.mdx | 15 + .../nemo-fabric-core/model-usage/index.mdx | 6 +- .../enum-cachereadaccounting.mdx | 116 ++ .../enum-pricingcatalogerror.mdx | 129 ++ .../price-catalog/enum-pricingunit.mdx | 138 ++ .../enum-ratescheduleapplication.mdx | 117 ++ .../price-catalog/enum-tokenrateschedule.mdx | 107 ++ .../nemo-fabric-core/price-catalog/index.mdx | 34 + .../price-catalog/struct-modelpricing.mdx | 131 ++ .../price-catalog/struct-pricingcatalog.mdx | 113 ++ .../struct-promptcachepricing.mdx | 103 ++ .../struct-tokenpricingrates.mdx | 111 ++ .../price-catalog/struct-tokenratetier.mdx | 107 ++ .../price-catalog/struct-userpricecatalog.mdx | 115 ++ .../nemo-fabric-core/pricing/index.mdx | 6 +- .../resource-snapshot/index.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 2 +- .../nemo-fabric-core/schema/index.mdx | 2 +- .../nemo-fabric-core/system-stats/index.mdx | 2 +- .../usage-accounting/index.mdx | 2 +- docs/sdk/python.mdx | 18 +- schemas/sdk/agent.schema.json | 25 + schemas/sdk/run-plan.schema.json | 322 +++++ .../src/nemo_fabric/__init__.py | 2 + .../src/nemo_fabric/models.py | 14 + tests/python/conftest.py | 13 +- tests/python/test_usage_reducer_parity.py | 36 +- 101 files changed, 5267 insertions(+), 1260 deletions(-) create mode 100644 crates/fabric-core/data/model-pricing.json create mode 100644 crates/fabric-core/src/price_catalog.rs create mode 100644 crates/fabric-core/tests/model_usage.rs create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx diff --git a/Cargo.lock b/Cargo.lock index 9e6be276f..8eaa11a9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1345,6 +1345,7 @@ dependencies = [ "schemars", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "windows-sys 0.61.2", ] diff --git a/Cargo.toml b/Cargo.toml index 76ee54284..f7dff9b70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,8 @@ libc = "0.2.186" schemars = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +# Standard digest for reproducible price-catalog snapshots; already in the lockfile. +sha2 = { version = "0.10", default-features = false } thiserror = "2" windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_System_Diagnostics_ToolHelp", "Win32_System_JobObjects", "Win32_System_Threading"] } pyo3 = "0.28.2" diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index fdfb09f2e..e8ccacf66 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -256,8 +256,8 @@ does not collect provider-reported cost or promote harness cost estimates. Native OpenAI pricing identity requires every observed request for that call to use `https://api.openai.com` on the standard HTTPS port. Custom endpoints, including environment-selected endpoints, keep their token and model evidence -without native-provider pricing identity. Catalog estimation is added separately; -this collector does not compute prices. +without native-provider pricing identity; an applicable user catalog can price +them explicitly. Unmanaged ChatOpenAI model overrides retain available observations with unverified scope. Other provider clients that expose usage only through LangChain diff --git a/crates/fabric-cli/src/presets.rs b/crates/fabric-cli/src/presets.rs index a3ec93272..44a3fbb6c 100644 --- a/crates/fabric-cli/src/presets.rs +++ b/crates/fabric-cli/src/presets.rs @@ -308,6 +308,7 @@ fn config( telemetry: None, system_stats: Some(Default::default()), relay: None, + pricing: None, extensions: BTreeMap::new(), } } diff --git a/crates/fabric-cli/templates/rust/main.rs.tmpl b/crates/fabric-cli/templates/rust/main.rs.tmpl index 611a62b46..c281b0105 100644 --- a/crates/fabric-cli/templates/rust/main.rs.tmpl +++ b/crates/fabric-cli/templates/rust/main.rs.tmpl @@ -62,6 +62,7 @@ fn build_config() -> FabricConfig { telemetry: None, system_stats: Some(Default::default()), relay: None, + pricing: None, extensions: BTreeMap::new(), } } diff --git a/crates/fabric-core/Cargo.toml b/crates/fabric-core/Cargo.toml index 440de80fa..1572f92e3 100644 --- a/crates/fabric-core/Cargo.toml +++ b/crates/fabric-core/Cargo.toml @@ -18,6 +18,7 @@ jsonschema.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true thiserror.workspace = true [target.'cfg(unix)'.dependencies] diff --git a/crates/fabric-core/data/model-pricing.json b/crates/fabric-core/data/model-pricing.json new file mode 100644 index 000000000..376658dda --- /dev/null +++ b/crates/fabric-core/data/model-pricing.json @@ -0,0 +1,71 @@ +{ + "version": 1, + "entries": [ + { + "provider": "anthropic", + "model_id": "claude-opus-4-5", + "aliases": ["claude-opus-4-5-20251101"], + "currency": "USD", + "unit": "per_token", + "rates": { + "input_per_million": 5.0, + "output_per_million": 25.0, + "cache_read_per_million": 0.5, + "cache_write_per_million": 6.25 + }, + "prompt_cache": {"read_accounting": "separate"}, + "pricing_as_of": "2026-10-08", + "pricing_source": "https://platform.claude.com/docs/en/about-claude/pricing" + }, + { + "provider": "anthropic", + "model_id": "claude-sonnet-4-5", + "aliases": ["claude-sonnet-4-5-20250929"], + "currency": "USD", + "unit": "per_token", + "rates": { + "input_per_million": 3.0, + "output_per_million": 15.0, + "cache_read_per_million": 0.3, + "cache_write_per_million": 3.75 + }, + "prompt_cache": {"read_accounting": "separate"}, + "pricing_as_of": "2026-10-08", + "pricing_source": "https://platform.claude.com/docs/en/about-claude/pricing" + }, + { + "provider": "openai", + "model_id": "gpt-5.4", + "aliases": ["gpt-5.4-2026-03-05"], + "currency": "USD", + "unit": "per_token", + "rate_schedule": { + "type": "prompt_token_threshold", + "applies_to": "full_request", + "tiers": [ + { + "max_prompt_tokens": 272000, + "rates": { + "input_per_million": 2.5, + "output_per_million": 15.0, + "cache_read_per_million": 0.25, + "cache_write_per_million": 2.5 + } + }, + { + "min_prompt_tokens": 272001, + "rates": { + "input_per_million": 5.0, + "output_per_million": 22.5, + "cache_read_per_million": 0.5, + "cache_write_per_million": 5.0 + } + } + ] + }, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"}, + "pricing_as_of": "2026-10-08", + "pricing_source": "https://developers.openai.com/api/docs/pricing" + } + ] +} diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index c5a57eb28..19fc846d8 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -23,6 +23,8 @@ pub use crate::agent_config::{ use crate::agent_execution::{AgentRunRequest, AgentRunResult}; use crate::error::{FabricError, Result}; +use crate::price_catalog::UserPriceCatalog; + /// Versioned NVIDIA NeMo Fabric agent config. /// /// NeMo Fabric-owned fields apply uniformly, while adapter-translated fields are @@ -72,11 +74,25 @@ pub struct FabricConfig { /// First-class NeMo Relay integration configuration. #[serde(default, skip_serializing_if = "Option::is_none")] pub relay: Option, + /// Optional catalog override for central token-price estimates. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pricing: Option, /// Additive fields not yet recognized by this core version. #[serde(default, flatten)] pub extensions: BTreeMap, } +/// Central token-price configuration; never changes provider requests. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct PricingConfig { + /// Relay-compatible v1 catalog, resolved relative to the config base directory. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub catalog_path: Option, + /// Additive pricing configuration, preserved for future versions. + #[serde(default, flatten)] + pub extensions: BTreeMap, +} + /// How an instruction value is applied to the selected harness. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] @@ -2597,7 +2613,9 @@ fn resolve_selected_descriptors( descriptor, adapter_target_descriptor.as_ref(), ); + let user_price_catalog = resolve_user_price_catalog(&config, &base_dir)?; Ok(RunPlan { + user_price_catalog, agent_name: config.metadata.name.clone(), base_dir, config, @@ -2612,6 +2630,33 @@ fn resolve_selected_descriptors( }) } +fn resolve_user_price_catalog( + config: &FabricConfig, + base_dir: &Path, +) -> Result> { + let Some(path) = config + .pricing + .as_ref() + .and_then(|p| p.catalog_path.as_ref()) + else { + return Ok(None); + }; + if path.as_os_str().to_string_lossy().trim().is_empty() { + return invalid_config("pricing.catalog_path", "must not be blank"); + } + let path = resolve_path(base_dir, path); + let bytes = std::fs::read(&path).map_err(|error| FabricError::InvalidConfig { + field: "pricing.catalog_path".into(), + reason: format!("cannot read {}: {error}", path.display()), + })?; + UserPriceCatalog::from_bytes(path, &bytes) + .map(Some) + .map_err(|error| FabricError::InvalidConfig { + field: "pricing.catalog_path".into(), + reason: error.to_string(), + }) +} + fn validate_capability_plan_compatibility( capability_plan: &CapabilityPlan, descriptor: Option<&AdapterDescriptor>, @@ -4021,6 +4066,9 @@ fn normalize_path(path: PathBuf) -> PathBuf { /// Resolved NeMo Fabric run plan. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct RunPlan { + /// Catalog bytes are frozen during planning. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_price_catalog: Option, /// Stable agent name. pub agent_name: String, /// Base directory used to resolve relative NeMo Fabric paths. diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index d0bd8d4f6..9122aade4 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -12,6 +12,7 @@ mod environment; pub mod error; mod measurement; pub mod model_usage; +pub mod price_catalog; pub mod pricing; pub mod resource_snapshot; pub mod runtime; @@ -38,10 +39,11 @@ pub use config::{ EnvironmentOwnership, EnvironmentPlan, FabricConfig, HarnessConfig, InstructionConfig, InstructionMode, InstructionsConfig, McpAuthenticationConfig, McpConfig, McpExposure, McpServerPlan, McpTransport, MetadataConfig, ModelConfig, OAuthTokenEndpointAuthMethod, - ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, ResolvedAdapterTargetDescriptor, - RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, TelemetryConfig, TelemetryPlan, - TelemetryProvider, TelemetryProviderConfig, ToolDefinitionConfig, ToolsConfig, WorkflowConfig, - WorkflowEntrypointConfig, WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, + PricingConfig, ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, + ResolvedAdapterTargetDescriptor, RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, + TelemetryConfig, TelemetryPlan, TelemetryProvider, TelemetryProviderConfig, + ToolDefinitionConfig, ToolsConfig, WorkflowConfig, WorkflowEntrypointConfig, + WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, load_adapter_target_descriptor, resolve_diagnostic_plan_from_config, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config, resolve_run_plan_from_config_with_adapter_directories, diff --git a/crates/fabric-core/src/model_usage.rs b/crates/fabric-core/src/model_usage.rs index 9d9aaff80..829a7ebad 100644 --- a/crates/fabric-core/src/model_usage.rs +++ b/crates/fabric-core/src/model_usage.rs @@ -139,13 +139,30 @@ pub struct ModelUsage { /// Normalize supported evidence. Legacy usage remains raw rather than gaining /// a completeness guarantee merely because a newer core read it. -pub(crate) fn normalize_model_usage( +pub fn normalize_model_usage( usage: &AgentUsage, + catalog: Option<&crate::price_catalog::UserPriceCatalog>, + bundled_pricing_allowed: bool, ) -> Result, AccountingError> { - let Some(_) = UsageAccounting::from_usage(usage)? else { + normalize_model_usage_for_routes(usage, catalog, |_, _| bundled_pricing_allowed) +} + +pub(crate) fn normalize_model_usage_for_routes( + usage: &AgentUsage, + catalog: Option<&crate::price_catalog::UserPriceCatalog>, + bundled_pricing_allowed: impl Fn(Option<&str>, &str) -> bool, +) -> Result, AccountingError> { + let Some(evidence) = UsageAccounting::from_usage(usage)? else { return Ok(None); }; - Ok(Some(view(normalized::describe_model_usage(usage)?))) + let mut result = normalized::describe_model_usage(usage)?; + crate::pricing::apply_pricing_with_evidence( + &mut result, + catalog, + bundled_pricing_allowed, + Some(&evidence), + ); + Ok(Some(view(result))) } fn tokens(values: &normalized::ModelMeasurements) -> TokenCounts { @@ -262,370 +279,3 @@ fn view(value: normalized::ModelUsage) -> ModelUsage { cost: cost(&measurements.token_cost, &accounting.observed), } } - -#[cfg(test)] -mod accounting_tests { - use super::normalize_model_usage; - use crate::AgentUsage; - use serde_json::{Value, json}; - - fn normalize(base: Value, accounting: Value) -> Value { - let mut usage: AgentUsage = serde_json::from_value(base).unwrap(); - usage - .extensions - .insert("nemo_fabric.usage_accounting.v1".into(), accounting); - serde_json::to_value(normalize_model_usage(&usage).unwrap().unwrap()).unwrap() - } - - #[test] - fn shared_source_vectors_keep_the_reduced_complete_and_observed_contract() { - let cases: Vec = serde_json::from_str(include_str!( - "../../../tests/fixtures/usage_accounting/conformance.json" - )) - .unwrap(); - let dimensions = [ - "input_tokens", - "cached_input_tokens", - "cache_write_input_tokens", - "output_tokens", - "total_tokens", - ]; - for case in cases { - let result = normalize(case["usage"].clone(), case["accounting"].clone()); - for (field, expected) in case["expected"].as_object().unwrap() { - if dimensions.contains(&field.as_str()) { - assert_eq!( - result[field], expected["complete"], - "{} {field}", - case["name"] - ); - assert_eq!( - result["observed"][field], expected["observed"], - "{} {field}", - case["name"] - ); - } - } - assert!(result.get("accounting").is_none()); - assert!(result.get("reasoning_tokens").is_none()); - assert!(result.get("peak_request_input_tokens").is_none()); - assert!(result["input_tokens"].is_number() || result["input_tokens"].is_null()); - } - } - - #[test] - fn legacy_usage_does_not_gain_a_completeness_guarantee() { - let usage: AgentUsage = serde_json::from_value(json!({"input_tokens": 100})).unwrap(); - assert!(normalize_model_usage(&usage).unwrap().is_none()); - } - - #[test] - fn normalized_total_does_not_replace_the_independent_raw_provider_total() { - let mut usage: AgentUsage = serde_json::from_value(json!({ - "input_tokens": 100, "input_tokens_include_cache": true, - "output_tokens": 5, "total_tokens": 999, - "extensions": {"nemo_fabric.detailed_token_usage.v1": {"output_tokens": 5, "output_tokens_include_reasoning": true}} - })).unwrap(); - usage.extensions.insert( - "nemo_fabric.usage_accounting.v1".into(), - json!({"models_exhaustive": false, "gaps": []}), - ); - let normalized = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(normalized.tokens.total_tokens, Some(105)); - assert_eq!(usage.total_tokens, Some(999)); - } - - #[test] - fn overflow_does_not_erase_independent_valid_counts() { - let result = normalize( - json!({}), - json!({"models_exhaustive": true, "gaps": [], "models": [{ - "model": "x", "gaps": [], "reported": {"input_tokens": u64::MAX, "input_tokens_include_cache": true, - "output_tokens": 1, "output_tokens_include_reasoning": true}}]}), - ); - assert_eq!(result["input_tokens"], json!(u64::MAX)); - assert_eq!(result["output_tokens"], 1); - assert!(result["total_tokens"].is_null()); - } - - fn cost_with_requests( - aggregate: f64, - requests: &[Option], - exhaustive: bool, - ) -> AgentUsage { - let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, - "output_tokens": 5, "output_tokens_include_reasoning": true, "cost_usd": aggregate}); - let mut usage = AgentUsage { - input_tokens: Some(100), - input_tokens_include_cache: Some(true), - output_tokens: Some(5), - cost_usd: Some(aggregate), - ..Default::default() - }; - usage - .extensions - .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); - usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ - "models_exhaustive": true, "gaps": [], "models": [{ - "provider": "test", "model": "test-model", "gaps": [], "reported": counts, - "requests_exhaustive": exhaustive, - "requests": requests.iter().map(|cost| json!({"usage": {"cost_usd": cost}})).collect::>() - }] - })); - usage - } - - #[test] - fn contradictory_request_reported_costs_are_rejected() { - for (requests, exhaustive) in [ - (vec![Some(0.2)], true), - (vec![Some(0.2)], false), - (vec![Some(0.04)], true), - (vec![Some(10_000_000_000.0); 2], true), - ] { - let usage = cost_with_requests(0.1, &requests, exhaustive); - assert!( - normalize_model_usage(&usage).is_err(), - "contradictory cost must not become complete: {requests:?}, exhaustive={exhaustive}" - ); - } - } - - #[test] - fn independently_rounded_request_costs_keep_fractional_nanos_and_large_values() { - for (aggregate, requests) in [ - (1.6e-9, vec![Some(0.4e-9); 4]), - ( - 200_000_000.72337076, - vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], - ), - ] { - let usage = cost_with_requests(aggregate, &requests, true); - let result = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(result.tokens.input_tokens, Some(100)); - assert_eq!(result.tokens.total_tokens, Some(105)); - assert!(result.cost.reported.amount_nano_usd.is_some()); - } - } - - #[test] - fn partial_or_missing_request_costs_preserve_independent_complete_dimensions() { - for (requests, exhaustive) in [ - (vec![Some(0.04)], false), - (vec![Some(0.04), None], true), - (vec![None], true), - (vec![Some(0.0)], false), - ] { - let usage = cost_with_requests(0.1, &requests, exhaustive); - let result = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(result.tokens.input_tokens, Some(100)); - assert_eq!(result.tokens.output_tokens, Some(5)); - assert_eq!(result.tokens.total_tokens, Some(105)); - assert_eq!(result.cost.reported.amount_nano_usd, Some(100_000_000)); - } - } - - fn omit_model_cost(usage: &mut AgentUsage) { - usage - .extensions - .get_mut("nemo_fabric.usage_accounting.v1") - .unwrap()["models"][0]["reported"] - .as_object_mut() - .unwrap() - .remove("cost_usd"); - } - - #[test] - fn request_costs_without_model_cost_cannot_certify_a_contradictory_root() { - let mut usage = cost_with_requests(0.1, &[Some(0.2)], true); - omit_model_cost(&mut usage); - let result = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(result.cost.reported.amount_nano_usd, None); - assert_eq!(result.cost.reported.observed_nano_usd, None); - assert_eq!(result.tokens.total_tokens, Some(105)); - } - - #[test] - fn exhaustive_requests_recover_cost_without_losing_legitimate_root_rounding() { - for (aggregate, requests) in [ - (0.1, vec![Some(0.04), Some(0.06)]), - ( - 200_000_000.72337076, - vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], - ), - ] { - let mut usage = cost_with_requests(aggregate, &requests, true); - omit_model_cost(&mut usage); - let with_root = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!( - with_root.cost.reported.amount_nano_usd, - Some((aggregate * 1e9).round() as u64) - ); - assert_eq!(with_root.tokens.total_tokens, Some(105)); - usage.cost_usd = None; - let recovered = normalize_model_usage(&usage).unwrap().unwrap(); - assert!(recovered.cost.reported.amount_nano_usd.is_some()); - assert_eq!( - recovered.cost.reported.amount_nano_usd, - recovered.models[0].cost.reported.amount_nano_usd - ); - } - } - - #[test] - fn incomplete_requests_recover_only_observed_cost_when_no_complete_cost_exists() { - for (requests, exhaustive) in [(vec![Some(0.04)], false), (vec![Some(0.04), None], true)] { - let mut usage = cost_with_requests(0.1, &requests, exhaustive); - omit_model_cost(&mut usage); - usage.cost_usd = None; - let result = normalize_model_usage(&usage).unwrap().unwrap(); - assert_eq!(result.cost.reported.amount_nano_usd, None); - assert_eq!(result.cost.reported.observed_nano_usd, Some(40_000_000)); - assert_eq!(result.tokens.total_tokens, Some(105)); - } - } - - #[test] - fn broad_request_gap_cannot_certify_complete_model_or_invocation_counts() { - let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, - "output_tokens": 5, "output_tokens_include_reasoning": true}); - for gap in ["unfinished_work", "missing_usage", "scope_unverified"] { - for root_complete in [false, true] { - let mut usage = AgentUsage::default(); - if root_complete { - usage.input_tokens = Some(100); - usage.input_tokens_include_cache = Some(true); - usage.output_tokens = Some(5); - usage - .extensions - .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); - } - usage.extensions.insert( - "nemo_fabric.usage_accounting.v1".into(), - json!({ - "models_exhaustive": true, "gaps": [], "models": [{ - "model": "model", "gaps": [], - "reported": if root_complete { json!({}) } else { counts.clone() }, - "requests_exhaustive": true, - "requests": [{"usage": {}, "gap": gap}] - }] - }), - ); - assert!( - normalize_model_usage(&usage).is_err(), - "broad request gap must conflict with complete counts: {gap}, root={root_complete}" - ); - } - } - } - - #[test] - fn broad_request_gap_keeps_observations_when_no_complete_value_is_claimed() { - let observed = json!({"input_tokens": 100, "input_tokens_include_cache": true, - "output_tokens": 5, "output_tokens_include_reasoning": true}); - let result = normalize( - json!({}), - json!({ - "models_exhaustive": false, "gaps": [], "models": [{ - "model": "model", "gaps": [], "observed": observed, - "observed_tokens_aligned": true, "requests_exhaustive": true, - "requests": [{"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}] - }] - }), - ); - assert!(result["input_tokens"].is_null()); - assert!(result["output_tokens"].is_null()); - assert_eq!(result["observed"]["input_tokens"], 100); - assert_eq!(result["observed"]["output_tokens"], 5); - for scope in [&result, &result["models"][0]] { - for dimension in [ - "input_tokens", - "cached_input_tokens", - "cache_write_input_tokens", - "output_tokens", - "total_tokens", - "reported_cost", - ] { - assert!( - scope["gaps"].as_array().unwrap().iter().any(|gap| { - gap["dimensions"] == json!([dimension]) - && gap["reason"] == "unfinished_work" - }), - "request reason missing for {dimension}: {}", - scope["gaps"] - ); - } - } - - let mixed = normalize( - json!({}), - json!({ - "models_exhaustive": true, "gaps": [], "models": [{ - "model": "pending", "gaps": [{"dimensions": ["output_tokens"], "reason": "missing_usage"}], - "observed": observed, "observed_tokens_aligned": true, - "requests_exhaustive": true, - "requests": [ - {"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}, - {"usage": {}, "gap": "scope_unverified"}, - {"usage": {}, "gap": "unfinished_work"} - ] - }, {"model": "settled", "reported": observed, "gaps": []}] - }), - ); - assert!(mixed["input_tokens"].is_null()); - assert_eq!(mixed["observed"]["input_tokens"], 200); - assert_eq!(mixed["observed"]["output_tokens"], 10); - assert_eq!(mixed["models"][1]["total_tokens"], 105); - assert!( - !mixed["models"][1]["gaps"] - .as_array() - .unwrap() - .iter() - .any(|gap| { - gap["reason"] == "unfinished_work" || gap["reason"] == "scope_unverified" - }) - ); - for scope in [&mixed, &mixed["models"][0]] { - let gaps = scope["gaps"].as_array().unwrap(); - for reason in ["unfinished_work", "scope_unverified"] { - assert_eq!( - gaps.iter() - .filter(|gap| { - gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == reason - }) - .count(), - 1, - "each request reason must survive once: {gaps:?}" - ); - } - } - assert!( - mixed["models"][0]["gaps"] - .as_array() - .unwrap() - .iter() - .any(|gap| { - gap["dimensions"] == json!(["output_tokens"]) - && gap["reason"] == "missing_usage" - }) - ); - } - - #[test] - fn dimension_scoped_request_omission_keeps_independent_complete_output() { - let result = normalize( - json!({}), - json!({ - "models_exhaustive": true, "gaps": [], "models": [{ - "model": "model", "gaps": [{"dimensions": ["input_tokens"], "reason": "missing_usage"}], - "reported": {"output_tokens": 5, "output_tokens_include_reasoning": true}, - "requests_exhaustive": true, - "requests": [{"usage": {"output_tokens": 5, "output_tokens_include_reasoning": true}, "tokens_aligned": true}] - }] - }), - ); - assert!(result["input_tokens"].is_null()); - assert_eq!(result["output_tokens"], 5); - assert!(result["total_tokens"].is_null()); - } -} diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs new file mode 100644 index 000000000..f457829f1 --- /dev/null +++ b/crates/fabric-core/src/price_catalog.rs @@ -0,0 +1,1128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Model price catalogs for estimating token cost. +//! +//! A catalog uses the NeMo Relay model pricing catalog format, version 1, so +//! one file can configure both tools. NeMo Fabric parses the format itself and +//! does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. +//! An explicit provider restricts aliases and model-path suffixes to that provider. +//! Without a provider, model-only lookup uses the first matching entry in listed order. +//! +//! NeMo Fabric ships one catalog, embedded at build time, with the list prices +//! of models that its presets, docs, and examples use. Each entry cites the +//! provider pricing page it was taken from and the date it was checked. A +//! caller can add a catalog of their own, whose entries take precedence. + +use std::collections::HashSet; +use std::fmt::Write as _; +use std::path::PathBuf; +use std::sync::LazyLock; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// The only catalog format version NeMo Fabric reads. +const PRICING_CATALOG_VERSION: u32 = 1; + +/// Rates must be below one million US dollars per million tokens. NeMo Relay +/// sets no upper bound; this one keeps every rate exact in nano-USD per +/// million tokens and every cost sum within 128 bits. +const MAX_RATE_PER_MILLION: f64 = 1_000_000.0; + +/// Nano-USD per US dollar. +const NANO_USD_PER_USD: f64 = 1e9; + +/// The catalog NeMo Fabric ships. +const BUNDLED_CATALOG_JSON: &str = include_str!("../data/model-pricing.json"); + +static BUNDLED_CATALOG: LazyLock = LazyLock::new(|| BundledCatalog { + // A unit test checks that the shipped file is valid, so `None` only + // guards against a build that skipped the tests. + catalog: PricingCatalog::from_json(BUNDLED_CATALOG_JSON.as_bytes()).ok(), + sha256: sha256_fingerprint(BUNDLED_CATALOG_JSON.as_bytes()), +}); + +struct BundledCatalog { + catalog: Option, + sha256: String, +} + +/// A model price catalog in the NeMo Relay model pricing catalog format, +/// version 1. +/// +/// NeMo Fabric reads and writes the format unchanged, with one extra check: +/// every rate must be below one million US dollars per million tokens. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PricingCatalog { + /// Catalog format version. Must be `1`. + pub version: u32, + /// Price entries. Lookup uses the first entry that matches. + pub entries: Vec, +} + +/// Prices of one model, in `currency` per million tokens. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ModelPricing { + /// Provider that sets the price, such as `anthropic` or `openai`. + pub provider: String, + /// Canonical model identifier. + pub model_id: String, + /// Other identifiers of the same model, such as dated snapshots. + #[serde(default)] + pub aliases: Vec, + /// ISO 4217 currency of the rates. NeMo Fabric estimates cost only from + /// `USD` entries. + #[serde(default = "default_currency")] + pub currency: String, + /// What a rate is charged per. NeMo Fabric estimates cost only from + /// `per_token` entries. + #[serde(default)] + pub unit: PricingUnit, + /// Flat rates. Ignored when `rate_schedule` is set. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rates: Option, + /// Rates that depend on the number of prompt tokens in a request. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate_schedule: Option, + /// How the provider counts cached prompt tokens. NeMo Relay uses it to + /// split provider usage; NeMo Fabric prices usage that is already + /// normalized, so it does not read it. + pub prompt_cache: PromptCachePricing, + /// Date the price was taken from `pricing_source`, as `YYYY-MM-DD` in + /// the bundled catalog. + pub pricing_as_of: String, + /// Where the price was taken from, such as the provider's pricing page. + pub pricing_source: String, +} + +/// What a model price is charged per. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum PricingUnit { + /// Per token. + #[default] + PerToken, + /// Per request. + PerRequest, + /// Per second. + PerSecond, + /// Per GPU hour, for self-hosted models. + GpuHour, +} + +/// Token rates in currency units per million tokens. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TokenPricingRates { + /// Rate for input tokens that were neither read from nor written to the + /// prompt cache. + pub input_per_million: f64, + /// Rate for output tokens, including reasoning tokens. + pub output_per_million: f64, + /// Rate for input tokens read from the prompt cache. Without it, cached + /// reads cannot be priced; set `0.0` when they are free. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_per_million: Option, + /// Rate for input tokens written to the prompt cache. Without it, cache + /// writes cannot be priced; set `0.0` when they are free. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_write_per_million: Option, +} + +/// Rates that depend on the size of a request. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, tag = "type", rename_all = "snake_case")] +pub enum TokenRateSchedule { + /// One tier's rates apply to a whole request, chosen by the request's + /// prompt tokens. + PromptTokenThreshold { + /// How the chosen tier's rates apply. + #[serde(default)] + applies_to: RateScheduleApplication, + /// Tiers in the order they are tried. The first tier whose bounds + /// contain a request's prompt tokens applies to it. + tiers: Vec, + }, +} + +/// How a chosen rate tier applies to a request. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RateScheduleApplication { + /// The tier's rates apply to every token of the request. + #[default] + FullRequest, +} + +/// Rates for requests whose prompt tokens fall within inclusive bounds. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TokenRateTier { + /// Smallest number of prompt tokens in the tier, or no lower bound. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min_prompt_tokens: Option, + /// Largest number of prompt tokens in the tier, or no upper bound. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_prompt_tokens: Option, + /// Rates of the tier. + pub rates: TokenPricingRates, +} + +/// How a provider counts cached prompt tokens in its usage reports. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PromptCachePricing { + /// Whether the provider's prompt token count includes cached reads. + pub read_accounting: CacheReadAccounting, +} + +/// Whether a provider's prompt token count includes cached reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CacheReadAccounting { + /// Prompt tokens include cached reads. + IncludedInPromptTokens, + /// Prompt tokens exclude cached reads. + Separate, +} + +/// A price catalog the caller supplied, read and validated when the run plan +/// is resolved. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct UserPriceCatalog { + /// Absolute path the catalog was read from. + pub path: PathBuf, + /// `sha256:` followed by the lowercase hexadecimal SHA-256 of the file. + pub sha256: String, + /// The validated catalog. + pub catalog: PricingCatalog, +} + +/// Why a catalog is invalid. The messages match NeMo Relay's. +#[derive(Debug, thiserror::Error)] +pub enum PricingCatalogError { + /// The catalog is not JSON of the catalog format. + #[error("invalid model pricing catalog JSON: {0}")] + Json(#[from] serde_json::Error), + /// The catalog format version is not 1. + #[error("unsupported model pricing catalog version {version}")] + UnsupportedVersion { + /// Version in the catalog. + version: u32, + }, + /// Two entries or aliases of one provider name the same model. + #[error("duplicate model pricing alias '{model}'")] + DuplicateModelAlias { + /// Normalized `provider/model` key. + model: String, + }, + /// A required text field is blank. + #[error("model pricing entry {entry_index} has empty {field}")] + EmptyField { + /// Zero-based entry index. + entry_index: usize, + /// Field name. + field: String, + }, + /// A rate or a tier bound is out of range. + #[error("model pricing entry {entry_index} has invalid {field}: {value}")] + InvalidRate { + /// Zero-based entry index. + entry_index: usize, + /// Field name. + field: String, + /// Invalid value. + value: f64, + }, +} + +impl UserPriceCatalog { + /// Validate bytes and retain their exact catalog fingerprint. + pub fn from_bytes(path: PathBuf, bytes: &[u8]) -> Result { + Ok(Self { + path, + sha256: sha256_fingerprint(bytes), + catalog: PricingCatalog::from_json(bytes)?, + }) + } +} + +impl PricingCatalog { + /// Parse and validate a catalog. + pub fn from_json(json: &[u8]) -> Result { + let catalog: Self = serde_json::from_slice(json)?; + catalog.validate()?; + Ok(catalog) + } + + /// Find `model`, keeping an explicit provider on every alias or suffix. + /// Without a provider, try the model path's provider prefix before model-only + /// suffixes. The first matching entry in listed order wins for each key. + pub(crate) fn pricing_for(&self, provider: Option<&str>, model: &str) -> Option<&ModelPricing> { + lookup_keys(provider, model) + .iter() + .find_map(|key| self.entries.iter().find(|entry| entry.matches(key))) + } + + fn validate(&self) -> Result<(), PricingCatalogError> { + if self.version != PRICING_CATALOG_VERSION { + return Err(PricingCatalogError::UnsupportedVersion { + version: self.version, + }); + } + let mut seen = HashSet::new(); + for (entry_index, entry) in self.entries.iter().enumerate() { + entry.validate(entry_index)?; + let provider = normalized_provider(&entry.provider); + for model in entry.model_keys() { + let key = format!("{provider}/{model}"); + if !seen.insert(key.clone()) { + return Err(PricingCatalogError::DuplicateModelAlias { model: key }); + } + } + } + Ok(()) + } +} + +impl ModelPricing { + pub(crate) fn rates_at(&self, prompt_tokens: u64) -> Option { + match &self.rate_schedule { + None => self.rates, + Some(TokenRateSchedule::PromptTokenThreshold { tiers, .. }) => tiers + .iter() + .find(|tier| { + tier.min_prompt_tokens.unwrap_or(0) <= prompt_tokens + && tier + .max_prompt_tokens + .is_none_or(|max| prompt_tokens <= max) + }) + .map(|tier| tier.rates), + } + } + /// Rates that apply to every request of a model whose largest request + /// had `peak_prompt_tokens` prompt tokens. `None` when the entry has a + /// rate schedule and the peak is unknown or no single tier covers every + /// request up to it. + pub(crate) fn rates_up_to(&self, peak_prompt_tokens: Option) -> Option { + match &self.rate_schedule { + Some(schedule) => schedule.rates_up_to(peak_prompt_tokens?), + None => self.rates, + } + } + + fn matches(&self, key: &LookupKey) -> bool { + key.provider + .as_deref() + .is_none_or(|provider| normalized_provider(&self.provider) == provider) + && self.model_keys().any(|model| model == key.model) + } + + /// The normalized model identifier and aliases, without blank ones. + fn model_keys(&self) -> impl Iterator + '_ { + std::iter::once(&self.model_id) + .chain(&self.aliases) + .map(|model| normalized_model(model)) + .filter(|model| !model.is_empty()) + } + + fn validate(&self, entry_index: usize) -> Result<(), PricingCatalogError> { + for (field, value) in [ + ("provider", &self.provider), + ("model_id", &self.model_id), + ("currency", &self.currency), + ("pricing_as_of", &self.pricing_as_of), + ("pricing_source", &self.pricing_source), + ] { + if value.trim().is_empty() { + return Err(empty_field(entry_index, field)); + } + } + if self.unit == PricingUnit::PerToken + && self.rates.is_none() + && self.rate_schedule.is_none() + { + return Err(empty_field(entry_index, "rates or rate_schedule")); + } + if let Some(rates) = &self.rates { + rates.validate(entry_index, "rates")?; + } + if let Some(schedule) = &self.rate_schedule { + schedule.validate(entry_index)?; + } + Ok(()) + } +} + +impl TokenPricingRates { + fn validate(&self, entry_index: usize, prefix: &str) -> Result<(), PricingCatalogError> { + for (field, value) in [ + ("input_per_million", Some(self.input_per_million)), + ("output_per_million", Some(self.output_per_million)), + ("cache_read_per_million", self.cache_read_per_million), + ("cache_write_per_million", self.cache_write_per_million), + ] { + if let Some(value) = value + && nano_usd_per_million_tokens(value).is_none() + { + return Err(PricingCatalogError::InvalidRate { + entry_index, + field: format!("{prefix}.{field}"), + value, + }); + } + } + Ok(()) + } +} + +impl TokenRateSchedule { + fn rates_up_to(&self, peak_prompt_tokens: u64) -> Option { + match self { + // NeMo Relay prices each request with the first tier whose bounds + // contain its prompt tokens. Every request had between zero and + // the peak prompt tokens, so one tier's rates apply to all of them + // exactly when the first tier that overlaps that range contains + // all of it. + Self::PromptTokenThreshold { tiers, .. } => { + let tier = tiers + .iter() + .find(|tier| tier.overlaps_up_to(peak_prompt_tokens))?; + tier.contains_up_to(peak_prompt_tokens) + .then_some(tier.rates) + } + } + } + + fn validate(&self, entry_index: usize) -> Result<(), PricingCatalogError> { + let Self::PromptTokenThreshold { tiers, .. } = self; + if tiers.is_empty() { + return Err(empty_field(entry_index, "rate_schedule.tiers")); + } + for (tier_index, tier) in tiers.iter().enumerate() { + if let (Some(min), Some(max)) = (tier.min_prompt_tokens, tier.max_prompt_tokens) + && min > max + { + return Err(PricingCatalogError::InvalidRate { + entry_index, + field: "rate_schedule.tiers.prompt_tokens".to_string(), + // NeMo Relay reports the lower bound in the same way. + value: min as f64, + }); + } + tier.rates.validate( + entry_index, + &format!("rate_schedule.tiers[{tier_index}].rates"), + )?; + } + Ok(()) + } +} + +impl TokenRateTier { + /// Whether the tier contains any prompt token count from zero to `peak`. + fn overlaps_up_to(&self, peak: u64) -> bool { + self.min_prompt_tokens.unwrap_or(0) <= peak + } + + /// Whether the tier contains every prompt token count from zero to `peak`. + fn contains_up_to(&self, peak: u64) -> bool { + self.min_prompt_tokens.unwrap_or(0) == 0 + && self.max_prompt_tokens.is_none_or(|max| peak <= max) + } +} + +fn empty_field(entry_index: usize, field: &str) -> PricingCatalogError { + PricingCatalogError::EmptyField { + entry_index, + field: field.to_string(), + } +} + +fn normalized_model(model: &str) -> String { + model.trim().to_ascii_lowercase() +} + +fn normalized_provider(provider: &str) -> String { + provider.trim().trim_matches('/').to_ascii_lowercase() +} + +/// A lookup key: a normalized model, optionally qualified by a normalized +/// provider. +#[derive(Debug, PartialEq, Eq)] +struct LookupKey { + provider: Option, + model: String, +} + +/// Prefer exact model keys, preserving an explicitly established provider. +fn lookup_keys(provider: Option<&str>, model: &str) -> Vec { + let model = normalized_model(model); + if model.is_empty() { + return Vec::new(); + } + let parts: Vec<&str> = model + .split('/') + .map(str::trim) + .filter(|part| !part.is_empty()) + .collect(); + let last = parts.last().copied().unwrap_or(model.as_str()); + let mut keys = Vec::new(); + let mut push = |provider: Option, model: String| { + let key = LookupKey { provider, model }; + if !key.model.is_empty() && !keys.contains(&key) { + keys.push(key); + } + }; + let explicit_provider = provider + .map(normalized_provider) + .filter(|provider| !provider.is_empty()); + match &explicit_provider { + Some(provider) => { + push(Some(provider.clone()), model.clone()); + push(Some(provider.clone()), last.to_string()); + } + None if parts.len() > 1 => { + push(Some(parts[..parts.len() - 1].join("/")), last.to_string()); + } + None => {} + } + for start in 0..parts.len() { + push(explicit_provider.clone(), parts[start..].join("/")); + } + keys +} + +/// Where a price came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PriceOrigin { + /// The catalog NeMo Fabric ships. + Bundled, + /// The catalog the caller supplied. + User, +} + +/// The price catalogs an estimate consults, the caller's first. +pub(crate) struct PriceCatalogs<'a> { + bundled: Option<&'a PricingCatalog>, + bundled_sha256: &'a str, + user: Option<&'a UserPriceCatalog>, +} + +impl<'a> PriceCatalogs<'a> { + /// The shipped catalog, preceded by the caller's catalog when given. + pub(crate) fn with_user_catalog(user: Option<&'a UserPriceCatalog>) -> Self { + Self::from_parts( + BUNDLED_CATALOG.catalog.as_ref(), + &BUNDLED_CATALOG.sha256, + user, + ) + } + + pub(crate) fn from_parts( + bundled: Option<&'a PricingCatalog>, + bundled_sha256: &'a str, + user: Option<&'a UserPriceCatalog>, + ) -> Self { + Self { + bundled, + bundled_sha256, + user, + } + } + + /// Fingerprint of the shipped catalog. + pub(crate) fn bundled_sha256(&self) -> &'a str { + self.bundled_sha256 + } + + /// Fingerprint of the caller's catalog, when one is configured. + pub(crate) fn user_sha256(&self) -> Option<&'a str> { + self.user.map(|user| user.sha256.as_str()) + } + + /// Whether the shipped catalog parsed. + pub(crate) fn bundled_is_valid(&self) -> bool { + self.bundled.is_some() + } + + /// The entry for a model, from the caller's catalog first. With + /// `user_only`, the shipped catalog is not consulted. + pub(crate) fn find( + &self, + provider: Option<&str>, + model: &str, + user_only: bool, + ) -> Option<(&'a ModelPricing, PriceOrigin)> { + let user = self + .user + .and_then(|user| user.catalog.pricing_for(provider, model)) + .map(|entry| (entry, PriceOrigin::User)); + if user.is_some() || user_only { + return user; + } + self.bundled + .and_then(|bundled| bundled.pricing_for(provider, model)) + .map(|entry| (entry, PriceOrigin::Bundled)) + } +} + +/// Nano-USD per million tokens of a rate in US dollars per million tokens, +/// rounded to the nearest whole number. `None` for a rate no valid catalog +/// holds: negative, not finite, or not below one million. +pub(crate) fn nano_usd_per_million_tokens(rate: f64) -> Option { + (rate.is_finite() && (0.0..MAX_RATE_PER_MILLION).contains(&rate)) + // In range, so the rounded value is a whole number below 10^15 and + // the conversion is exact. + .then(|| (rate * NANO_USD_PER_USD).round() as u128) +} + +/// `sha256:` followed by the lowercase hexadecimal SHA-256 of `bytes`. +pub(crate) fn sha256_fingerprint(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut text = String::with_capacity("sha256:".len() + digest.len() * 2); + text.push_str("sha256:"); + for byte in digest.iter() { + // Writing to a String cannot fail. + let _ = write!(text, "{byte:02x}"); + } + text +} + +fn default_currency() -> String { + "USD".to_string() +} + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + + use super::*; + + fn catalog(entries: Value) -> PricingCatalog { + PricingCatalog::from_json( + json!({"version": 1, "entries": entries}) + .to_string() + .as_bytes(), + ) + .expect("valid catalog") + } + + fn catalog_error(entries: Value) -> String { + PricingCatalog::from_json( + json!({"version": 1, "entries": entries}) + .to_string() + .as_bytes(), + ) + .expect_err("invalid catalog") + .to_string() + } + + fn flat_entry(provider: &str, model_id: &str, input: f64, output: f64) -> Value { + json!({ + "provider": provider, + "model_id": model_id, + "pricing_as_of": "2026-06-04", + "pricing_source": format!("https://example.test/{provider}"), + "rates": {"input_per_million": input, "output_per_million": output}, + "prompt_cache": {"read_accounting": "separate"} + }) + } + + fn threshold_entry(tiers: Value) -> Value { + json!({ + "provider": "threshold-ai", + "model_id": "threshold-model", + "pricing_as_of": "2026-06-05", + "pricing_source": "https://example.test/pricing", + "rate_schedule": {"type": "prompt_token_threshold", "tiers": tiers}, + "prompt_cache": {"read_accounting": "separate"} + }) + } + + fn rates(input: f64) -> Value { + json!({"input_per_million": input, "output_per_million": input * 2.0}) + } + + fn found<'a>( + catalog: &'a PricingCatalog, + provider: Option<&str>, + model: &str, + ) -> (&'a str, &'a str) { + let entry = catalog + .pricing_for(provider, model) + .unwrap_or_else(|| panic!("no entry for {provider:?} {model}")); + (entry.provider.as_str(), entry.model_id.as_str()) + } + + // The shipped catalog. + + #[test] + fn bundled_catalog_is_valid_and_cites_a_dated_provider_page_for_every_entry() { + let catalog = PricingCatalog::from_json(BUNDLED_CATALOG_JSON.as_bytes()) + .expect("the shipped catalog is valid"); + + assert!(!catalog.entries.is_empty()); + for entry in &catalog.entries { + let name = format!("{}/{}", entry.provider, entry.model_id); + assert_eq!(entry.currency, "USD", "{name}"); + assert_eq!(entry.unit, PricingUnit::PerToken, "{name}"); + assert!( + entry.pricing_source.starts_with("https://"), + "{name}: {}", + entry.pricing_source + ); + assert!( + is_iso_date(&entry.pricing_as_of), + "{name}: {}", + entry.pricing_as_of + ); + } + assert!(BUNDLED_CATALOG.catalog.is_some()); + } + + /// `YYYY-MM-DD` with a plausible month and day. The test never compares + /// the date with today's, so an old date cannot fail CI. + fn is_iso_date(text: &str) -> bool { + let parts: Vec<&str> = text.split('-').collect(); + let [year, month, day] = parts.as_slice() else { + return false; + }; + let number = |part: &str, digits: usize| { + (part.len() == digits && part.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| part.parse::().ok()) + .flatten() + }; + matches!( + (number(year, 4), number(month, 2), number(day, 2)), + (Some(_), Some(1..=12), Some(1..=31)) + ) + } + + #[test] + fn bundled_catalog_keeps_native_provider_boundaries_for_routed_models() { + let catalog = BUNDLED_CATALOG.catalog.as_ref().expect("bundled catalog"); + assert!( + catalog + .pricing_for(Some("nvidia"), "aws/anthropic/claude-opus-4-5") + .is_none() + ); + assert!( + catalog + .pricing_for(Some("nvidia"), "azure/openai/gpt-5.4") + .is_none() + ); + assert_eq!( + found(catalog, Some("anthropic"), "aws/anthropic/claude-opus-4-5"), + ("anthropic", "claude-opus-4-5") + ); + assert_eq!( + found(catalog, Some("openai"), "azure/openai/gpt-5.4"), + ("openai", "gpt-5.4") + ); + assert_eq!( + found(catalog, Some("anthropic"), "claude-sonnet-4-5-20250929"), + ("anthropic", "claude-sonnet-4-5") + ); + } + + #[test] + fn catalog_fingerprint_is_the_sha256_of_the_file_bytes() { + assert_eq!( + sha256_fingerprint(b"abc"), + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + let fingerprint = PriceCatalogs::with_user_catalog(None).bundled_sha256(); + assert_eq!( + fingerprint, + sha256_fingerprint(BUNDLED_CATALOG_JSON.as_bytes()) + ); + assert_eq!(fingerprint.len(), "sha256:".len() + 64); + } + + // Lookup, mirroring NeMo Relay's model pricing tests. + + #[test] + fn lookup_matches_aliases_case_insensitively() { + let catalog = catalog(json!([{ + "provider": "configured", + "model_id": "configured-model", + "aliases": ["configured-model-2026-06-04"], + "pricing_as_of": "2026-06-04", + "pricing_source": "file:///tmp/pricing.json", + "rates": {"input_per_million": 0.15, "output_per_million": 0.60}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"} + }])); + + let entry = catalog + .pricing_for(None, "CONFIGURED-MODEL-2026-06-04") + .expect("alias"); + + assert_eq!(entry.model_id, "configured-model"); + assert_eq!(entry.currency, "USD"); + assert_eq!(entry.unit, PricingUnit::PerToken); + } + + #[test] + fn lookup_matches_routed_model_names_by_suffix() { + let catalog = catalog(json!([flat_entry("openai", "gpt-4o-mini", 0.15, 0.6)])); + + assert_eq!( + found(&catalog, None, "azure/openai/gpt-4o-mini"), + ("openai", "gpt-4o-mini") + ); + assert_eq!( + found(&catalog, None, "openai/openai/gpt-4o-mini"), + ("openai", "gpt-4o-mini") + ); + } + + #[test] + fn lookup_prefers_the_exact_routed_model_before_a_suffix() { + let catalog = catalog(json!([ + flat_entry("openai", "gpt-4o-mini", 0.15, 0.6), + flat_entry("azure-openai", "azure/openai/gpt-4o-mini", 1.0, 2.0) + ])); + + assert_eq!( + found(&catalog, None, "azure/openai/gpt-4o-mini"), + ("azure-openai", "azure/openai/gpt-4o-mini") + ); + } + + #[test] + fn lookup_tells_apart_one_model_id_of_two_providers() { + let catalog = catalog(json!([ + flat_entry("openai", "gpt-4o-mini", 0.15, 0.6), + flat_entry("azure/openai", "gpt-4o-mini", 1.0, 2.0) + ])); + + assert_eq!( + found(&catalog, Some("openai"), "gpt-4o-mini"), + ("openai", "gpt-4o-mini") + ); + assert_eq!( + found(&catalog, Some("azure/openai"), "gpt-4o-mini"), + ("azure/openai", "gpt-4o-mini") + ); + } + + #[test] + fn lookup_normalizes_providers_and_ignores_blank_or_unknown_models() { + let catalog = catalog(json!([{ + "provider": "provider-a", + "model_id": "priced-model", + "aliases": ["alias-model"], + "pricing_as_of": "2026-06-04", + "pricing_source": "test", + "rates": {"input_per_million": 1.0, "output_per_million": 2.0}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"} + }])); + + assert_eq!( + found(&catalog, Some("/provider-a/"), "provider-a/priced-model"), + ("provider-a", "priced-model") + ); + assert_eq!( + found(&catalog, None, " ALIAS-MODEL "), + ("provider-a", "priced-model") + ); + assert!(catalog.pricing_for(Some("other"), "priced-model").is_none()); + assert_eq!( + found(&catalog, Some("provider-a"), "path/alias-model"), + ("provider-a", "priced-model") + ); + assert!(catalog.pricing_for(None, " ").is_none()); + assert!(catalog.pricing_for(None, "missing-model").is_none()); + } + + #[test] + fn caller_catalog_takes_precedence_and_alone_prices_self_hosted_models() { + let bundled = catalog(json!([ + flat_entry("openai", "gpt-4o-mini", 0.15, 0.6), + flat_entry("openai", "gpt-4o", 2.5, 10.0) + ])); + let user = UserPriceCatalog { + path: PathBuf::from("/prices.json"), + sha256: "sha256:user".to_string(), + catalog: catalog(json!([flat_entry("openai", "gpt-4o-mini", 1.0, 2.0)])), + }; + let prices = PriceCatalogs::from_parts(Some(&bundled), "sha256:bundled", Some(&user)); + + let origin = |model: &str, user_only: bool| { + prices + .find(Some("openai"), model, user_only) + .map(|(entry, origin)| (entry.provider.as_str(), origin)) + }; + + assert_eq!( + origin("gpt-4o-mini", false), + Some(("openai", PriceOrigin::User)) + ); + assert_eq!( + origin("gpt-4o", false), + Some(("openai", PriceOrigin::Bundled)) + ); + assert_eq!(origin("gpt-4o", true), None); + assert_eq!( + origin("gpt-4o-mini", true), + Some(("openai", PriceOrigin::User)) + ); + assert_eq!(prices.bundled_sha256(), "sha256:bundled"); + assert_eq!(prices.user_sha256(), Some("sha256:user")); + } + + #[test] + fn unrelated_user_provider_cannot_shadow_matching_bundled_price() { + let bundled = catalog(json!([flat_entry("openai", "model", 1.0, 2.0)])); + let user = UserPriceCatalog { + path: PathBuf::from("/prices.json"), + sha256: "sha256:user".to_string(), + catalog: catalog(json!([flat_entry("custom-route", "model", 3.0, 4.0)])), + }; + let prices = PriceCatalogs::from_parts(Some(&bundled), "sha256:bundled", Some(&user)); + let (entry, origin) = prices.find(Some("openai"), "model", false).unwrap(); + assert_eq!(entry.provider, "openai"); + assert_eq!(origin, PriceOrigin::Bundled); + let (entry, origin) = prices.find(Some("custom-route"), "model", true).unwrap(); + assert_eq!(entry.provider, "custom-route"); + assert_eq!(origin, PriceOrigin::User); + } + + // Validation, mirroring NeMo Relay's catalog validation tests. + + #[test] + fn validation_rejects_an_unsupported_version() { + let error = PricingCatalog::from_json(br#"{"version": 2, "entries": []}"#) + .expect_err("version 2") + .to_string(); + + assert!( + error.contains("unsupported model pricing catalog version 2"), + "{error}" + ); + } + + #[test] + fn validation_rejects_duplicate_model_aliases_of_one_provider() { + let mut other = flat_entry("a", "other-model", 1.0, 1.0); + other["aliases"] = json!(["SAME-MODEL"]); + + let error = catalog_error(json!([flat_entry("a", "same-model", 1.0, 1.0), other])); + + assert!( + error.contains("duplicate model pricing alias 'a/same-model'"), + "{error}" + ); + } + + #[test] + fn validation_rejects_blank_required_fields() { + for field in [ + "provider", + "model_id", + "currency", + "pricing_as_of", + "pricing_source", + ] { + let mut entry = flat_entry("configured", "configured-model", 1.0, 2.0); + entry[field] = json!(" "); + + let error = catalog_error(json!([entry])); + + assert!( + error.contains(&format!("empty {field}")), + "{field}: {error}" + ); + } + } + + #[test] + fn validation_rejects_negative_or_too_large_rates() { + for (field, value) in [ + ("input_per_million", -0.1), + ("output_per_million", -0.1), + ("cache_read_per_million", -0.1), + ("cache_write_per_million", -0.1), + ("input_per_million", 1_000_000.0), + ] { + let mut entry = flat_entry("configured", "configured-model", 1.0, 2.0); + entry["rates"][field] = json!(value); + + let error = catalog_error(json!([entry])); + + assert!( + error.contains(&format!("rates.{field}")), + "{field}: {error}" + ); + } + } + + #[test] + fn validation_requires_rates_for_per_token_entries() { + let mut entry = flat_entry("broken", "missing-rates", 1.0, 2.0); + entry.as_object_mut().expect("entry").remove("rates"); + + let error = catalog_error(json!([entry])); + + assert!(error.contains("empty rates or rate_schedule"), "{error}"); + } + + #[test] + fn non_token_units_are_valid_without_rates() { + let mut entry = flat_entry("self-hosted", "nemotron-owned", 1.0, 2.0); + entry.as_object_mut().expect("entry").remove("rates"); + entry["unit"] = json!("gpu_hour"); + + let catalog = catalog(json!([entry])); + + assert_eq!(catalog.entries[0].unit, PricingUnit::GpuHour); + } + + #[test] + fn validation_rejects_invalid_rate_schedules() { + let empty = catalog_error(json!([threshold_entry(json!([]))])); + let reversed = catalog_error(json!([threshold_entry(json!([ + {"min_prompt_tokens": 20, "max_prompt_tokens": 10, "rates": rates(1.0)} + ]))])); + let negative = catalog_error(json!([threshold_entry(json!([{"rates": rates(-1.0)}]))])); + + assert!(empty.contains("rate_schedule.tiers"), "{empty}"); + assert!( + reversed.contains("rate_schedule.tiers.prompt_tokens"), + "{reversed}" + ); + assert!( + negative.contains("rate_schedule.tiers[0].rates.input_per_million"), + "{negative}" + ); + } + + #[test] + fn validation_rejects_unknown_fields_at_every_level() { + let entry = flat_entry("configured", "configured-model", 1.0, 2.0); + let with = |pointer: &str, field: &str| { + let mut catalog = json!({"version": 1, "entries": [entry.clone()]}); + catalog + .pointer_mut(pointer) + .and_then(Value::as_object_mut) + .expect("object") + .insert(field.to_string(), json!(true)); + catalog + }; + let mut tiers = json!({"version": 1, "entries": [threshold_entry(json!([ + {"min_prompt_tokens": 0, "rates": rates(1.0)} + ]))]}); + tiers["entries"][0]["rate_schedule"]["tierz"] = json!([]); + let tier = json!({"version": 1, "entries": [threshold_entry(json!([ + {"min_prompt_toknes": 0, "rates": rates(1.0)} + ]))]}); + + for (catalog, field) in [ + (with("", "unexpected_catalog"), "unexpected_catalog"), + (with("/entries/0", "unexpected_entry"), "unexpected_entry"), + ( + with("/entries/0/rates", "cache_writ_per_million"), + "cache_writ_per_million", + ), + ( + with("/entries/0/prompt_cache", "read_accountng"), + "read_accountng", + ), + (tiers, "tierz"), + (tier, "min_prompt_toknes"), + ] { + let error = PricingCatalog::from_json(catalog.to_string().as_bytes()) + .expect_err("unknown field") + .to_string(); + + assert!( + error.contains(&format!("unknown field `{field}`")), + "{field}: {error}" + ); + } + } + + // Rate tiers. + + #[test] + fn a_rate_schedule_resolves_only_when_one_tier_covers_every_request() { + let catalog = catalog(json!([threshold_entry(json!([ + {"max_prompt_tokens": 200_000, "rates": rates(1.0)}, + {"min_prompt_tokens": 200_001, "rates": rates(10.0)} + ]))])); + let entry = &catalog.entries[0]; + let input = + |peak: Option| entry.rates_up_to(peak).map(|rates| rates.input_per_million); + + assert_eq!(input(Some(0)), Some(1.0)); + assert_eq!(input(Some(200_000)), Some(1.0)); + assert_eq!(input(Some(200_001)), None); + assert_eq!(input(None), None); + } + + #[test] + fn a_rate_schedule_follows_tier_order_like_nemo_relay() { + let catalog = catalog(json!([threshold_entry(json!([ + {"min_prompt_tokens": 100, "rates": rates(10.0)}, + {"max_prompt_tokens": 99, "rates": rates(1.0)} + ]))])); + let gap = catalog_with_first_tier_from(1); + let entry = &catalog.entries[0]; + + assert_eq!( + entry + .rates_up_to(Some(99)) + .map(|rates| rates.input_per_million), + Some(1.0) + ); + assert_eq!(entry.rates_up_to(Some(100)), None); + assert_eq!(gap.entries[0].rates_up_to(Some(10)), None); + } + + fn catalog_with_first_tier_from(min_prompt_tokens: u64) -> PricingCatalog { + catalog(json!([threshold_entry(json!([ + {"min_prompt_tokens": min_prompt_tokens, "rates": rates(1.0)} + ]))])) + } + + #[test] + fn flat_rates_apply_whatever_the_peak() { + let catalog = catalog(json!([flat_entry("openai", "gpt-4o-mini", 0.15, 0.6)])); + + for peak in [None, Some(0), Some(u64::MAX)] { + assert_eq!( + catalog.entries[0] + .rates_up_to(peak) + .map(|rates| rates.input_per_million), + Some(0.15) + ); + } + } + + // Rate conversion. + + #[test] + fn rates_convert_to_whole_nano_usd_per_million_tokens() { + for (rate, expected) in [ + (0.0, Some(0)), + (0.075, Some(75_000_000)), + (0.3, Some(300_000_000)), + (2.5, Some(2_500_000_000)), + (3.75, Some(3_750_000_000)), + (1e-10, Some(0)), + (6e-10, Some(1)), + (999_999.999_999_999, Some(999_999_999_999_999)), + (1_000_000.0, None), + (-0.000_001, None), + (f64::NAN, None), + (f64::INFINITY, None), + ] { + assert_eq!(nano_usd_per_million_tokens(rate), expected, "rate {rate}"); + } + } +} diff --git a/crates/fabric-core/src/pricing.rs b/crates/fabric-core/src/pricing.rs index d8c37bcf9..191c1f965 100644 --- a/crates/fabric-core/src/pricing.rs +++ b/crates/fabric-core/src/pricing.rs @@ -1,11 +1,108 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Persisted pricing-basis types for the stable model-usage result contract. +//! Catalog prices are estimates, separate from provider-reported amounts. use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use crate::measurement::{ + Attribution, Availability, Measurement, MeasurementSource, MeasurementUnit, +}; +use crate::price_catalog::{ + PriceCatalogs, PriceOrigin, PricingUnit, TokenPricingRates, UserPriceCatalog, + nano_usd_per_million_tokens, +}; +use crate::usage_accounting::GapReason; +use crate::usage_normalization::{AccountingDimension, AccountingGap, ModelUsage, ObservedUsage}; + +pub(crate) const OPENAI_ENDPOINT_ENV_KEYS: [&str; 2] = ["OPENAI_BASE_URL", "OPENAI_API_BASE"]; +pub(crate) const ANTHROPIC_ENDPOINT_ENV_KEYS: [&str; 1] = ["ANTHROPIC_BASE_URL"]; + +/// Configured and inherited endpoint overrides must still identify a native +/// endpoint before provider list prices apply. An adapter can inherit URL +/// settings even when models..base_url is absent. +pub(crate) fn native_catalog_route( + model: &crate::ModelConfig, + forwarded: &std::collections::BTreeMap, + inherited: impl Fn(&str) -> Option, +) -> bool { + let (keys, urls): (&[&str], &[&str]) = match model.provider.as_str() { + "openai" => (&OPENAI_ENDPOINT_ENV_KEYS, &["https://api.openai.com/v1"]), + "anthropic" => ( + &ANTHROPIC_ENDPOINT_ENV_KEYS, + &["https://api.anthropic.com", "https://api.anthropic.com/v1"], + ), + _ => return false, + }; + let native = |url: &str| urls.contains(&url.trim_end_matches('/')); + if let Some(url) = &model.base_url { + return native(url); + } + keys.iter().all(|key| { + forwarded + .get(*key) + .cloned() + .or_else(|| inherited(key)) + .is_none_or(|url| native(&url)) + }) +} + +/// Startup route evidence, qualified for the provider/model actually accounted. +/// An unused custom role must not disable a different model's native tariff. +#[derive(Debug, Clone)] +pub(crate) struct NativePricingRoutes { + models: Vec, +} + +#[derive(Debug, Clone)] +struct NativeModelRoute { + provider: String, + model: String, + native: bool, +} + +impl NativePricingRoutes { + pub(crate) fn from_models( + models: &std::collections::BTreeMap, + endpoints: &std::collections::BTreeMap, + ) -> Self { + Self { + models: models + .values() + .map(|model| NativeModelRoute { + provider: model.provider.clone(), + model: model.model.clone(), + native: native_catalog_route(model, endpoints, |_| None), + }) + .collect(), + } + } + + pub(crate) fn allows(&self, provider: Option<&str>, model: &str) -> bool { + let mut exact = self + .models + .iter() + .filter(|route| Some(route.provider.as_str()) == provider && route.model == model) + .peekable(); + if exact.peek().is_some() { + return exact.all(|route| route.native); + } + // A native provider route still supports a served-model alias. Mixed + // routes for that provider cannot establish which endpoint served it. + let mut provider_routes = self + .models + .iter() + .filter(|route| Some(route.provider.as_str()) == provider) + .peekable(); + if provider_routes.peek().is_some() { + return provider_routes.all(|route| route.native); + } + // Preserve the prior all-native case for source-qualified child models. + !self.models.is_empty() && self.models.iter().all(|route| route.native) + } +} + /// Fields that must agree before a report combines catalog estimates. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] @@ -60,3 +157,587 @@ pub struct AppliedPricing { /// One selected rate set; distinct selections are retained in `model_usage::TokenCost.applied_rates`. pub rates: RateSnapshot, } + +impl RateSnapshot { + fn from_rates(rates: TokenPricingRates) -> Result { + let convert = |rate| { + nano_usd_per_million_tokens(rate) + .and_then(|value| u64::try_from(value).ok()) + .ok_or("price_catalog_invalid") + }; + Ok(Self { + input: convert(rates.input_per_million)?, + output: convert(rates.output_per_million)?, + cache_read: rates.cache_read_per_million.map(convert).transpose()?, + cache_write: rates.cache_write_per_million.map(convert).transpose()?, + }) + } + + fn price( + &self, + input: Option, + cached: Option, + written: Option, + output: Option, + ) -> Result { + rounded_price(self.charges(input, cached, written, output)?) + } + + fn charges( + &self, + input: Option, + cached: Option, + written: Option, + output: Option, + ) -> Result { + let input = input.ok_or("pricing_input_unreported")?; + let output = output.ok_or("pricing_output_unreported")?; + // A missing cache category cannot affect the price only when it is + // charged exactly like ordinary input, or the entire input is zero. + let component = |count: Option, rate: Option| match count { + Some(count) => match rate { + Some(rate) => Ok((count, rate)), + None if count == 0 => Ok((0, 0)), + None => Err("cache_rate_unreported"), + }, + None if input == 0 || rate == Some(self.input) => Ok((0, self.input)), + None => Err("pricing_cache_unreported"), + }; + let (cached, read_rate) = component(cached, self.cache_read)?; + let (written, write_rate) = component(written, self.cache_write)?; + let uncached = input + .checked_sub(cached) + .and_then(|v| v.checked_sub(written)) + .ok_or("inconsistent_usage")?; + let charges = [ + (uncached, self.input), + (cached, read_rate), + (written, write_rate), + (output, self.output), + ]; + let sum = charges + .into_iter() + .try_fold(0_u128, |sum, (count, rate)| { + sum.checked_add(u128::from(count).checked_mul(u128::from(rate))?) + }) + .ok_or("cost_out_of_range")?; + Ok(sum) + } + + fn observed_charges( + &self, + observed: &ObservedUsage, + aligned: bool, + write_tariff_known: bool, + ) -> Option { + // Categories are disjoint. Without aligned input/cache observations, + // retain independently priceable categories instead of subtracting + // counters from different source-call subsets. + let mut charges = 0_u128; + let mut any = false; + let mut add = |count: Option, rate: Option| -> Option<()> { + if let (Some(count), Some(rate)) = (count, rate) { + charges = charges.checked_add(u128::from(count).checked_mul(u128::from(rate))?)?; + any = true; + } + Some(()) + }; + add(observed.output_tokens, Some(self.output))?; + let equal_rates = + self.cache_read == Some(self.input) && self.cache_write == Some(self.input); + if let Some(input) = observed + .input_tokens + .filter(|_| equal_rates && write_tariff_known) + { + add(Some(input), Some(self.input))?; + } else { + if aligned { + let cached = observed + .cached_input_tokens + .or((self.cache_read == Some(self.input)).then_some(0)); + let written = observed + .cache_write_input_tokens + .or((write_tariff_known && self.cache_write == Some(self.input)).then_some(0)); + if let (Some(input), Some(cached), Some(written)) = + (observed.input_tokens, cached, written) + { + add( + input + .checked_sub(cached) + .and_then(|v| v.checked_sub(written)), + Some(self.input), + )?; + } + } + add(observed.cached_input_tokens, self.cache_read)?; + if write_tariff_known { + add(observed.cache_write_input_tokens, self.cache_write)?; + } + } + any.then_some(charges) + } + + fn observed_price( + &self, + observed: &ObservedUsage, + aligned: bool, + write_tariff_known: bool, + ) -> Option { + rounded_price(self.observed_charges(observed, aligned, write_tariff_known)?).ok() + } +} + +fn rounded_price(charges: u128) -> Result { + u64::try_from(charges.checked_add(500_000).ok_or("cost_out_of_range")? / 1_000_000) + .map_err(|_| "cost_out_of_range") +} + +fn estimate(value: Result, source: MeasurementSource) -> Measurement { + Measurement { + availability: if value.is_ok() { + Availability::Measured + } else { + Availability::Unavailable + }, + value: value.ok(), + unit: MeasurementUnit::NanoUsd, + source, + attribution: if value.is_ok() { + Attribution::Invocation + } else { + Attribution::None + }, + reason: value.err().map(str::to_owned), + } +} + +fn update_price_gaps(gaps: &mut Vec, cost: &Measurement) { + gaps.retain(|gap| gap.dimensions != [AccountingDimension::EstimatedCost]); + if cost.value.is_some() { + return; + } + let mut reasons: std::collections::BTreeSet<_> = gaps + .iter() + .filter(|gap| { + gap.dimensions.iter().any(|dimension| { + matches!( + dimension, + AccountingDimension::InputTokens | AccountingDimension::OutputTokens + ) + }) + }) + .map(|gap| gap.reason) + .collect(); + if reasons.is_empty() { + reasons.insert(match cost.reason.as_deref() { + Some("model_scope_unverified" | "inconsistent_usage" | "provider_price_unverified") => { + GapReason::ScopeUnverified + } + _ => GapReason::MissingUsage, + }); + } + gaps.extend(reasons.into_iter().map(|reason| AccountingGap { + dimensions: vec![AccountingDimension::EstimatedCost], + reason, + })); +} + +/// Price validated normalized usage with a route decision for each accounted model. +/// An unqualified route permits only explicitly supplied prices. +/// +/// Price with validated original request boundaries when an aggregate spans tiers. +/// Keep this evidence explicit: normalized aggregates cannot recover request sizes. +pub(crate) fn apply_pricing_with_evidence( + usage: &mut ModelUsage, + user: Option<&UserPriceCatalog>, + bundled_allowed: impl Fn(Option<&str>, &str) -> bool, + evidence: Option<&crate::usage_accounting::UsageAccounting>, +) { + let prices = PriceCatalogs::with_user_catalog(user); + if !prices.bundled_is_valid() { + usage.token_cost.estimated = estimate( + Err("price_catalog_invalid"), + MeasurementSource::FabricPriceCatalog, + ); + return; + } + let basis = PricingBasis { + basis: "catalog".into(), + currency: "USD".into(), + normalization_version: "cache_reasoning_inclusive_v1".into(), + assumption_policy: "applicable_tariff_no_missing_counts_v1".into(), + bundled_catalog_sha256: prices.bundled_sha256().into(), + user_catalog_sha256: prices.user_sha256().map(str::to_owned), + }; + usage.token_cost.pricing = Some(basis.clone()); + if usage.accounting.is_none() { + usage.token_cost.estimated = estimate( + Err("legacy_unverified"), + MeasurementSource::FabricPriceCatalog, + ); + return; + } + let mut any_user = false; + let mut root_cache_read_required = false; + let mut root_cache_write_required = false; + for model in &mut usage.models { + let cost = &mut model.usage.token_cost; + cost.applied = None; + cost.applied_calls.clear(); + model.accounting.observed.estimated_cost_nano_usd = None; + cost.pricing = Some(basis.clone()); + let selected = prices.find( + model.provider.as_deref(), + &model.model, + !bundled_allowed(model.provider.as_deref(), &model.model), + ); + let Some((entry, origin)) = selected else { + cost.estimated = estimate( + Err("model_not_in_catalog"), + MeasurementSource::FabricPriceCatalog, + ); + update_price_gaps(&mut model.accounting.gaps, &cost.estimated); + continue; + }; + let source = match origin { + PriceOrigin::User => MeasurementSource::UserPriceCatalog, + PriceOrigin::Bundled => MeasurementSource::FabricPriceCatalog, + }; + any_user |= origin == PriceOrigin::User; + if origin == PriceOrigin::Bundled + && !model + .provider + .as_ref() + .is_some_and(|provider| provider.eq_ignore_ascii_case(&entry.provider)) + { + cost.estimated = estimate(Err("provider_price_unverified"), source); + update_price_gaps(&mut model.accounting.gaps, &cost.estimated); + continue; + } + let mut result = (|| { + if entry.unit != PricingUnit::PerToken || !entry.currency.eq_ignore_ascii_case("USD") { + return Err("price_currency_or_unit_unsupported"); + } + // A complete invocation's total input is also a safe upper bound + // on each request; a subset peak never prices a larger complete set. + let complete_peak = model + .usage + .peak_request_input_tokens + .value + .or(model.usage.input_tokens.value); + let complete_rates = entry + .rates_up_to(complete_peak) + .map(RateSnapshot::from_rates) + .transpose()?; + let observed_peak = if model.accounting.observed_tokens_aligned { + model + .accounting + .observed + .peak_request_input_tokens + .or(model.accounting.observed.input_tokens) + } else { + None + }; + let observed_rates = match &complete_rates { + Some(rates) => Some(rates.clone()), + None => entry + .rates_up_to(observed_peak) + .map(RateSnapshot::from_rates) + .transpose()?, + }; + let write_tariff_known = !entry.provider.eq_ignore_ascii_case("anthropic") + || model.usage.cache_write_input_tokens.value == Some(0); + let complete = match &complete_rates { + Some(rates) => { + root_cache_read_required |= rates.cache_read != Some(rates.input); + root_cache_write_required |= rates.cache_write != Some(rates.input); + if !write_tariff_known && model.usage.input_tokens.value != Some(0) { + Err("cache_write_tariff_unverified") + } else { + rates.price( + model.usage.input_tokens.value, + model.usage.cached_input_tokens.value, + model.usage.cache_write_input_tokens.value, + model.usage.output_tokens.value, + ) + } + } + None => Err("tiered_price_unresolvable"), + }; + let observed = complete.ok().or_else(|| { + observed_rates.as_ref().and_then(|rates| { + rates.observed_price( + &model.accounting.observed, + model.accounting.observed_tokens_aligned + || (model.usage.input_tokens.value.is_some() + && model.usage.cached_input_tokens.value.is_some() + && model.usage.cache_write_input_tokens.value.is_some()), + !entry.provider.eq_ignore_ascii_case("anthropic") + || model.accounting.observed.cache_write_input_tokens == Some(0), + ) + }) + }); + model.accounting.observed.estimated_cost_nano_usd = observed; + let rates = if complete.is_ok() { + complete_rates + } else { + observed_rates.or(complete_rates) + }; + let Some(rates) = rates else { + return complete; + }; + cost.applied = Some(AppliedPricing { + catalog_sha256: match origin { + PriceOrigin::Bundled => prices.bundled_sha256(), + PriceOrigin::User => prices.user_sha256().expect("selected user catalog"), + } + .into(), + provider: entry.provider.clone(), + model_id: entry.model_id.clone(), + pricing_as_of: entry.pricing_as_of.clone(), + pricing_source: entry.pricing_source.clone(), + rates, + }); + complete + })(); + if let Some(raw) = evidence + .and_then(|e| { + e.models + .iter() + .find(|raw| raw.model == model.model && raw.provider == model.provider) + }) + .filter(|raw| { + !raw.requests.is_empty() + && entry.unit == PricingUnit::PerToken + && entry.currency.eq_ignore_ascii_case("USD") + }) + { + let mut charges = Some(0_u128); + let mut complete = raw.requests_exhaustive; + let mut any = false; + for request in &raw.requests { + let aligned = request.gap.is_none() || request.tokens_aligned; + let normalized = + crate::usage_normalization::normalized_request(&request.usage, !aligned); + let priced: Result<(u128, RateSnapshot, bool), &'static str> = (|| { + let selected = if entry.rate_schedule.is_some() { + entry.rates_at( + normalized + .input_tokens + .value + .ok_or("pricing_input_unreported")?, + ) + } else { + entry.rates + }; + let rates = + RateSnapshot::from_rates(selected.ok_or("tiered_price_unresolvable")?)?; + let write_tariff_known = !entry.provider.eq_ignore_ascii_case("anthropic") + || normalized.cache_write_input_tokens.value == Some(0) + || normalized.input_tokens.value == Some(0); + let complete_charge = if aligned && write_tariff_known { + rates + .charges( + normalized.input_tokens.value, + normalized.cached_input_tokens.value, + normalized.cache_write_input_tokens.value, + normalized.output_tokens.value, + ) + .ok() + } else { + None + }; + let observed = ObservedUsage { + input_tokens: normalized.input_tokens.value, + cached_input_tokens: normalized.cached_input_tokens.value, + cache_write_input_tokens: normalized.cache_write_input_tokens.value, + output_tokens: normalized.output_tokens.value, + ..Default::default() + }; + let charge = complete_charge + .or_else(|| rates.observed_charges(&observed, aligned, write_tariff_known)) + .ok_or("pricing_usage_unavailable")?; + Ok((charge, rates, complete_charge.is_some())) + })(); + match priced { + Ok((charge, rates, all_parts_priced)) => { + any = true; + charges = charges.and_then(|total| total.checked_add(charge)); + root_cache_read_required |= rates.cache_read != Some(rates.input); + root_cache_write_required |= rates.cache_write != Some(rates.input); + cost.applied_calls.push(Some(AppliedPricing { + catalog_sha256: match origin { + PriceOrigin::Bundled => prices.bundled_sha256(), + PriceOrigin::User => { + prices.user_sha256().expect("selected catalog") + } + } + .into(), + provider: entry.provider.clone(), + model_id: entry.model_id.clone(), + pricing_as_of: entry.pricing_as_of.clone(), + pricing_source: entry.pricing_source.clone(), + rates, + })); + complete &= request.gap.is_none() && all_parts_priced; + } + Err(_) => { + complete = false; + cost.applied_calls.push(None); + } + } + } + let amount = charges + .and_then(|charges| rounded_price(charges).ok()) + .filter(|_| any); + if result.is_err() + && complete + && model.usage.input_tokens.value.is_some() + && model.usage.output_tokens.value.is_some() + { + result = amount.ok_or("cost_out_of_range"); + cost.applied = None; + } + if let Some(amount) = result.as_ref().ok().copied().or(amount) { + model.accounting.observed.estimated_cost_nano_usd = Some(amount); + } + } + cost.estimated = estimate(result, source); + update_price_gaps(&mut model.accounting.gaps, &cost.estimated); + } + let inconsistent = [ + &usage.input_tokens, + &usage.cached_input_tokens, + &usage.cache_write_input_tokens, + &usage.output_tokens, + ] + .iter() + .any(|m| m.reason.as_deref() == Some("inconsistent_usage")); + let accounting = usage.accounting.as_mut().expect("checked accounting"); + let complete = if !accounting.models_exhaustive || usage.models.is_empty() { + Err("model_scope_unverified") + } else if inconsistent { + Err("inconsistent_usage") + } else if usage.models.iter().any(|model| { + model.usage.token_cost.estimated.reason.as_deref() == Some("cache_write_tariff_unverified") + }) { + Err("cache_write_tariff_unverified") + } else if usage.input_tokens.value.is_none() + || usage.output_tokens.value.is_none() + || (root_cache_read_required + && usage.cached_input_tokens.value.is_none() + && usage.input_tokens.value != Some(0)) + || (root_cache_write_required + && usage.cache_write_input_tokens.value.is_none() + && usage.input_tokens.value != Some(0)) + { + Err("pricing_usage_incomplete") + } else { + usage.models.iter().try_fold(0_u64, |sum, model| { + sum.checked_add( + model + .usage + .token_cost + .estimated + .value + .ok_or("unpriced_model")?, + ) + .ok_or("cost_out_of_range") + }) + }; + let observed = if inconsistent { + None + } else { + let values: Vec<_> = usage + .models + .iter() + .filter_map(|model| model.accounting.observed.estimated_cost_nano_usd) + .collect(); + if values.is_empty() { + None + } else { + values.into_iter().try_fold(0_u64, u64::checked_add) + } + }; + accounting.observed.estimated_cost_nano_usd = complete.ok().or(observed); + usage.token_cost.estimated = estimate( + complete, + if any_user { + MeasurementSource::UserPriceCatalog + } else { + MeasurementSource::FabricPriceCatalog + }, + ); + update_price_gaps(&mut accounting.gaps, &usage.token_cost.estimated); +} + +#[cfg(test)] +mod tests { + use super::native_catalog_route; + use serde_json::json; + use std::collections::BTreeMap; + + #[test] + fn native_pricing_routes_preserve_ambiguity_and_served_aliases() { + let mut models = serde_json::from_value(json!({ + "default": {"provider": "openai", "model": "gpt-5.4"}, + "auxiliary": {"provider": "openai", "model": "gpt-5.4", + "base_url": "https://custom.invalid/v1"} + })) + .unwrap(); + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(!routes.allows(Some("openai"), "gpt-5.4")); + assert!(!routes.allows(Some("openai"), "served-alias")); + assert!(!routes.allows(Some("anthropic"), "child-model")); + + models.remove("auxiliary"); + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(routes.allows(Some("openai"), "served-alias")); + assert!(routes.allows(Some("anthropic"), "source-qualified-child")); + + models.get_mut("default").unwrap().base_url = Some("https://custom.invalid/v1".into()); + models.insert( + "unused_native".into(), + serde_json::from_value(json!({ + "provider": "openai", "model": "other-model" + })) + .unwrap(), + ); + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(!routes.allows(Some("openai"), "gpt-5.4")); + assert!(routes.allows(Some("openai"), "other-model")); + assert!(!routes.allows(Some("openai"), "served-alias")); + + let empty = super::NativePricingRoutes::from_models(&BTreeMap::new(), &BTreeMap::new()); + assert!(!empty.allows(Some("openai"), "gpt-5.4")); + } + + #[test] + fn inherited_and_forwarded_custom_routes_require_explicit_prices() { + for (configured, inherited, forwarded, expected) in [ + (None, None, None, true), + ( + Some("https://api.openai.com/v1/"), + Some("https://proxy.invalid/v1"), + None, + true, + ), + (Some("https://proxy.invalid/v1"), None, None, false), + (None, Some("https://proxy.invalid/v1"), None, false), + (None, None, Some("https://proxy.invalid/v1"), false), + (None, Some("https://api.openai.com/v1"), None, true), + ] { + let model = serde_json::from_value( + json!({"provider":"openai","model":"gpt-5.4","base_url":configured}), + ) + .unwrap(); + let env = forwarded + .map(|url| BTreeMap::from([("OPENAI_BASE_URL".into(), url.into())])) + .unwrap_or_default(); + assert_eq!( + native_catalog_route(&model, &env, |_| inherited.map(str::to_owned)), + expected + ); + } + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 2b7a780b4..349bafef5 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -873,6 +873,7 @@ struct LocalAdapterHost { relay_config: Option, // Adapter startup values stay in memory, never in a serialized handle or plan. environment_env: BTreeMap, + bundled_pricing_allowed: crate::pricing::NativePricingRoutes, } struct LocalServiceRecord { @@ -2141,6 +2142,7 @@ fn run_local_host_invocation_with_timeout( relay_config, fabric_home, fabric_invocation, + host_guard.bundled_pricing_allowed.clone(), )) } Err(error) => { @@ -2166,6 +2168,7 @@ fn run_local_host_invocation_with_timeout( relay_config, fabric_home, fabric_invocation, + bundled_pricing_allowed, ) = exchange_result?; let agent_result: AgentRunResult = serde_json::from_value(adapter_output).map_err(|error| { @@ -2324,7 +2327,13 @@ fn run_local_host_invocation_with_timeout( let model_usage = match agent_result .usage .as_ref() - .map(|raw| crate::model_usage::normalize_model_usage(raw)) + .map(|raw| { + crate::model_usage::normalize_model_usage_for_routes( + raw, + plan.user_price_catalog.as_ref(), + |provider, model| bundled_pricing_allowed.allows(provider, model), + ) + }) .transpose() { Ok(usage) => usage.flatten(), @@ -2584,6 +2593,10 @@ fn spawn_local_host( if let Some(root) = artifacts.root.as_ref() { command.env("FABRIC_ARTIFACTS", root); } + let bundled_pricing_allowed = + snapshot_pricing_route(&plan.config.models, &mut command, |name| { + std::env::var_os(name) + }); // Start suspended so the host joins its job before it can start a process. #[cfg(windows)] std::os::windows::process::CommandExt::creation_flags( @@ -2682,9 +2695,46 @@ fn spawn_local_host( artifacts, relay_config, environment_env, + bundled_pricing_allowed, }) } +// Freeze endpoint inheritance on the command itself so pricing describes the +// persistent child's startup route, not a later mutation of the controller's env. +fn snapshot_pricing_route( + models: &BTreeMap, + command: &mut Command, + inherited: impl Fn(&str) -> Option, +) -> crate::pricing::NativePricingRoutes { + let mut endpoints = BTreeMap::new(); + for name in crate::pricing::OPENAI_ENDPOINT_ENV_KEYS + .into_iter() + .chain(crate::pricing::ANTHROPIC_ENDPOINT_ENV_KEYS) + { + let value = command + .get_envs() + .find_map(|(key, value)| { + let matches = if cfg!(windows) { + key.to_string_lossy().eq_ignore_ascii_case(name) + } else { + key == name + }; + matches.then(|| value.map(std::ffi::OsStr::to_owned)) + }) + .unwrap_or_else(|| inherited(name)); + match value { + Some(value) => { + endpoints.insert(name.to_string(), value.to_string_lossy().into_owned()); + command.env(name, value); + } + None => { + command.env_remove(name); + } + } + } + crate::pricing::NativePricingRoutes::from_models(models, &endpoints) +} + fn local_host_command(plan: &RunPlan, runtime: &RuntimeHandle) -> Result<(Command, String)> { match adapter_kind(plan) { AdapterKind::Process => process_local_host_command(plan, runtime), @@ -6112,6 +6162,39 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn unused_custom_model_does_not_disable_native_call_pricing() { + let models = serde_json::from_value(serde_json::json!({ + "default": {"provider": "openai", "model": "gpt-5.4"}, + "unused": { + "provider": "openai", "model": "other-model", + "base_url": "https://custom.invalid/v1" + } + })) + .expect("model roles"); + let allowed = snapshot_pricing_route(&models, &mut Command::new("unused"), |_| None); + let raw = serde_json::from_value(serde_json::json!({ + "extensions": {"nemo_fabric.usage_accounting.v1": { + "models_exhaustive": true, "gaps": [], "models": [{ + "provider": "openai", "model": "gpt-5.4", "gaps": [], + "reported": { + "input_tokens": 100, "cached_input_tokens": 20, + "cache_write_input_tokens": 0, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true + } + }] + }} + })) + .expect("source-established usage"); + let usage = + crate::model_usage::normalize_model_usage_for_routes(&raw, None, |provider, model| { + allowed.allows(provider, model) + }) + .expect("valid accounting") + .expect("normalized accounting"); + assert!(usage.cost.estimated.amount_nano_usd.is_some()); + } + #[test] fn local_host_returns_flat_supported_accounting_with_the_normalized_result() { let (_root, plan) = local_host_plan("accounted_usage_result"); @@ -6125,10 +6208,6 @@ for line in sys.stdin: assert!(usage.models_exhaustive); assert_eq!(usage.models[0].model, "gpt-5.4"); assert_eq!(usage.observed.input_tokens, Some(100)); - assert_eq!(usage.cost.estimated.amount_nano_usd, None); - assert_eq!(usage.cost.estimated.observed_nano_usd, None); - assert!(usage.cost.pricing.is_none()); - assert!(usage.cost.applied_rates.is_empty()); } #[test] diff --git a/crates/fabric-core/src/usage_normalization.rs b/crates/fabric-core/src/usage_normalization.rs index f236d7623..9ff9fb638 100644 --- a/crates/fabric-core/src/usage_normalization.rs +++ b/crates/fabric-core/src/usage_normalization.rs @@ -588,6 +588,10 @@ fn normalize_request(raw: &ReportedUsage, observation: bool) -> [Count; 8] { counts } +pub(crate) fn normalized_request(raw: &ReportedUsage, observation: bool) -> ModelMeasurements { + measurements(normalize_request(raw, observation)) +} + fn add(left: Count, right: Count) -> Count { left.and_then(|left| { right.and_then(|right| left.checked_add(right).ok_or("token_count_out_of_range")) @@ -687,7 +691,7 @@ fn measurements(counts: [Count; 8]) -> ModelMeasurements { unit: MeasurementUnit::NanoUsd, source: MeasurementSource::FabricPriceCatalog, attribution: Attribution::None, - reason: Some("catalog_pricing_unavailable".into()), + reason: Some("price_catalog_not_configured".into()), }, }, } diff --git a/crates/fabric-core/tests/model_usage.rs b/crates/fabric-core/tests/model_usage.rs new file mode 100644 index 000000000..21e8396df --- /dev/null +++ b/crates/fabric-core/tests/model_usage.rs @@ -0,0 +1,521 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Reduced public accounting and honest partial pricing. + +use nemo_fabric_core::AgentUsage; +use nemo_fabric_core::model_usage::normalize_model_usage; +use nemo_fabric_core::price_catalog::UserPriceCatalog; +use serde_json::{Value, json}; + +fn normalize(base: Value, accounting: Value) -> Value { + let mut usage: AgentUsage = serde_json::from_value(base).unwrap(); + usage + .extensions + .insert("nemo_fabric.usage_accounting.v1".into(), accounting); + serde_json::to_value(normalize_model_usage(&usage, None, false).unwrap().unwrap()).unwrap() +} + +#[test] +fn shared_source_vectors_keep_the_reduced_complete_and_observed_contract() { + let cases: Vec = serde_json::from_str(include_str!( + "../../../tests/fixtures/usage_accounting/conformance.json" + )) + .unwrap(); + let dimensions = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ]; + for case in cases { + let result = normalize(case["usage"].clone(), case["accounting"].clone()); + for (field, expected) in case["expected"].as_object().unwrap() { + if dimensions.contains(&field.as_str()) { + assert_eq!( + result[field], expected["complete"], + "{} {field}", + case["name"] + ); + assert_eq!( + result["observed"][field], expected["observed"], + "{} {field}", + case["name"] + ); + } + } + assert!(result.get("accounting").is_none()); + assert!(result.get("reasoning_tokens").is_none()); + assert!(result.get("peak_request_input_tokens").is_none()); + assert!(result["input_tokens"].is_number() || result["input_tokens"].is_null()); + } +} + +#[test] +fn legacy_usage_does_not_gain_a_completeness_guarantee() { + let usage: AgentUsage = serde_json::from_value(json!({"input_tokens": 100})).unwrap(); + assert!( + normalize_model_usage(&usage, None, false) + .unwrap() + .is_none() + ); +} + +#[test] +fn normalized_total_does_not_replace_the_independent_raw_provider_total() { + let mut usage: AgentUsage = serde_json::from_value(json!({ + "input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "total_tokens": 999, + "extensions": {"nemo_fabric.detailed_token_usage.v1": {"output_tokens": 5, "output_tokens_include_reasoning": true}} + })).unwrap(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": false, "gaps": []}), + ); + let normalized = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(normalized.tokens.total_tokens, Some(105)); + assert_eq!(usage.total_tokens, Some(999)); +} + +fn catalog(provider: &str) -> UserPriceCatalog { + catalog_with_cache_rates(provider, 0.1, 1.25) +} + +fn catalog_with_cache_rates(provider: &str, read: f64, write: f64) -> UserPriceCatalog { + UserPriceCatalog::from_bytes("test-catalog.json".into(), &serde_json::to_vec(&json!({ + "version": 1, "entries": [{"provider": provider, "model_id": "model", "currency": "USD", "unit": "per_token", + "rates": {"input_per_million": 1.0, "output_per_million": 2.0, "cache_read_per_million": read, "cache_write_per_million": write}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"}, + "pricing_as_of": "2026-10-21", "pricing_source": "test fixture"}] + })).unwrap()).unwrap() +} + +fn priced(provider: &str, counts: Value) -> nemo_fabric_core::model_usage::ModelUsage { + let mut usage = AgentUsage::default(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "provider": provider, "reported": counts, "gaps": []}] + }), + ); + normalize_model_usage(&usage, Some(&catalog(provider)), false) + .unwrap() + .unwrap() +} + +#[test] +fn nonzero_anthropic_writes_without_tariff_keep_tokens_and_only_observed_cost() { + let result = priced( + "anthropic", + json!({ + "input_tokens": 100, "cached_input_tokens": 20, "cache_write_input_tokens": 10, + "input_tokens_include_cache": true, "output_tokens": 5, "output_tokens_include_reasoning": true + }), + ); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + assert_eq!(result.cost.estimated.observed_nano_usd, Some(82_000)); + assert!( + result.models[0] + .gaps + .iter() + .any(|gap| gap.code.as_deref() == Some("cache_write_tariff_unverified")) + ); +} + +#[test] +fn zero_anthropic_writes_need_no_expiry_evidence() { + let result = priced( + "anthropic", + json!({ + "input_tokens": 100, "cached_input_tokens": 20, "cache_write_input_tokens": 0, + "input_tokens_include_cache": true, "output_tokens": 5, "output_tokens_include_reasoning": true + }), + ); + assert_eq!(result.cost.estimated.amount_nano_usd, Some(92_000)); + assert_eq!(result.cost.estimated.observed_nano_usd, Some(92_000)); +} + +#[test] +fn output_only_observations_are_priceable_without_claiming_complete_spend() { + let result = priced( + "openai", + json!({"output_tokens": 20, "output_tokens_include_reasoning": true}), + ); + assert_eq!(result.tokens.output_tokens, Some(20)); + assert_eq!(result.tokens.input_tokens, None); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + assert_eq!(result.cost.estimated.observed_nano_usd, Some(40_000)); +} + +#[test] +fn overflow_does_not_erase_independent_valid_counts() { + let result = normalize( + json!({}), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "x", "gaps": [], "reported": {"input_tokens": u64::MAX, "input_tokens_include_cache": true, + "output_tokens": 1, "output_tokens_include_reasoning": true}}]}), + ); + assert_eq!(result["input_tokens"], json!(u64::MAX)); + assert_eq!(result["output_tokens"], 1); + assert!(result["total_tokens"].is_null()); +} + +#[test] +fn equal_catalog_rates_do_not_prove_an_unknown_anthropic_write_tariff() { + let mut usage = AgentUsage::default(); + usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ + "models_exhaustive": true, "gaps": [], "models": [{"model": "model", "provider": "anthropic", "gaps": [], + "reported": {"input_tokens": 100, "cached_input_tokens": 20, "cache_write_input_tokens": 10, + "input_tokens_include_cache": true, "output_tokens": 5, "output_tokens_include_reasoning": true}}]})); + let result = normalize_model_usage( + &usage, + Some(&catalog_with_cache_rates("anthropic", 1.0, 1.0)), + false, + ) + .unwrap() + .unwrap(); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + // Uncached input 70 + reads 20 + output 5*2; writes remain unpriced. + assert_eq!(result.cost.estimated.observed_nano_usd, Some(100_000)); +} + +#[test] +fn explicit_provider_never_uses_another_providers_catalog_rate() { + let mut usage = AgentUsage::default(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "provider": "custom-route", "gaps": [], + "reported": {"input_tokens": 100, "cached_input_tokens": 0, + "input_tokens_include_cache": true, "output_tokens": 5, + "output_tokens_include_reasoning": true} + }]}), + ); + let result = normalize_model_usage(&usage, Some(&catalog("openai")), false) + .unwrap() + .unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.estimated.amount_nano_usd, None); + assert_eq!(result.cost.estimated.observed_nano_usd, None); + assert!( + result.models[0] + .gaps + .iter() + .any(|gap| gap.code.as_deref() == Some("model_not_in_catalog")) + ); +} + +#[test] +fn unverified_route_cannot_use_native_prices_for_a_bundled_model() { + let mut usage = AgentUsage::default(); + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({"models_exhaustive": true, "gaps": [], "models": [{ + "model": "gpt-5.4", "gaps": [], + "reported": {"input_tokens": 100, "cached_input_tokens": 0, + "input_tokens_include_cache": true, "output_tokens": 5, + "output_tokens_include_reasoning": true} + }]}), + ); + // Global permission for bundled prices cannot establish an individual route. + let custom = normalize_model_usage(&usage, None, true).unwrap().unwrap(); + assert_eq!(custom.tokens.input_tokens, Some(100)); + assert_eq!(custom.tokens.total_tokens, Some(105)); + assert_eq!(custom.cost.estimated.amount_nano_usd, None); + assert_eq!(custom.cost.estimated.observed_nano_usd, None); + assert!( + custom.models[0] + .gaps + .iter() + .any(|gap| gap.code.as_deref() == Some("provider_price_unverified")) + ); + + // The same bundled model is priceable when the source establishes OpenAI. + usage + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]["provider"] = json!("openai"); + let native = normalize_model_usage(&usage, None, true).unwrap().unwrap(); + assert!(native.cost.estimated.amount_nano_usd.is_some()); + assert_eq!(native.tokens.total_tokens, Some(105)); +} + +fn cost_with_requests(aggregate: f64, requests: &[Option], exhaustive: bool) -> AgentUsage { + let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true, "cost_usd": aggregate}); + let mut usage = AgentUsage { + input_tokens: Some(100), + input_tokens_include_cache: Some(true), + output_tokens: Some(5), + cost_usd: Some(aggregate), + ..Default::default() + }; + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); + usage.extensions.insert("nemo_fabric.usage_accounting.v1".into(), json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "provider": "test", "model": "test-model", "gaps": [], "reported": counts, + "requests_exhaustive": exhaustive, + "requests": requests.iter().map(|cost| json!({"usage": {"cost_usd": cost}})).collect::>() + }] + })); + usage +} + +#[test] +fn contradictory_request_reported_costs_are_rejected() { + for (requests, exhaustive) in [ + (vec![Some(0.2)], true), + (vec![Some(0.2)], false), + (vec![Some(0.04)], true), + (vec![Some(10_000_000_000.0); 2], true), + ] { + let usage = cost_with_requests(0.1, &requests, exhaustive); + assert!( + normalize_model_usage(&usage, None, false).is_err(), + "contradictory cost must not become complete: {requests:?}, exhaustive={exhaustive}" + ); + } +} + +#[test] +fn independently_rounded_request_costs_keep_fractional_nanos_and_large_values() { + for (aggregate, requests) in [ + (1.6e-9, vec![Some(0.4e-9); 4]), + ( + 200_000_000.72337076, + vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], + ), + ] { + let usage = cost_with_requests(aggregate, &requests, true); + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert!(result.cost.reported.amount_nano_usd.is_some()); + } +} + +#[test] +fn partial_or_missing_request_costs_preserve_independent_complete_dimensions() { + for (requests, exhaustive) in [ + (vec![Some(0.04)], false), + (vec![Some(0.04), None], true), + (vec![None], true), + (vec![Some(0.0)], false), + ] { + let usage = cost_with_requests(0.1, &requests, exhaustive); + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.tokens.input_tokens, Some(100)); + assert_eq!(result.tokens.output_tokens, Some(5)); + assert_eq!(result.tokens.total_tokens, Some(105)); + assert_eq!(result.cost.reported.amount_nano_usd, Some(100_000_000)); + } +} + +fn omit_model_cost(usage: &mut AgentUsage) { + usage + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]["reported"] + .as_object_mut() + .unwrap() + .remove("cost_usd"); +} + +#[test] +fn request_costs_without_model_cost_cannot_certify_a_contradictory_root() { + let mut usage = cost_with_requests(0.1, &[Some(0.2)], true); + omit_model_cost(&mut usage); + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.cost.reported.amount_nano_usd, None); + assert_eq!(result.cost.reported.observed_nano_usd, None); + assert_eq!(result.tokens.total_tokens, Some(105)); +} + +#[test] +fn exhaustive_requests_recover_cost_without_losing_legitimate_root_rounding() { + for (aggregate, requests) in [ + (0.1, vec![Some(0.04), Some(0.06)]), + ( + 200_000_000.72337076, + vec![Some(100_000_000.65093447), Some(100_000_000.07243629)], + ), + ] { + let mut usage = cost_with_requests(aggregate, &requests, true); + omit_model_cost(&mut usage); + let with_root = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!( + with_root.cost.reported.amount_nano_usd, + Some((aggregate * 1e9).round() as u64) + ); + assert_eq!(with_root.tokens.total_tokens, Some(105)); + usage.cost_usd = None; + let recovered = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert!(recovered.cost.reported.amount_nano_usd.is_some()); + assert_eq!( + recovered.cost.reported.amount_nano_usd, + recovered.models[0].cost.reported.amount_nano_usd + ); + } +} + +#[test] +fn incomplete_requests_recover_only_observed_cost_when_no_complete_cost_exists() { + for (requests, exhaustive) in [(vec![Some(0.04)], false), (vec![Some(0.04), None], true)] { + let mut usage = cost_with_requests(0.1, &requests, exhaustive); + omit_model_cost(&mut usage); + usage.cost_usd = None; + let result = normalize_model_usage(&usage, None, false).unwrap().unwrap(); + assert_eq!(result.cost.reported.amount_nano_usd, None); + assert_eq!(result.cost.reported.observed_nano_usd, Some(40_000_000)); + assert_eq!(result.tokens.total_tokens, Some(105)); + } +} + +#[test] +fn broad_request_gap_cannot_certify_complete_model_or_invocation_counts() { + let counts = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true}); + for gap in ["unfinished_work", "missing_usage", "scope_unverified"] { + for root_complete in [false, true] { + let mut usage = AgentUsage::default(); + if root_complete { + usage.input_tokens = Some(100); + usage.input_tokens_include_cache = Some(true); + usage.output_tokens = Some(5); + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), counts.clone()); + } + usage.extensions.insert( + "nemo_fabric.usage_accounting.v1".into(), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "gaps": [], + "reported": if root_complete { json!({}) } else { counts.clone() }, + "requests_exhaustive": true, + "requests": [{"usage": {}, "gap": gap}] + }] + }), + ); + assert!( + normalize_model_usage(&usage, None, false).is_err(), + "broad request gap must conflict with complete counts: {gap}, root={root_complete}" + ); + } + } +} + +#[test] +fn broad_request_gap_keeps_observations_when_no_complete_value_is_claimed() { + let observed = json!({"input_tokens": 100, "input_tokens_include_cache": true, + "output_tokens": 5, "output_tokens_include_reasoning": true}); + let result = normalize( + json!({}), + json!({ + "models_exhaustive": false, "gaps": [], "models": [{ + "model": "model", "gaps": [], "observed": observed, + "observed_tokens_aligned": true, "requests_exhaustive": true, + "requests": [{"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}] + }] + }), + ); + assert!(result["input_tokens"].is_null()); + assert!(result["output_tokens"].is_null()); + assert_eq!(result["observed"]["input_tokens"], 100); + assert_eq!(result["observed"]["output_tokens"], 5); + for scope in [&result, &result["models"][0]] { + for dimension in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + "reported_cost", + ] { + assert!( + scope["gaps"].as_array().unwrap().iter().any(|gap| { + gap["dimensions"] == json!([dimension]) && gap["reason"] == "unfinished_work" + }), + "request reason missing for {dimension}: {}", + scope["gaps"] + ); + } + } + + let mixed = normalize( + json!({}), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "pending", "gaps": [{"dimensions": ["output_tokens"], "reason": "missing_usage"}], + "observed": observed, "observed_tokens_aligned": true, + "requests_exhaustive": true, + "requests": [ + {"usage": observed, "gap": "unfinished_work", "tokens_aligned": true}, + {"usage": {}, "gap": "scope_unverified"}, + {"usage": {}, "gap": "unfinished_work"} + ] + }, {"model": "settled", "reported": observed, "gaps": []}] + }), + ); + assert!(mixed["input_tokens"].is_null()); + assert_eq!(mixed["observed"]["input_tokens"], 200); + assert_eq!(mixed["observed"]["output_tokens"], 10); + assert_eq!(mixed["models"][1]["total_tokens"], 105); + assert!( + !mixed["models"][1]["gaps"] + .as_array() + .unwrap() + .iter() + .any(|gap| { + gap["reason"] == "unfinished_work" || gap["reason"] == "scope_unverified" + }) + ); + for scope in [&mixed, &mixed["models"][0]] { + let gaps = scope["gaps"].as_array().unwrap(); + for reason in ["unfinished_work", "scope_unverified"] { + assert_eq!( + gaps.iter() + .filter(|gap| { + gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == reason + }) + .count(), + 1, + "each request reason must survive once: {gaps:?}" + ); + } + } + assert!( + mixed["models"][0]["gaps"] + .as_array() + .unwrap() + .iter() + .any(|gap| { + gap["dimensions"] == json!(["output_tokens"]) && gap["reason"] == "missing_usage" + }) + ); +} + +#[test] +fn dimension_scoped_request_omission_keeps_independent_complete_output() { + let result = normalize( + json!({}), + json!({ + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "gaps": [{"dimensions": ["input_tokens"], "reason": "missing_usage"}], + "reported": {"output_tokens": 5, "output_tokens_include_reasoning": true}, + "requests_exhaustive": true, + "requests": [{"usage": {"output_tokens": 5, "output_tokens_include_reasoning": true}, "tokens_aligned": true}] + }] + }), + ); + assert!(result["input_tokens"].is_null()); + assert_eq!(result["output_tokens"], 5); + assert!(result["total_tokens"].is_null()); +} diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 42cd74b99..9778985f8 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -99,7 +99,7 @@ Collectors own source identity and invocation lifetime; the helper does not wait A request-wide `UsageRequest.gap` affects every dimension of that call and cannot coexist with complete model or invocation totals. For a dimension-specific gap, omit the affected request fields and declare the scoped gap without setting the request-wide `gap`. Other dimensions can remain complete. -A provider name must identify the source-established provider, not merely an API-compatible protocol. Custom endpoints and arbitrary gateways do not establish a native provider tariff. Preserve the model and counts when provider identity cannot be established. Catalog estimation is added separately and never changes reported usage. +A provider name must identify the source-established provider, not merely an API-compatible protocol. Custom endpoints and arbitrary gateways do not establish a native provider tariff. Preserve the model and counts when provider identity cannot be established; explicit user catalogs can price that evidence with labeled assumptions. Core normalizes cache-inclusive input and reasoning-inclusive output once and returns the flat `RunResult.model_usage` view. Raw reasoning/peak/request evidence diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index 6a624bb47..f6aa0ba97 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -45,6 +45,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`models.McpServerConfig`](./nemo_fabric.models.md#class-mcpserverconfig): MCP server configuration. - [`models.MetadataConfig`](./nemo_fabric.models.md#class-metadataconfig): Human-readable agent identity. - [`models.ModelConfig`](./nemo_fabric.models.md#class-modelconfig): Configuration for one model role. +- [`models.PricingConfig`](./nemo_fabric.models.md#class-pricingconfig): Frozen Relay-compatible price-catalog override, read during planning. - [`models.RelayAtifConfig`](./nemo_fabric.models.md#class-relayatifconfig): NeMo Relay ATIF export configuration. - [`models.RelayAtofConfig`](./nemo_fabric.models.md#class-relayatofconfig): NeMo Relay ATOF export configuration. - [`models.RelayAtofFileSinkConfig`](./nemo_fabric.models.md#class-relayatoffilesinkconfig): NeMo Relay ATOF file sink configuration. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index 3aa659030..03e7e0335 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -16,7 +16,6 @@ The Rust core remains the source of truth for persisted schema snapshots. These --- - ## class `FabricBaseModel` Base class for SDK-facing Pydantic models. @@ -54,7 +53,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -65,7 +63,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -77,7 +74,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `MetadataConfig` Human-readable agent identity. @@ -125,7 +121,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -136,7 +131,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -148,7 +142,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `HarnessConfig` Harness adapter selection plus adapter-owned settings. @@ -197,7 +190,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -208,7 +200,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -220,7 +211,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `WorkflowConfig` Registered workflow target and immutable construction settings. @@ -268,7 +258,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -279,7 +268,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -291,7 +279,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `DiscoveryConfig` Explicit local descriptor discovery paths. @@ -338,7 +325,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -349,7 +335,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -361,7 +346,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `InstructionConfig` One portable instruction value. @@ -409,7 +393,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -420,7 +403,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -432,7 +414,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `InstructionsConfig` Harness-neutral agent instructions. @@ -479,7 +460,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -490,7 +470,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -502,7 +481,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RuntimeConfig` Invocation runtime contract. @@ -553,7 +531,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -564,7 +541,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -576,7 +552,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `EnvironmentConfig` Execution environment configuration supplied by the consumer. @@ -634,7 +609,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -645,7 +619,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -657,7 +630,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `ModelConfig` Configuration for one model role. @@ -711,7 +683,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -722,7 +693,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -734,7 +704,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `SkillConfig` Skill capability configuration. @@ -781,7 +750,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_path` ```python @@ -792,7 +760,6 @@ Add a skill path if absent. --- - ### classmethod `from_mapping` ```python @@ -803,7 +770,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_path` ```python @@ -814,7 +780,6 @@ Remove a skill path if present. --- - ### method `to_mapping` ```python @@ -826,7 +791,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `McpAuthenticationConfig` MCP server authentication configuration. @@ -883,7 +847,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -894,7 +857,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -906,7 +868,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `McpServerConfig` MCP server configuration. @@ -961,7 +922,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -972,7 +932,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -984,7 +943,6 @@ Return the server mapping without collapsing an explicit empty allowlist. --- - ## class `McpConfig` MCP capability configuration. @@ -1031,7 +989,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_server` ```python @@ -1057,7 +1014,6 @@ For stdio, set ``url`` to the executable and pass each command-line argument as --- - ### classmethod `from_mapping` ```python @@ -1068,7 +1024,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_server` ```python @@ -1079,7 +1034,6 @@ Remove a named MCP server if present. --- - ### method `to_mapping` ```python @@ -1091,7 +1045,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayConfigPolicy` NVIDIA NeMo Relay config validation policy. @@ -1140,7 +1093,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1151,7 +1103,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1163,7 +1114,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtofFileSinkConfig` NeMo Relay ATOF file sink configuration. @@ -1213,7 +1163,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1224,7 +1173,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1236,7 +1184,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtofStreamSinkConfig` NeMo Relay ATOF stream sink configuration. @@ -1290,7 +1237,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1301,7 +1247,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1313,7 +1258,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtofConfig` NeMo Relay ATOF export configuration. @@ -1361,7 +1305,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1372,7 +1315,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1384,7 +1326,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayS3StorageConfig` NeMo Relay ATIF S3 storage configuration. @@ -1439,7 +1380,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1450,7 +1390,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1462,7 +1401,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayHttpStorageConfig` NeMo Relay ATIF HTTP storage configuration. @@ -1513,7 +1451,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1524,7 +1461,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1536,7 +1472,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayAtifConfig` NeMo Relay ATIF export configuration. @@ -1591,7 +1526,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1602,7 +1536,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1614,7 +1547,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayOpenTelemetryEndpointConfig` One typed NeMo Relay OpenTelemetry destination. @@ -1674,7 +1606,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1685,7 +1616,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1697,7 +1627,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayOpenTelemetryConfig` NeMo Relay typed OpenTelemetry destination configuration. @@ -1745,7 +1674,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1756,7 +1684,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1768,7 +1695,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayObservabilityConfig` NeMo Relay observability component configuration. @@ -1820,7 +1746,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1831,7 +1756,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1843,7 +1767,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayComponentConfig` Generic NeMo Relay plugin component configuration. @@ -1892,7 +1815,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1903,7 +1825,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1915,7 +1836,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `RelayConfig` First-class NeMo Relay integration configuration. @@ -1966,7 +1886,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -1977,7 +1896,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -1989,7 +1907,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `TelemetryProviderConfig` Provider-specific telemetry configuration. @@ -2036,7 +1953,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2047,7 +1963,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2059,7 +1974,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `TelemetryConfig` Telemetry configuration. @@ -2106,7 +2020,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `enable_native` ```python @@ -2117,7 +2030,6 @@ Let the selected adapter handle telemetry natively. --- - ### method `enable_relay` ```python @@ -2128,7 +2040,6 @@ Enable NeMo Relay telemetry for subsequently started runtimes. --- - ### classmethod `from_mapping` ```python @@ -2139,7 +2050,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_provider` ```python @@ -2150,7 +2060,6 @@ Remove a configured telemetry provider. --- - ### method `to_mapping` ```python @@ -2162,7 +2071,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `ToolDefinitionConfig` One named normalized tool or tool-group definition. @@ -2211,7 +2119,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2222,7 +2129,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2234,7 +2140,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `ToolsConfig` Harness-neutral tool capability configuration. @@ -2283,7 +2188,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_definition` ```python @@ -2301,7 +2205,6 @@ Add or replace one named definition and return this tools config. --- - ### classmethod `from_mapping` ```python @@ -2312,7 +2215,6 @@ Validate a mapping using this Pydantic model. --- - ### method `remove_definition` ```python @@ -2323,7 +2225,6 @@ Remove one named definition and return this tools config. --- - ### method `to_mapping` ```python @@ -2335,7 +2236,6 @@ Return the tool mapping without collapsing an explicit empty policy. --- - ## class `SystemStatsSamplingConfig` Opt-in periodic collection; bounds include both endpoint samples. @@ -2383,7 +2283,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2394,7 +2293,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2406,7 +2304,6 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- - ## class `SystemStatsConfig` Native invocation stats selection, independent of Relay and recording. @@ -2456,7 +2353,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2467,7 +2363,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python @@ -2479,6 +2374,72 @@ Return a detached JSON-compatible mapping for Rust/core calls. --- +## class `PricingConfig` + +Frozen Relay-compatible price-catalog override, read during planning. + + + +### Fields + +The model defines the following fields: + +| Field | Type | Required | Default | Constraints | Description | +| --- | --- | --- | --- | --- | --- | +| `catalog_path` | `str \| Path \| None` | No | `None` | — | — | + +--- + +### property extra_fields + +Return fields preserved by the extension point for this model. + +--- + +### property model_extra + +Get extra fields set during validation. + + + +**Returns:** + A dictionary of extra fields, or `None` if `config.extra` is not set to `"allow"`. + +--- + +### property model_fields_set + +Returns the set of fields that have been explicitly set on this model instance. + + + +**Returns:** + A set of strings representing the fields that have been set, i.e. that were not filled from defaults. + + + +--- + +### classmethod `from_mapping` + +```python +def from_mapping(value: Mapping[str, Any]) -> Self +``` + +Validate a mapping using this Pydantic model. + +--- + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached JSON-compatible mapping for Rust/core calls. + + +--- ## class `FabricConfig` @@ -2508,6 +2469,7 @@ The model defines the following fields: | `system_stats` | `SystemStatsConfig \| None` | No | `SystemStatsConfig()` | — | — | | `telemetry` | `TelemetryConfig \| None` | No | `None` | — | — | | `relay` | `RelayConfig \| None` | No | `None` | — | — | +| `pricing` | `PricingConfig \| None` | No | `None` | — | — | | `tools` | `ToolsConfig \| None` | No | `None` | — | — | --- @@ -2542,7 +2504,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### method `add_mcp_server` ```python @@ -2568,7 +2529,6 @@ For stdio, set ``url`` to the executable and pass each command-line argument as --- - ### method `add_skill_path` ```python @@ -2579,7 +2539,6 @@ Add a skill path and return this config. --- - ### method `add_tool_definition` ```python @@ -2597,7 +2556,6 @@ Add or replace one named tool definition and return this config. --- - ### method `block_tools` ```python @@ -2608,7 +2566,6 @@ Block adapter-native tool names and return this config. --- - ### method `enable_relay` ```python @@ -2626,7 +2583,6 @@ Enable NeMo Relay telemetry and return this config. --- - ### classmethod `from_mapping` ```python @@ -2637,7 +2593,6 @@ Validate the public agent config mapping shape. --- - ### method `remove_mcp_server` ```python @@ -2648,7 +2603,6 @@ Remove a named MCP server and return this config. --- - ### method `remove_skill_path` ```python @@ -2659,7 +2613,6 @@ Remove a skill path and return this config. --- - ### method `remove_tool_definition` ```python @@ -2670,7 +2623,6 @@ Remove one named tool definition and return this config. --- - ### method `to_mapping` ```python @@ -2682,7 +2634,6 @@ Return a detached mapping matching the Rust ``FabricConfig`` schema. --- - ## class `RunRequest` One validated NeMo Fabric invocation request. @@ -2733,7 +2684,6 @@ Returns the set of fields that have been explicitly set on this model instance. --- - ### classmethod `from_mapping` ```python @@ -2744,7 +2694,6 @@ Validate a mapping using this Pydantic model. --- - ### method `to_mapping` ```python diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 8a789e763..12f68b13e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 142 +position: 143 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 359d1c117..40d3afc67 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 143 +position: 144 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 6bbdf7d45..8d39f22be 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 144 +position: 145 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx index 40e387ace..9690f011e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Kind" sidebar-title: "CapabilityKind" description: "Capability kind." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,23 +43,23 @@ MCP server. ### `impl Clone for CapabilityKind` -
Clone for CapabilityKind"}} />
+
Clone for CapabilityKind"}} />
#### `clone` -
clone(&self) -> CapabilityKind"}} />
+
clone(&self) -> CapabilityKind"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for CapabilityKind` -
Debug for CapabilityKind"}} />
+
Debug for CapabilityKind"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for CapabilityKind` @@ -67,7 +67,7 @@ MCP server. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for CapabilityKind` @@ -75,11 +75,11 @@ MCP server. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ MCP server. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for CapabilityKind` -
PartialEq for CapabilityKind"}} />
+
PartialEq for CapabilityKind"}} />
#### `eq` -
eq(&self, other: &CapabilityKind) -> bool"}} />
+
eq(&self, other: &CapabilityKind) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for CapabilityKind` @@ -107,16 +107,16 @@ MCP server. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for CapabilityKind` -
Copy for CapabilityKind"}} />
+
Copy for CapabilityKind"}} />
### `impl Eq for CapabilityKind` -
Eq for CapabilityKind"}} />
+
Eq for CapabilityKind"}} />
### `impl StructuralPartialEq for CapabilityKind` -
StructuralPartialEq for CapabilityKind"}} />
+
StructuralPartialEq for CapabilityKind"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx index e121e6b73..b0c744403 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Target" sidebar-title: "CapabilityTarget" description: "Component responsible for executing a configured capability." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -45,23 +45,23 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. ### `impl Clone for CapabilityTarget` -
Clone for CapabilityTarget"}} />
+
Clone for CapabilityTarget"}} />
#### `clone` -
clone(&self) -> CapabilityTarget"}} />
+
clone(&self) -> CapabilityTarget"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for CapabilityTarget` -
Debug for CapabilityTarget"}} />
+
Debug for CapabilityTarget"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for CapabilityTarget` @@ -69,7 +69,7 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for CapabilityTarget` @@ -77,11 +77,11 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -89,19 +89,19 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for CapabilityTarget` -
PartialEq for CapabilityTarget"}} />
+
PartialEq for CapabilityTarget"}} />
#### `eq` -
eq(&self, other: &CapabilityTarget) -> bool"}} />
+
eq(&self, other: &CapabilityTarget) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for CapabilityTarget` @@ -109,16 +109,16 @@ Neither the adapter nor NeMo Fabric can execute the configured capability. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for CapabilityTarget` -
Copy for CapabilityTarget"}} />
+
Copy for CapabilityTarget"}} />
### `impl Eq for CapabilityTarget` -
Eq for CapabilityTarget"}} />
+
Eq for CapabilityTarget"}} />
### `impl StructuralPartialEq for CapabilityTarget` -
StructuralPartialEq for CapabilityTarget"}} />
+
StructuralPartialEq for CapabilityTarget"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx index 74dc754f8..ad66e7ecf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx @@ -2,14 +2,14 @@ title: "Enum Relay Atif Storage Config" sidebar-title: "RelayAtifStorageConfig" description: "Relay ATIF remote storage configuration." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        extensions: BTreeMap<String, Value>,\n    },\n    S3 {\n        bucket: String,\n        key_prefix: Option<String>,\n        access_key_id: Option<String>,\n        secret_access_key_var: Option<String>,\n        session_token_var: Option<String>,\n        region: Option<String>,\n        endpoint_url: Option<String>,\n        allow_http: Option<bool>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
+
String,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        extensions: BTreeMap<String, Value>,\n    },\n    S3 {\n        bucket: String,\n        key_prefix: Option<String>,\n        access_key_id: Option<String>,\n        secret_access_key_var: Option<String>,\n        session_token_var: Option<String>,\n        region: Option<String>,\n        endpoint_url: Option<String>,\n        allow_http: Option<bool>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
Relay ATIF remote storage configuration. @@ -91,23 +91,23 @@ Additive S3 storage fields. ### `impl Clone for RelayAtifStorageConfig` -
Clone for RelayAtifStorageConfig"}} />
+
Clone for RelayAtifStorageConfig"}} />
#### `clone` -
clone(&self) -> RelayAtifStorageConfig"}} />
+
clone(&self) -> RelayAtifStorageConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtifStorageConfig` -
Debug for RelayAtifStorageConfig"}} />
+
Debug for RelayAtifStorageConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayAtifStorageConfig` @@ -115,7 +115,7 @@ Additive S3 storage fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtifStorageConfig` @@ -123,11 +123,11 @@ Additive S3 storage fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -135,19 +135,19 @@ Additive S3 storage fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtifStorageConfig` -
PartialEq for RelayAtifStorageConfig"}} />
+
PartialEq for RelayAtifStorageConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtifStorageConfig) -> bool"}} />
+
eq(&self, other: &RelayAtifStorageConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtifStorageConfig` @@ -155,8 +155,8 @@ Additive S3 storage fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtifStorageConfig` -
StructuralPartialEq for RelayAtifStorageConfig"}} />
+
StructuralPartialEq for RelayAtifStorageConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx index bbf8ff9e5..7f00685ce 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Mode" sidebar-title: "RelayAtofMode" description: "Relay ATOF file mode." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -36,31 +36,31 @@ Overwrite an existing ATOF file. ### `impl Clone for RelayAtofMode` -
Clone for RelayAtofMode"}} />
+
Clone for RelayAtofMode"}} />
#### `clone` -
clone(&self) -> RelayAtofMode"}} />
+
clone(&self) -> RelayAtofMode"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofMode` -
Debug for RelayAtofMode"}} />
+
Debug for RelayAtofMode"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofMode` -
Default for RelayAtofMode"}} />
+
Default for RelayAtofMode"}} />
#### `default` -
default() -> RelayAtofMode"}} />
+
default() -> RelayAtofMode"}} />
### `impl<'de> Deserialize<'de> for RelayAtofMode` @@ -68,7 +68,7 @@ Overwrite an existing ATOF file. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofMode` @@ -76,11 +76,11 @@ Overwrite an existing ATOF file. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -88,19 +88,19 @@ Overwrite an existing ATOF file. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofMode` -
PartialEq for RelayAtofMode"}} />
+
PartialEq for RelayAtofMode"}} />
#### `eq` -
eq(&self, other: &RelayAtofMode) -> bool"}} />
+
eq(&self, other: &RelayAtofMode) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofMode` @@ -108,16 +108,16 @@ Overwrite an existing ATOF file. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayAtofMode` -
Copy for RelayAtofMode"}} />
+
Copy for RelayAtofMode"}} />
### `impl Eq for RelayAtofMode` -
Eq for RelayAtofMode"}} />
+
Eq for RelayAtofMode"}} />
### `impl StructuralPartialEq for RelayAtofMode` -
StructuralPartialEq for RelayAtofMode"}} />
+
StructuralPartialEq for RelayAtofMode"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx index d53f00f8d..f71cbc8ff 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx @@ -2,14 +2,14 @@ title: "Enum Relay Atof Sink Config" sidebar-title: "RelayAtofSinkConfig" description: "Relay ATOF sink configuration." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Option<PathBuf>,\n        filename: Option<String>,\n        mode: RelayAtofMode,\n        extensions: BTreeMap<String, Value>,\n    },\n    Stream {\n        url: String,\n        transport: RelayAtofStreamTransport,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        field_name_policy: RelayAtofStreamFieldNamePolicy,\n        name: Option<String>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
+
Option<PathBuf>,\n        filename: Option<String>,\n        mode: RelayAtofMode,\n        extensions: BTreeMap<String, Value>,\n    },\n    Stream {\n        url: String,\n        transport: RelayAtofStreamTransport,\n        headers: BTreeMap<String, String>,\n        header_env: BTreeMap<String, String>,\n        timeout_millis: u64,\n        field_name_policy: RelayAtofStreamFieldNamePolicy,\n        name: Option<String>,\n        extensions: BTreeMap<String, Value>,\n    },\n}"}} />
Relay ATOF sink configuration. @@ -83,23 +83,23 @@ Additive stream sink fields. ### `impl Clone for RelayAtofSinkConfig` -
Clone for RelayAtofSinkConfig"}} />
+
Clone for RelayAtofSinkConfig"}} />
#### `clone` -
clone(&self) -> RelayAtofSinkConfig"}} />
+
clone(&self) -> RelayAtofSinkConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofSinkConfig` -
Debug for RelayAtofSinkConfig"}} />
+
Debug for RelayAtofSinkConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayAtofSinkConfig` @@ -107,7 +107,7 @@ Additive stream sink fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofSinkConfig` @@ -115,11 +115,11 @@ Additive stream sink fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -127,19 +127,19 @@ Additive stream sink fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofSinkConfig` -
PartialEq for RelayAtofSinkConfig"}} />
+
PartialEq for RelayAtofSinkConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtofSinkConfig) -> bool"}} />
+
eq(&self, other: &RelayAtofSinkConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofSinkConfig` @@ -147,8 +147,8 @@ Additive stream sink fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtofSinkConfig` -
StructuralPartialEq for RelayAtofSinkConfig"}} />
+
StructuralPartialEq for RelayAtofSinkConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx index 892413a9e..d01f8c91c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Field Name Policy" sidebar-title: "RelayAtofStreamFieldNamePolicy" description: "Relay ATOF stream field-name policy." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -36,31 +36,31 @@ Replace dots in field names. ### `impl Clone for RelayAtofStreamFieldNamePolicy` -
Clone for RelayAtofStreamFieldNamePolicy"}} />
+
Clone for RelayAtofStreamFieldNamePolicy"}} />
#### `clone` -
clone(&self) -> RelayAtofStreamFieldNamePolicy"}} />
+
clone(&self) -> RelayAtofStreamFieldNamePolicy"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofStreamFieldNamePolicy` -
Debug for RelayAtofStreamFieldNamePolicy"}} />
+
Debug for RelayAtofStreamFieldNamePolicy"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofStreamFieldNamePolicy` -
Default for RelayAtofStreamFieldNamePolicy"}} />
+
Default for RelayAtofStreamFieldNamePolicy"}} />
#### `default` -
default() -> RelayAtofStreamFieldNamePolicy"}} />
+
default() -> RelayAtofStreamFieldNamePolicy"}} />
### `impl<'de> Deserialize<'de> for RelayAtofStreamFieldNamePolicy` @@ -68,7 +68,7 @@ Replace dots in field names. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofStreamFieldNamePolicy` @@ -76,11 +76,11 @@ Replace dots in field names. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -88,19 +88,19 @@ Replace dots in field names. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofStreamFieldNamePolicy` -
PartialEq for RelayAtofStreamFieldNamePolicy"}} />
+
PartialEq for RelayAtofStreamFieldNamePolicy"}} />
#### `eq` -
eq(&self, other: &RelayAtofStreamFieldNamePolicy) -> bool"}} />
+
eq(&self, other: &RelayAtofStreamFieldNamePolicy) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofStreamFieldNamePolicy` @@ -108,16 +108,16 @@ Replace dots in field names. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayAtofStreamFieldNamePolicy` -
Copy for RelayAtofStreamFieldNamePolicy"}} />
+
Copy for RelayAtofStreamFieldNamePolicy"}} />
### `impl Eq for RelayAtofStreamFieldNamePolicy` -
Eq for RelayAtofStreamFieldNamePolicy"}} />
+
Eq for RelayAtofStreamFieldNamePolicy"}} />
### `impl StructuralPartialEq for RelayAtofStreamFieldNamePolicy` -
StructuralPartialEq for RelayAtofStreamFieldNamePolicy"}} />
+
StructuralPartialEq for RelayAtofStreamFieldNamePolicy"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx index ea48753dd..1f47f4384 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Transport" sidebar-title: "RelayAtofStreamTransport" description: "Relay ATOF stream transport." -position: 80 +position: 81 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,31 +43,31 @@ NDJSON transport. ### `impl Clone for RelayAtofStreamTransport` -
Clone for RelayAtofStreamTransport"}} />
+
Clone for RelayAtofStreamTransport"}} />
#### `clone` -
clone(&self) -> RelayAtofStreamTransport"}} />
+
clone(&self) -> RelayAtofStreamTransport"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofStreamTransport` -
Debug for RelayAtofStreamTransport"}} />
+
Debug for RelayAtofStreamTransport"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofStreamTransport` -
Default for RelayAtofStreamTransport"}} />
+
Default for RelayAtofStreamTransport"}} />
#### `default` -
default() -> RelayAtofStreamTransport"}} />
+
default() -> RelayAtofStreamTransport"}} />
### `impl<'de> Deserialize<'de> for RelayAtofStreamTransport` @@ -75,7 +75,7 @@ NDJSON transport. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofStreamTransport` @@ -83,11 +83,11 @@ NDJSON transport. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ NDJSON transport. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofStreamTransport` -
PartialEq for RelayAtofStreamTransport"}} />
+
PartialEq for RelayAtofStreamTransport"}} />
#### `eq` -
eq(&self, other: &RelayAtofStreamTransport) -> bool"}} />
+
eq(&self, other: &RelayAtofStreamTransport) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofStreamTransport` @@ -115,16 +115,16 @@ NDJSON transport. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayAtofStreamTransport` -
Copy for RelayAtofStreamTransport"}} />
+
Copy for RelayAtofStreamTransport"}} />
### `impl Eq for RelayAtofStreamTransport` -
Eq for RelayAtofStreamTransport"}} />
+
Eq for RelayAtofStreamTransport"}} />
### `impl StructuralPartialEq for RelayAtofStreamTransport` -
StructuralPartialEq for RelayAtofStreamTransport"}} />
+
StructuralPartialEq for RelayAtofStreamTransport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx index beb3da6eb..8689a98ef 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Endpoint Type" sidebar-title: "RelayOpenTelemetryEndpointType" description: "Relay OpenTelemetry span projection." -position: 81 +position: 82 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,23 +43,23 @@ Emit OpenInference semantic-convention spans. ### `impl Clone for RelayOpenTelemetryEndpointType` -
Clone for RelayOpenTelemetryEndpointType"}} />
+
Clone for RelayOpenTelemetryEndpointType"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryEndpointType"}} />
+
clone(&self) -> RelayOpenTelemetryEndpointType"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryEndpointType` -
Debug for RelayOpenTelemetryEndpointType"}} />
+
Debug for RelayOpenTelemetryEndpointType"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryEndpointType` @@ -67,7 +67,7 @@ Emit OpenInference semantic-convention spans. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryEndpointType` @@ -75,11 +75,11 @@ Emit OpenInference semantic-convention spans. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Emit OpenInference semantic-convention spans. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryEndpointType` -
PartialEq for RelayOpenTelemetryEndpointType"}} />
+
PartialEq for RelayOpenTelemetryEndpointType"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryEndpointType) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryEndpointType) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryEndpointType` @@ -107,16 +107,16 @@ Emit OpenInference semantic-convention spans. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayOpenTelemetryEndpointType` -
Copy for RelayOpenTelemetryEndpointType"}} />
+
Copy for RelayOpenTelemetryEndpointType"}} />
### `impl Eq for RelayOpenTelemetryEndpointType` -
Eq for RelayOpenTelemetryEndpointType"}} />
+
Eq for RelayOpenTelemetryEndpointType"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryEndpointType` -
StructuralPartialEq for RelayOpenTelemetryEndpointType"}} />
+
StructuralPartialEq for RelayOpenTelemetryEndpointType"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx index da13057aa..4069e7d1b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Mark Projection" sidebar-title: "RelayOpenTelemetryMarkProjection" description: "Relay OpenTelemetry mark projection policy." -position: 82 +position: 83 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,31 +43,31 @@ Project marks as tool spans. ### `impl Clone for RelayOpenTelemetryMarkProjection` -
Clone for RelayOpenTelemetryMarkProjection"}} />
+
Clone for RelayOpenTelemetryMarkProjection"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryMarkProjection"}} />
+
clone(&self) -> RelayOpenTelemetryMarkProjection"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryMarkProjection` -
Debug for RelayOpenTelemetryMarkProjection"}} />
+
Debug for RelayOpenTelemetryMarkProjection"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayOpenTelemetryMarkProjection` -
Default for RelayOpenTelemetryMarkProjection"}} />
+
Default for RelayOpenTelemetryMarkProjection"}} />
#### `default` -
default() -> RelayOpenTelemetryMarkProjection"}} />
+
default() -> RelayOpenTelemetryMarkProjection"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryMarkProjection` @@ -75,7 +75,7 @@ Project marks as tool spans. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryMarkProjection` @@ -83,11 +83,11 @@ Project marks as tool spans. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ Project marks as tool spans. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryMarkProjection` -
PartialEq for RelayOpenTelemetryMarkProjection"}} />
+
PartialEq for RelayOpenTelemetryMarkProjection"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryMarkProjection) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryMarkProjection) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryMarkProjection` @@ -115,16 +115,16 @@ Project marks as tool spans. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayOpenTelemetryMarkProjection` -
Copy for RelayOpenTelemetryMarkProjection"}} />
+
Copy for RelayOpenTelemetryMarkProjection"}} />
### `impl Eq for RelayOpenTelemetryMarkProjection` -
Eq for RelayOpenTelemetryMarkProjection"}} />
+
Eq for RelayOpenTelemetryMarkProjection"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryMarkProjection` -
StructuralPartialEq for RelayOpenTelemetryMarkProjection"}} />
+
StructuralPartialEq for RelayOpenTelemetryMarkProjection"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx index 8db6e0872..287821dd1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Otlp Transport" sidebar-title: "RelayOtlpTransport" description: "Relay OTLP transport." -position: 83 +position: 84 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -36,31 +36,31 @@ OTLP gRPC transport. ### `impl Clone for RelayOtlpTransport` -
Clone for RelayOtlpTransport"}} />
+
Clone for RelayOtlpTransport"}} />
#### `clone` -
clone(&self) -> RelayOtlpTransport"}} />
+
clone(&self) -> RelayOtlpTransport"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOtlpTransport` -
Debug for RelayOtlpTransport"}} />
+
Debug for RelayOtlpTransport"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayOtlpTransport` -
Default for RelayOtlpTransport"}} />
+
Default for RelayOtlpTransport"}} />
#### `default` -
default() -> RelayOtlpTransport"}} />
+
default() -> RelayOtlpTransport"}} />
### `impl<'de> Deserialize<'de> for RelayOtlpTransport` @@ -68,7 +68,7 @@ OTLP gRPC transport. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOtlpTransport` @@ -76,11 +76,11 @@ OTLP gRPC transport. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -88,19 +88,19 @@ OTLP gRPC transport. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOtlpTransport` -
PartialEq for RelayOtlpTransport"}} />
+
PartialEq for RelayOtlpTransport"}} />
#### `eq` -
eq(&self, other: &RelayOtlpTransport) -> bool"}} />
+
eq(&self, other: &RelayOtlpTransport) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOtlpTransport` @@ -108,16 +108,16 @@ OTLP gRPC transport. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayOtlpTransport` -
Copy for RelayOtlpTransport"}} />
+
Copy for RelayOtlpTransport"}} />
### `impl Eq for RelayOtlpTransport` -
Eq for RelayOtlpTransport"}} />
+
Eq for RelayOtlpTransport"}} />
### `impl StructuralPartialEq for RelayOtlpTransport` -
StructuralPartialEq for RelayOtlpTransport"}} />
+
StructuralPartialEq for RelayOtlpTransport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx index 95ec26616..bd688cc9f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Unsupported Behavior" sidebar-title: "RelayUnsupportedBehavior" description: "Relay unsupported/unknown config handling." -position: 84 +position: 85 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,31 +43,31 @@ Error on the unsupported or unknown value. ### `impl Clone for RelayUnsupportedBehavior` -
Clone for RelayUnsupportedBehavior"}} />
+
Clone for RelayUnsupportedBehavior"}} />
#### `clone` -
clone(&self) -> RelayUnsupportedBehavior"}} />
+
clone(&self) -> RelayUnsupportedBehavior"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayUnsupportedBehavior` -
Debug for RelayUnsupportedBehavior"}} />
+
Debug for RelayUnsupportedBehavior"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayUnsupportedBehavior` -
Default for RelayUnsupportedBehavior"}} />
+
Default for RelayUnsupportedBehavior"}} />
#### `default` -
default() -> RelayUnsupportedBehavior"}} />
+
default() -> RelayUnsupportedBehavior"}} />
### `impl<'de> Deserialize<'de> for RelayUnsupportedBehavior` @@ -75,7 +75,7 @@ Error on the unsupported or unknown value. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayUnsupportedBehavior` @@ -83,11 +83,11 @@ Error on the unsupported or unknown value. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ Error on the unsupported or unknown value. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayUnsupportedBehavior` -
PartialEq for RelayUnsupportedBehavior"}} />
+
PartialEq for RelayUnsupportedBehavior"}} />
#### `eq` -
eq(&self, other: &RelayUnsupportedBehavior) -> bool"}} />
+
eq(&self, other: &RelayUnsupportedBehavior) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayUnsupportedBehavior` @@ -115,16 +115,16 @@ Error on the unsupported or unknown value. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for RelayUnsupportedBehavior` -
Copy for RelayUnsupportedBehavior"}} />
+
Copy for RelayUnsupportedBehavior"}} />
### `impl Eq for RelayUnsupportedBehavior` -
Eq for RelayUnsupportedBehavior"}} />
+
Eq for RelayUnsupportedBehavior"}} />
### `impl StructuralPartialEq for RelayUnsupportedBehavior` -
StructuralPartialEq for RelayUnsupportedBehavior"}} />
+
StructuralPartialEq for RelayUnsupportedBehavior"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx index f130d1764..ee84e47b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx @@ -2,7 +2,7 @@ title: "Enum Resolution Strategy" sidebar-title: "ResolutionStrategy" description: "Adapter install or availability strategy." -position: 54 +position: 55 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -71,23 +71,23 @@ Adapter is installed through a harness-native plugin manager. ### `impl Clone for ResolutionStrategy` -
Clone for ResolutionStrategy"}} />
+
Clone for ResolutionStrategy"}} />
#### `clone` -
clone(&self) -> ResolutionStrategy"}} />
+
clone(&self) -> ResolutionStrategy"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolutionStrategy` -
Debug for ResolutionStrategy"}} />
+
Debug for ResolutionStrategy"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ResolutionStrategy` @@ -95,7 +95,7 @@ Adapter is installed through a harness-native plugin manager. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ResolutionStrategy` @@ -103,11 +103,11 @@ Adapter is installed through a harness-native plugin manager. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -115,19 +115,19 @@ Adapter is installed through a harness-native plugin manager. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ResolutionStrategy` -
PartialEq for ResolutionStrategy"}} />
+
PartialEq for ResolutionStrategy"}} />
#### `eq` -
eq(&self, other: &ResolutionStrategy) -> bool"}} />
+
eq(&self, other: &ResolutionStrategy) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ResolutionStrategy` @@ -135,16 +135,16 @@ Adapter is installed through a harness-native plugin manager. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for ResolutionStrategy` -
Copy for ResolutionStrategy"}} />
+
Copy for ResolutionStrategy"}} />
### `impl Eq for ResolutionStrategy` -
Eq for ResolutionStrategy"}} />
+
Eq for ResolutionStrategy"}} />
### `impl StructuralPartialEq for ResolutionStrategy` -
StructuralPartialEq for ResolutionStrategy"}} />
+
StructuralPartialEq for ResolutionStrategy"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx index 8ce5c9908..17b80b5bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx @@ -2,7 +2,7 @@ title: "Enum Telemetry Provider" sidebar-title: "TelemetryProvider" description: "Telemetry runtime provider." -position: 64 +position: 65 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -40,7 +40,7 @@ Let the selected adapter handle telemetry natively. #### `as_str` -
str"}} />
+
str"}} />
Return the stable configuration value for this provider. @@ -48,31 +48,31 @@ Return the stable configuration value for this provider. ### `impl Clone for TelemetryProvider` -
Clone for TelemetryProvider"}} />
+
Clone for TelemetryProvider"}} />
#### `clone` -
clone(&self) -> TelemetryProvider"}} />
+
clone(&self) -> TelemetryProvider"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryProvider` -
Debug for TelemetryProvider"}} />
+
Debug for TelemetryProvider"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for TelemetryProvider` -
Default for TelemetryProvider"}} />
+
Default for TelemetryProvider"}} />
#### `default` -
default() -> TelemetryProvider"}} />
+
default() -> TelemetryProvider"}} />
### `impl<'de> Deserialize<'de> for TelemetryProvider` @@ -80,7 +80,7 @@ Return the stable configuration value for this provider. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryProvider` @@ -88,11 +88,11 @@ Return the stable configuration value for this provider. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -100,63 +100,63 @@ Return the stable configuration value for this provider. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl Ord for TelemetryProvider` -
Ord for TelemetryProvider"}} />
+
Ord for TelemetryProvider"}} />
#### `cmp` -
cmp(&self, other: &TelemetryProvider) -> Ordering"}} />
+
cmp(&self, other: &TelemetryProvider) -> Ordering"}} />
#### `max` -
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+
max(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
#### `min` -
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
+
min(self, other: Self) -> Selfwhere\n    Self: Sized,"}} />
#### `clamp` -
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
+
clamp(self, min: Self, max: Self) -> Selfwhere\n    Self: Sized,"}} />
### `impl PartialEq for TelemetryProvider` -
PartialEq for TelemetryProvider"}} />
+
PartialEq for TelemetryProvider"}} />
#### `eq` -
eq(&self, other: &TelemetryProvider) -> bool"}} />
+
eq(&self, other: &TelemetryProvider) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl PartialOrd for TelemetryProvider` -
PartialOrd for TelemetryProvider"}} />
+
PartialOrd for TelemetryProvider"}} />
#### `partial_cmp` -
partial_cmp(&self, other: &TelemetryProvider) -> Option<Ordering>"}} />
+
partial_cmp(&self, other: &TelemetryProvider) -> Option<Ordering>"}} />
#### `lt` -
lt(&self, other: &Rhs) -> bool"}} />
+
lt(&self, other: &Rhs) -> bool"}} />
#### `le` -
le(&self, other: &Rhs) -> bool"}} />
+
le(&self, other: &Rhs) -> bool"}} />
#### `gt` -
gt(&self, other: &Rhs) -> bool"}} />
+
gt(&self, other: &Rhs) -> bool"}} />
#### `ge` -
ge(&self, other: &Rhs) -> bool"}} />
+
ge(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryProvider` @@ -164,16 +164,16 @@ Return the stable configuration value for this provider. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for TelemetryProvider` -
Copy for TelemetryProvider"}} />
+
Copy for TelemetryProvider"}} />
### `impl Eq for TelemetryProvider` -
Eq for TelemetryProvider"}} />
+
Eq for TelemetryProvider"}} />
### `impl StructuralPartialEq for TelemetryProvider` -
StructuralPartialEq for TelemetryProvider"}} />
+
StructuralPartialEq for TelemetryProvider"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx index fc493bae7..377b2a224 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx @@ -2,13 +2,13 @@ title: "Function load_adapter_descriptor" sidebar-title: "load_adapter_descriptor" description: "Load an adapter descriptor from JSON package metadata." -position: 71 +position: 72 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
AsRef<Path>,\n) -> Result<AdapterDescriptor>"}} />
+
AsRef<Path>,\n) -> Result<AdapterDescriptor>"}} />
Load an adapter descriptor from JSON package metadata. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx index 142fdba30..de11d4144 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx @@ -2,13 +2,13 @@ title: "Function load_adapter_target_descriptor" sidebar-title: "load_adapter_target_descriptor" description: "Load an Adapter Target Descriptor from declarative JSON metadata." -position: 72 +position: 73 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
AsRef<Path>,\n) -> Result<AdapterTargetDescriptor>"}} />
+
AsRef<Path>,\n) -> Result<AdapterTargetDescriptor>"}} />
Load an Adapter Target Descriptor from declarative JSON metadata. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx index 6bc8a973b..a6d607703 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx @@ -2,7 +2,7 @@ title: "Function resolve_run_plan_from_config" sidebar-title: "resolve_run_plan_from_config" description: "Resolve a typed NVIDIA NeMo Fabric config into a runnable plan." -position: 73 +position: 74 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index f031abd58..61f33ba88 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 145 +position: 146 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -53,6 +53,7 @@ NeMo Fabric config models and loading helpers. - [McpServerPlan](struct-mcpserverplan.mdx): Resolved MCP server exposure. - [MetadataConfig](struct-metadataconfig.mdx): Human-readable metadata. - [ModelConfig](struct-modelconfig.mdx): Model configuration. +- [PricingConfig](struct-pricingconfig.mdx): Central token-price configuration; never changes provider requests. - [RelayAtifConfig](struct-relayatifconfig.mdx): Relay ATIF export configuration. - [RelayAtofConfig](struct-relayatofconfig.mdx): Relay ATOF export configuration. - [RelayComponentConfig](struct-relaycomponentconfig.mdx): Generic NeMo Relay plugin component configuration. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx index bfc4a1fcd..47ec3308d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-fabricconfig.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub metadata: MetadataConfig,\n    pub harness: Option<HarnessConfig>,\n    pub workflow: Option<WorkflowConfig>,\n    pub discovery: Option<DiscoveryConfig>,\n    pub models: BTreeMap<String, ModelConfig>,\n    pub instructions: Option<InstructionsConfig>,\n    pub runtime: RuntimeConfig,\n    pub environment: Option<EnvironmentConfig>,\n    pub tools: Option<ToolsConfig>,\n    pub skills: Option<SkillConfig>,\n    pub mcp: Option<McpConfig>,\n    pub telemetry: Option<TelemetryConfig>,\n    pub system_stats: Option<SystemStatsConfig>,\n    pub relay: Option<RelayConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub metadata: MetadataConfig,\n    pub harness: Option<HarnessConfig>,\n    pub workflow: Option<WorkflowConfig>,\n    pub discovery: Option<DiscoveryConfig>,\n    pub models: BTreeMap<String, ModelConfig>,\n    pub instructions: Option<InstructionsConfig>,\n    pub runtime: RuntimeConfig,\n    pub environment: Option<EnvironmentConfig>,\n    pub tools: Option<ToolsConfig>,\n    pub skills: Option<SkillConfig>,\n    pub mcp: Option<McpConfig>,\n    pub telemetry: Option<TelemetryConfig>,\n    pub system_stats: Option<SystemStatsConfig>,\n    pub relay: Option<RelayConfig>,\n    pub pricing: Option<PricingConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Versioned NVIDIA NeMo Fabric agent config. @@ -77,6 +77,10 @@ Invocation stats: omitted means all endpoint groups, explicit null disables. First-class NeMo Relay integration configuration. +### `pricing: Option` + +Optional catalog override for central token-price estimates. + ### `extensions: BTreeMap` Additive fields not yet recognized by this core version. @@ -85,23 +89,23 @@ Additive fields not yet recognized by this core version. ### `impl Clone for FabricConfig` -
Clone for FabricConfig"}} />
+
Clone for FabricConfig"}} />
#### `clone` -
clone(&self) -> FabricConfig"}} />
+
clone(&self) -> FabricConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for FabricConfig` -
Debug for FabricConfig"}} />
+
Debug for FabricConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for FabricConfig` @@ -109,7 +113,7 @@ Additive fields not yet recognized by this core version. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for FabricConfig` @@ -117,11 +121,11 @@ Additive fields not yet recognized by this core version. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -129,19 +133,19 @@ Additive fields not yet recognized by this core version. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for FabricConfig` -
PartialEq for FabricConfig"}} />
+
PartialEq for FabricConfig"}} />
#### `eq` -
eq(&self, other: &FabricConfig) -> bool"}} />
+
eq(&self, other: &FabricConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for FabricConfig` @@ -149,8 +153,8 @@ Additive fields not yet recognized by this core version. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for FabricConfig` -
StructuralPartialEq for FabricConfig"}} />
+
StructuralPartialEq for FabricConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx new file mode 100644 index 000000000..db976091d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-pricingconfig.mdx @@ -0,0 +1,106 @@ +--- +title: "Struct Pricing Config" +sidebar-title: "PricingConfig" +description: "Central token-price configuration; never changes provider requests." +position: 54 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<PathBuf>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+ +Central token-price configuration; never changes provider requests. + +## Fields + +### `catalog_path: Option` + +Relay-compatible v1 catalog, resolved relative to the config base directory. + +### `extensions: BTreeMap` + +Additive pricing configuration, preserved for future versions. + +## Trait Implementations + +### `impl Clone for PricingConfig` + +
Clone for PricingConfig"}} />
+ +#### `clone` + +
clone(&self) -> PricingConfig"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingConfig` + +
Debug for PricingConfig"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for PricingConfig` + +
Default for PricingConfig"}} />
+ +#### `default` + +
default() -> PricingConfig"}} />
+ +### `impl<'de> Deserialize<'de> for PricingConfig` + +
Deserialize<'de> for PricingConfig"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingConfig` + +
PricingConfig"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingConfig` + +
PartialEq for PricingConfig"}} />
+ +#### `eq` + +
eq(&self, other: &PricingConfig) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingConfig` + +
Serialize for PricingConfig"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for PricingConfig` + +
StructuralPartialEq for PricingConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx index 4a2989ac6..4968c698e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Atif Config" sidebar-title: "RelayAtifConfig" description: "Relay ATIF export configuration." -position: 38 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub agent_name: String,\n    pub agent_version: Option<String>,\n    pub model_name: Option<String>,\n    pub tool_definitions: Option<Vec<Value>>,\n    pub extra: Option<Value>,\n    pub output_directory: Option<PathBuf>,\n    pub filename_template: String,\n    pub storage: Option<Vec<RelayAtifStorageConfig>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub agent_name: String,\n    pub agent_version: Option<String>,\n    pub model_name: Option<String>,\n    pub tool_definitions: Option<Vec<Value>>,\n    pub extra: Option<Value>,\n    pub output_directory: Option<PathBuf>,\n    pub filename_template: String,\n    pub storage: Option<Vec<RelayAtifStorageConfig>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Relay ATIF export configuration. @@ -59,31 +59,31 @@ Additive ATIF fields. ### `impl Clone for RelayAtifConfig` -
Clone for RelayAtifConfig"}} />
+
Clone for RelayAtifConfig"}} />
#### `clone` -
clone(&self) -> RelayAtifConfig"}} />
+
clone(&self) -> RelayAtifConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtifConfig` -
Debug for RelayAtifConfig"}} />
+
Debug for RelayAtifConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtifConfig` -
Default for RelayAtifConfig"}} />
+
Default for RelayAtifConfig"}} />
#### `default` -
default() -> Self"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RelayAtifConfig` @@ -91,7 +91,7 @@ Additive ATIF fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtifConfig` @@ -99,11 +99,11 @@ Additive ATIF fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -111,19 +111,19 @@ Additive ATIF fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtifConfig` -
PartialEq for RelayAtifConfig"}} />
+
PartialEq for RelayAtifConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtifConfig) -> bool"}} />
+
eq(&self, other: &RelayAtifConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtifConfig` @@ -131,8 +131,8 @@ Additive ATIF fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtifConfig` -
StructuralPartialEq for RelayAtifConfig"}} />
+
StructuralPartialEq for RelayAtifConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx index 4ff5d7de9..a9819899e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Atof Config" sidebar-title: "RelayAtofConfig" description: "Relay ATOF export configuration." -position: 39 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub sinks: Vec<RelayAtofSinkConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub sinks: Vec<RelayAtofSinkConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Relay ATOF export configuration. @@ -31,31 +31,31 @@ Additive ATOF fields. ### `impl Clone for RelayAtofConfig` -
Clone for RelayAtofConfig"}} />
+
Clone for RelayAtofConfig"}} />
#### `clone` -
clone(&self) -> RelayAtofConfig"}} />
+
clone(&self) -> RelayAtofConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayAtofConfig` -
Debug for RelayAtofConfig"}} />
+
Debug for RelayAtofConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayAtofConfig` -
Default for RelayAtofConfig"}} />
+
Default for RelayAtofConfig"}} />
#### `default` -
default() -> RelayAtofConfig"}} />
+
default() -> RelayAtofConfig"}} />
### `impl<'de> Deserialize<'de> for RelayAtofConfig` @@ -63,7 +63,7 @@ Additive ATOF fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayAtofConfig` @@ -71,11 +71,11 @@ Additive ATOF fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Additive ATOF fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayAtofConfig` -
PartialEq for RelayAtofConfig"}} />
+
PartialEq for RelayAtofConfig"}} />
#### `eq` -
eq(&self, other: &RelayAtofConfig) -> bool"}} />
+
eq(&self, other: &RelayAtofConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayAtofConfig` @@ -103,8 +103,8 @@ Additive ATOF fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayAtofConfig` -
StructuralPartialEq for RelayAtofConfig"}} />
+
StructuralPartialEq for RelayAtofConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx index 56259d7a5..7c1330a78 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Component Config" sidebar-title: "RelayComponentConfig" description: "Generic NeMo Relay plugin component configuration." -position: 40 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub enabled: bool,\n    pub config: BTreeMap<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub enabled: bool,\n    pub config: BTreeMap<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Generic NeMo Relay plugin component configuration. @@ -35,23 +35,23 @@ Additive component fields. ### `impl Clone for RelayComponentConfig` -
Clone for RelayComponentConfig"}} />
+
Clone for RelayComponentConfig"}} />
#### `clone` -
clone(&self) -> RelayComponentConfig"}} />
+
clone(&self) -> RelayComponentConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayComponentConfig` -
Debug for RelayComponentConfig"}} />
+
Debug for RelayComponentConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayComponentConfig` @@ -59,7 +59,7 @@ Additive component fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayComponentConfig` @@ -67,11 +67,11 @@ Additive component fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive component fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayComponentConfig` -
PartialEq for RelayComponentConfig"}} />
+
PartialEq for RelayComponentConfig"}} />
#### `eq` -
eq(&self, other: &RelayComponentConfig) -> bool"}} />
+
eq(&self, other: &RelayComponentConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayComponentConfig` @@ -99,8 +99,8 @@ Additive component fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayComponentConfig` -
StructuralPartialEq for RelayComponentConfig"}} />
+
StructuralPartialEq for RelayComponentConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx index 570ef8d8e..f1aae1fe7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Config" sidebar-title: "RelayConfig" description: "NeMo Relay integration configuration." -position: 41 +position: 42 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Option<String>,\n    pub output_dir: Option<PathBuf>,\n    pub observability: Option<RelayObservabilityConfig>,\n    pub components: Vec<RelayComponentConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
Option<String>,\n    pub output_dir: Option<PathBuf>,\n    pub observability: Option<RelayObservabilityConfig>,\n    pub components: Vec<RelayComponentConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
NeMo Relay integration configuration. @@ -43,31 +43,31 @@ Additive Relay fields. ### `impl Clone for RelayConfig` -
Clone for RelayConfig"}} />
+
Clone for RelayConfig"}} />
#### `clone` -
clone(&self) -> RelayConfig"}} />
+
clone(&self) -> RelayConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayConfig` -
Debug for RelayConfig"}} />
+
Debug for RelayConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayConfig` -
Default for RelayConfig"}} />
+
Default for RelayConfig"}} />
#### `default` -
default() -> RelayConfig"}} />
+
default() -> RelayConfig"}} />
### `impl<'de> Deserialize<'de> for RelayConfig` @@ -75,7 +75,7 @@ Additive Relay fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayConfig` @@ -83,11 +83,11 @@ Additive Relay fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -95,19 +95,19 @@ Additive Relay fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayConfig` -
PartialEq for RelayConfig"}} />
+
PartialEq for RelayConfig"}} />
#### `eq` -
eq(&self, other: &RelayConfig) -> bool"}} />
+
eq(&self, other: &RelayConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayConfig` @@ -115,8 +115,8 @@ Additive Relay fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayConfig` -
StructuralPartialEq for RelayConfig"}} />
+
StructuralPartialEq for RelayConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx index 712c72795..753715ba6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Config Policy" sidebar-title: "RelayConfigPolicy" description: "Relay validation policy." -position: 42 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -31,31 +31,31 @@ Policy for unsupported values. ### `impl Clone for RelayConfigPolicy` -
Clone for RelayConfigPolicy"}} />
+
Clone for RelayConfigPolicy"}} />
#### `clone` -
clone(&self) -> RelayConfigPolicy"}} />
+
clone(&self) -> RelayConfigPolicy"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayConfigPolicy` -
Debug for RelayConfigPolicy"}} />
+
Debug for RelayConfigPolicy"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayConfigPolicy` -
Default for RelayConfigPolicy"}} />
+
Default for RelayConfigPolicy"}} />
#### `default` -
default() -> Self"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RelayConfigPolicy` @@ -63,7 +63,7 @@ Policy for unsupported values. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayConfigPolicy` @@ -71,11 +71,11 @@ Policy for unsupported values. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Policy for unsupported values. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayConfigPolicy` -
PartialEq for RelayConfigPolicy"}} />
+
PartialEq for RelayConfigPolicy"}} />
#### `eq` -
eq(&self, other: &RelayConfigPolicy) -> bool"}} />
+
eq(&self, other: &RelayConfigPolicy) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayConfigPolicy` @@ -103,8 +103,8 @@ Policy for unsupported values. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayConfigPolicy` -
StructuralPartialEq for RelayConfigPolicy"}} />
+
StructuralPartialEq for RelayConfigPolicy"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx index a01db8716..eb68b2061 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Observability Config" sidebar-title: "RelayObservabilityConfig" description: "NeMo Relay observability component configuration." -position: 43 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
u32,\n    pub atof: Option<RelayAtofConfig>,\n    pub atif: Option<RelayAtifConfig>,\n    pub opentelemetry: Option<RelayOpenTelemetryConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub enable_full_payloads: bool,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
u32,\n    pub atof: Option<RelayAtofConfig>,\n    pub atif: Option<RelayAtifConfig>,\n    pub opentelemetry: Option<RelayOpenTelemetryConfig>,\n    pub policy: Option<RelayConfigPolicy>,\n    pub enable_full_payloads: bool,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
NeMo Relay observability component configuration. @@ -47,31 +47,31 @@ Additive observability fields. ### `impl Clone for RelayObservabilityConfig` -
Clone for RelayObservabilityConfig"}} />
+
Clone for RelayObservabilityConfig"}} />
#### `clone` -
clone(&self) -> RelayObservabilityConfig"}} />
+
clone(&self) -> RelayObservabilityConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayObservabilityConfig` -
Debug for RelayObservabilityConfig"}} />
+
Debug for RelayObservabilityConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayObservabilityConfig` -
Default for RelayObservabilityConfig"}} />
+
Default for RelayObservabilityConfig"}} />
#### `default` -
default() -> Self"}} />
+
default() -> Self"}} />
### `impl<'de> Deserialize<'de> for RelayObservabilityConfig` @@ -79,7 +79,7 @@ Additive observability fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayObservabilityConfig` @@ -87,11 +87,11 @@ Additive observability fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -99,19 +99,19 @@ Additive observability fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayObservabilityConfig` -
PartialEq for RelayObservabilityConfig"}} />
+
PartialEq for RelayObservabilityConfig"}} />
#### `eq` -
eq(&self, other: &RelayObservabilityConfig) -> bool"}} />
+
eq(&self, other: &RelayObservabilityConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayObservabilityConfig` @@ -119,8 +119,8 @@ Additive observability fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayObservabilityConfig` -
StructuralPartialEq for RelayObservabilityConfig"}} />
+
StructuralPartialEq for RelayObservabilityConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx index 4bade1462..e46864ec7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Open Telemetry Config" sidebar-title: "RelayOpenTelemetryConfig" description: "Relay OpenTelemetry export section." -position: 44 +position: 45 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub endpoints: Vec<RelayOpenTelemetryEndpointConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub endpoints: Vec<RelayOpenTelemetryEndpointConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Relay OpenTelemetry export section. @@ -31,31 +31,31 @@ Additive OpenTelemetry section fields. ### `impl Clone for RelayOpenTelemetryConfig` -
Clone for RelayOpenTelemetryConfig"}} />
+
Clone for RelayOpenTelemetryConfig"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryConfig"}} />
+
clone(&self) -> RelayOpenTelemetryConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryConfig` -
Debug for RelayOpenTelemetryConfig"}} />
+
Debug for RelayOpenTelemetryConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RelayOpenTelemetryConfig` -
Default for RelayOpenTelemetryConfig"}} />
+
Default for RelayOpenTelemetryConfig"}} />
#### `default` -
default() -> RelayOpenTelemetryConfig"}} />
+
default() -> RelayOpenTelemetryConfig"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryConfig` @@ -63,7 +63,7 @@ Additive OpenTelemetry section fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryConfig` @@ -71,11 +71,11 @@ Additive OpenTelemetry section fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Additive OpenTelemetry section fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryConfig` -
PartialEq for RelayOpenTelemetryConfig"}} />
+
PartialEq for RelayOpenTelemetryConfig"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryConfig) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryConfig` @@ -103,8 +103,8 @@ Additive OpenTelemetry section fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryConfig` -
StructuralPartialEq for RelayOpenTelemetryConfig"}} />
+
StructuralPartialEq for RelayOpenTelemetryConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx index da8138f4e..aafe4a8a3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Relay Open Telemetry Endpoint Config" sidebar-title: "RelayOpenTelemetryEndpointConfig" description: "One typed Relay OpenTelemetry OTLP destination." -position: 45 +position: 46 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
RelayOpenTelemetryEndpointType,\n    pub endpoint: String,\n    pub mark_projection: RelayOpenTelemetryMarkProjection,\n    pub mark_exclude_names: Vec<String>,\n    pub attribute_mappings: Vec<BTreeMap<String, String>>,\n    pub transport: RelayOtlpTransport,\n    pub service_name: String,\n    pub service_namespace: Option<String>,\n    pub service_version: Option<String>,\n    pub instrumentation_scope: String,\n    pub timeout_millis: u64,\n    pub headers: BTreeMap<String, String>,\n    pub header_env: BTreeMap<String, String>,\n    pub resource_attributes: BTreeMap<String, String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
RelayOpenTelemetryEndpointType,\n    pub endpoint: String,\n    pub mark_projection: RelayOpenTelemetryMarkProjection,\n    pub mark_exclude_names: Vec<String>,\n    pub attribute_mappings: Vec<BTreeMap<String, String>>,\n    pub transport: RelayOtlpTransport,\n    pub service_name: String,\n    pub service_namespace: Option<String>,\n    pub service_version: Option<String>,\n    pub instrumentation_scope: String,\n    pub timeout_millis: u64,\n    pub headers: BTreeMap<String, String>,\n    pub header_env: BTreeMap<String, String>,\n    pub resource_attributes: BTreeMap<String, String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
One typed Relay OpenTelemetry OTLP destination. @@ -79,23 +79,23 @@ Additive endpoint fields. ### `impl Clone for RelayOpenTelemetryEndpointConfig` -
Clone for RelayOpenTelemetryEndpointConfig"}} />
+
Clone for RelayOpenTelemetryEndpointConfig"}} />
#### `clone` -
clone(&self) -> RelayOpenTelemetryEndpointConfig"}} />
+
clone(&self) -> RelayOpenTelemetryEndpointConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RelayOpenTelemetryEndpointConfig` -
Debug for RelayOpenTelemetryEndpointConfig"}} />
+
Debug for RelayOpenTelemetryEndpointConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RelayOpenTelemetryEndpointConfig` @@ -103,7 +103,7 @@ Additive endpoint fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RelayOpenTelemetryEndpointConfig` @@ -111,11 +111,11 @@ Additive endpoint fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -123,19 +123,19 @@ Additive endpoint fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RelayOpenTelemetryEndpointConfig` -
PartialEq for RelayOpenTelemetryEndpointConfig"}} />
+
PartialEq for RelayOpenTelemetryEndpointConfig"}} />
#### `eq` -
eq(&self, other: &RelayOpenTelemetryEndpointConfig) -> bool"}} />
+
eq(&self, other: &RelayOpenTelemetryEndpointConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RelayOpenTelemetryEndpointConfig` @@ -143,8 +143,8 @@ Additive endpoint fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RelayOpenTelemetryEndpointConfig` -
StructuralPartialEq for RelayOpenTelemetryEndpointConfig"}} />
+
StructuralPartialEq for RelayOpenTelemetryEndpointConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx index 8e70ade88..a2f7b346d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx @@ -2,14 +2,14 @@ title: "Struct Resolve Context" sidebar-title: "ResolveContext" description: "Source context used when resolving an in-memory NeMo Fabric config." -position: 55 +position: 56 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n}"}} />
+
PathBuf,\n}"}} />
Source context used when resolving an in-memory NeMo Fabric config. @@ -27,7 +27,7 @@ Base directory used to resolve relative NeMo Fabric paths. #### `new` -
Into<PathBuf>) -> Self"}} />
+
Into<PathBuf>) -> Self"}} />
Build a context with an explicit base directory. @@ -35,36 +35,36 @@ Build a context with an explicit base directory. ### `impl Clone for ResolveContext` -
Clone for ResolveContext"}} />
+
Clone for ResolveContext"}} />
#### `clone` -
clone(&self) -> ResolveContext"}} />
+
clone(&self) -> ResolveContext"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolveContext` -
Debug for ResolveContext"}} />
+
Debug for ResolveContext"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl PartialEq for ResolveContext` -
PartialEq for ResolveContext"}} />
+
PartialEq for ResolveContext"}} />
#### `eq` -
eq(&self, other: &ResolveContext) -> bool"}} />
+
eq(&self, other: &ResolveContext) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl StructuralPartialEq for ResolveContext` -
StructuralPartialEq for ResolveContext"}} />
+
StructuralPartialEq for ResolveContext"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx index 7c1e65663..e246b640b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx @@ -2,14 +2,14 @@ title: "Struct Resolved Adapter Descriptor" sidebar-title: "ResolvedAdapterDescriptor" description: "Adapter descriptor selected for a run plan." -position: 56 +position: 57 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterDescriptor,\n}"}} />
+
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterDescriptor,\n}"}} />
Adapter descriptor selected for a run plan. @@ -27,23 +27,23 @@ Adapter-owned compatibility and capability metadata. ### `impl Clone for ResolvedAdapterDescriptor` -
Clone for ResolvedAdapterDescriptor"}} />
+
Clone for ResolvedAdapterDescriptor"}} />
#### `clone` -
clone(&self) -> ResolvedAdapterDescriptor"}} />
+
clone(&self) -> ResolvedAdapterDescriptor"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolvedAdapterDescriptor` -
Debug for ResolvedAdapterDescriptor"}} />
+
Debug for ResolvedAdapterDescriptor"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ResolvedAdapterDescriptor` @@ -51,7 +51,7 @@ Adapter-owned compatibility and capability metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ResolvedAdapterDescriptor` @@ -59,11 +59,11 @@ Adapter-owned compatibility and capability metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -71,19 +71,19 @@ Adapter-owned compatibility and capability metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ResolvedAdapterDescriptor` -
PartialEq for ResolvedAdapterDescriptor"}} />
+
PartialEq for ResolvedAdapterDescriptor"}} />
#### `eq` -
eq(&self, other: &ResolvedAdapterDescriptor) -> bool"}} />
+
eq(&self, other: &ResolvedAdapterDescriptor) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ResolvedAdapterDescriptor` @@ -91,8 +91,8 @@ Adapter-owned compatibility and capability metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ResolvedAdapterDescriptor` -
StructuralPartialEq for ResolvedAdapterDescriptor"}} />
+
StructuralPartialEq for ResolvedAdapterDescriptor"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx index 8adac3efc..3a0831bf1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx @@ -2,14 +2,14 @@ title: "Struct Resolved Adapter Target Descriptor" sidebar-title: "ResolvedAdapterTargetDescriptor" description: "Adapter target descriptor selected for a run plan." -position: 57 +position: 58 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterTargetDescriptor,\n}"}} />
+
Vec<DescriptorProvenance>,\n    pub descriptor: AdapterTargetDescriptor,\n}"}} />
Adapter target descriptor selected for a run plan. @@ -27,23 +27,23 @@ Target-specific resolution and validation metadata. ### `impl Clone for ResolvedAdapterTargetDescriptor` -
Clone for ResolvedAdapterTargetDescriptor"}} />
+
Clone for ResolvedAdapterTargetDescriptor"}} />
#### `clone` -
clone(&self) -> ResolvedAdapterTargetDescriptor"}} />
+
clone(&self) -> ResolvedAdapterTargetDescriptor"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ResolvedAdapterTargetDescriptor` -
Debug for ResolvedAdapterTargetDescriptor"}} />
+
Debug for ResolvedAdapterTargetDescriptor"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ResolvedAdapterTargetDescriptor` @@ -51,7 +51,7 @@ Target-specific resolution and validation metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ResolvedAdapterTargetDescriptor` @@ -59,11 +59,11 @@ Target-specific resolution and validation metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -71,19 +71,19 @@ Target-specific resolution and validation metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ResolvedAdapterTargetDescriptor` -
PartialEq for ResolvedAdapterTargetDescriptor"}} />
+
PartialEq for ResolvedAdapterTargetDescriptor"}} />
#### `eq` -
eq(&self, other: &ResolvedAdapterTargetDescriptor) -> bool"}} />
+
eq(&self, other: &ResolvedAdapterTargetDescriptor) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ResolvedAdapterTargetDescriptor` @@ -91,8 +91,8 @@ Target-specific resolution and validation metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ResolvedAdapterTargetDescriptor` -
StructuralPartialEq for ResolvedAdapterTargetDescriptor"}} />
+
StructuralPartialEq for ResolvedAdapterTargetDescriptor"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx index e3c43e4e1..62cae0e6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx @@ -2,19 +2,23 @@ title: "Struct RunPlan" sidebar-title: "RunPlan" description: "Resolved NeMo Fabric run plan." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub base_dir: PathBuf,\n    pub config: FabricConfig,\n    pub agent_config: AgentConfig,\n    pub adapter_descriptor: Option<ResolvedAdapterDescriptor>,\n    pub adapter_target_descriptor: Option<ResolvedAdapterTargetDescriptor>,\n    pub resolution: Option<ResolutionStrategy>,\n    pub environment_plan: Option<EnvironmentPlan>,\n    pub capability_plan: CapabilityPlan,\n    pub capabilities: RuntimeCapabilities,\n    pub telemetry_plan: Option<TelemetryPlan>,\n}"}} />
+
Option<UserPriceCatalog>,\n    pub agent_name: String,\n    pub base_dir: PathBuf,\n    pub config: FabricConfig,\n    pub agent_config: AgentConfig,\n    pub adapter_descriptor: Option<ResolvedAdapterDescriptor>,\n    pub adapter_target_descriptor: Option<ResolvedAdapterTargetDescriptor>,\n    pub resolution: Option<ResolutionStrategy>,\n    pub environment_plan: Option<EnvironmentPlan>,\n    pub capability_plan: CapabilityPlan,\n    pub capabilities: RuntimeCapabilities,\n    pub telemetry_plan: Option<TelemetryPlan>,\n}"}} />
Resolved NeMo Fabric run plan. ## Fields +### `user_price_catalog: Option` + +Catalog bytes are frozen during planning. + ### `agent_name: String` Stable agent name. @@ -63,23 +67,23 @@ Resolved telemetry pass-through plan. ### `impl Clone for RunPlan` -
Clone for RunPlan"}} />
+
Clone for RunPlan"}} />
#### `clone` -
clone(&self) -> RunPlan"}} />
+
clone(&self) -> RunPlan"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RunPlan` -
Debug for RunPlan"}} />
+
Debug for RunPlan"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RunPlan` @@ -87,7 +91,7 @@ Resolved telemetry pass-through plan. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RunPlan` @@ -95,11 +99,11 @@ Resolved telemetry pass-through plan. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -107,19 +111,19 @@ Resolved telemetry pass-through plan. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RunPlan` -
PartialEq for RunPlan"}} />
+
PartialEq for RunPlan"}} />
#### `eq` -
eq(&self, other: &RunPlan) -> bool"}} />
+
eq(&self, other: &RunPlan) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RunPlan` @@ -127,8 +131,8 @@ Resolved telemetry pass-through plan. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RunPlan` -
StructuralPartialEq for RunPlan"}} />
+
StructuralPartialEq for RunPlan"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx index 0d972cb87..6df2d0128 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx @@ -2,14 +2,14 @@ title: "Struct Runtime Capabilities" sidebar-title: "RuntimeCapabilities" description: "Lifecycle behavior implemented by a resolved runtime path." -position: 59 +position: 60 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
bool,\n    pub streaming: bool,\n    pub updates: bool,\n    pub cancellation: bool,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
bool,\n    pub streaming: bool,\n    pub updates: bool,\n    pub cancellation: bool,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
Lifecycle behavior implemented by a resolved runtime path. @@ -39,31 +39,31 @@ Additional adapter-specific capability metadata. ### `impl Clone for RuntimeCapabilities` -
Clone for RuntimeCapabilities"}} />
+
Clone for RuntimeCapabilities"}} />
#### `clone` -
clone(&self) -> RuntimeCapabilities"}} />
+
clone(&self) -> RuntimeCapabilities"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RuntimeCapabilities` -
Debug for RuntimeCapabilities"}} />
+
Debug for RuntimeCapabilities"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for RuntimeCapabilities` -
Default for RuntimeCapabilities"}} />
+
Default for RuntimeCapabilities"}} />
#### `default` -
default() -> RuntimeCapabilities"}} />
+
default() -> RuntimeCapabilities"}} />
### `impl<'de> Deserialize<'de> for RuntimeCapabilities` @@ -71,7 +71,7 @@ Additional adapter-specific capability metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RuntimeCapabilities` @@ -79,11 +79,11 @@ Additional adapter-specific capability metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -91,19 +91,19 @@ Additional adapter-specific capability metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RuntimeCapabilities` -
PartialEq for RuntimeCapabilities"}} />
+
PartialEq for RuntimeCapabilities"}} />
#### `eq` -
eq(&self, other: &RuntimeCapabilities) -> bool"}} />
+
eq(&self, other: &RuntimeCapabilities) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RuntimeCapabilities` @@ -111,8 +111,8 @@ Additional adapter-specific capability metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RuntimeCapabilities` -
StructuralPartialEq for RuntimeCapabilities"}} />
+
StructuralPartialEq for RuntimeCapabilities"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx index 2c89a11b2..662d07246 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Runtime Config" sidebar-title: "RuntimeConfig" description: "Invocation runtime contract." -position: 60 +position: 61 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Invocation runtime contract. @@ -43,23 +43,23 @@ Additive normalized runtime fields. ### `impl Clone for RuntimeConfig` -
Clone for RuntimeConfig"}} />
+
Clone for RuntimeConfig"}} />
#### `clone` -
clone(&self) -> RuntimeConfig"}} />
+
clone(&self) -> RuntimeConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for RuntimeConfig` -
Debug for RuntimeConfig"}} />
+
Debug for RuntimeConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for RuntimeConfig` @@ -67,7 +67,7 @@ Additive normalized runtime fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for RuntimeConfig` @@ -75,11 +75,11 @@ Additive normalized runtime fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Additive normalized runtime fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for RuntimeConfig` -
PartialEq for RuntimeConfig"}} />
+
PartialEq for RuntimeConfig"}} />
#### `eq` -
eq(&self, other: &RuntimeConfig) -> bool"}} />
+
eq(&self, other: &RuntimeConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for RuntimeConfig` @@ -107,8 +107,8 @@ Additive normalized runtime fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for RuntimeConfig` -
StructuralPartialEq for RuntimeConfig"}} />
+
StructuralPartialEq for RuntimeConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx index 47644f228..9fac093e1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Skill Config" sidebar-title: "SkillConfig" description: "Skill capability configuration." -position: 61 +position: 62 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<PathBuf>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
Vec<PathBuf>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Skill capability configuration. @@ -27,31 +27,31 @@ Additive skill fields. ### `impl Clone for SkillConfig` -
Clone for SkillConfig"}} />
+
Clone for SkillConfig"}} />
#### `clone` -
clone(&self) -> SkillConfig"}} />
+
clone(&self) -> SkillConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for SkillConfig` -
Debug for SkillConfig"}} />
+
Debug for SkillConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for SkillConfig` -
Default for SkillConfig"}} />
+
Default for SkillConfig"}} />
#### `default` -
default() -> SkillConfig"}} />
+
default() -> SkillConfig"}} />
### `impl<'de> Deserialize<'de> for SkillConfig` @@ -59,7 +59,7 @@ Additive skill fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for SkillConfig` @@ -67,11 +67,11 @@ Additive skill fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive skill fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for SkillConfig` -
PartialEq for SkillConfig"}} />
+
PartialEq for SkillConfig"}} />
#### `eq` -
eq(&self, other: &SkillConfig) -> bool"}} />
+
eq(&self, other: &SkillConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for SkillConfig` @@ -99,8 +99,8 @@ Additive skill fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for SkillConfig` -
StructuralPartialEq for SkillConfig"}} />
+
StructuralPartialEq for SkillConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx index 2e1b4af6b..7f1987b33 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Telemetry Config" sidebar-title: "TelemetryConfig" description: "Telemetry configuration." -position: 62 +position: 63 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<TelemetryProvider, TelemetryProviderConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
BTreeMap<TelemetryProvider, TelemetryProviderConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Telemetry configuration. @@ -27,23 +27,23 @@ Additive telemetry fields. ### `impl Clone for TelemetryConfig` -
Clone for TelemetryConfig"}} />
+
Clone for TelemetryConfig"}} />
#### `clone` -
clone(&self) -> TelemetryConfig"}} />
+
clone(&self) -> TelemetryConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryConfig` -
Debug for TelemetryConfig"}} />
+
Debug for TelemetryConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for TelemetryConfig` @@ -51,7 +51,7 @@ Additive telemetry fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryConfig` @@ -59,11 +59,11 @@ Additive telemetry fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -71,19 +71,19 @@ Additive telemetry fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for TelemetryConfig` -
PartialEq for TelemetryConfig"}} />
+
PartialEq for TelemetryConfig"}} />
#### `eq` -
eq(&self, other: &TelemetryConfig) -> bool"}} />
+
eq(&self, other: &TelemetryConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryConfig` @@ -91,8 +91,8 @@ Additive telemetry fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for TelemetryConfig` -
StructuralPartialEq for TelemetryConfig"}} />
+
StructuralPartialEq for TelemetryConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx index f12176331..20f00922c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx @@ -2,14 +2,14 @@ title: "Struct Telemetry Plan" sidebar-title: "TelemetryPlan" description: "Resolved telemetry plan." -position: 63 +position: 64 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Vec<TelemetryProvider>,\n    pub relay_enabled: bool,\n    pub relay_project: Option<String>,\n    pub relay_output_dir: Option<PathBuf>,\n    pub relay_config: Option<Value>,\n    pub native_config: Option<Value>,\n    pub adapter_outputs: Vec<String>,\n}"}} />
+
Vec<TelemetryProvider>,\n    pub relay_enabled: bool,\n    pub relay_project: Option<String>,\n    pub relay_output_dir: Option<PathBuf>,\n    pub relay_config: Option<Value>,\n    pub native_config: Option<Value>,\n    pub adapter_outputs: Vec<String>,\n}"}} />
Resolved telemetry plan. @@ -47,23 +47,23 @@ Telemetry outputs declared by the selected adapter descriptor. ### `impl Clone for TelemetryPlan` -
Clone for TelemetryPlan"}} />
+
Clone for TelemetryPlan"}} />
#### `clone` -
clone(&self) -> TelemetryPlan"}} />
+
clone(&self) -> TelemetryPlan"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryPlan` -
Debug for TelemetryPlan"}} />
+
Debug for TelemetryPlan"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for TelemetryPlan` @@ -71,7 +71,7 @@ Telemetry outputs declared by the selected adapter descriptor. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryPlan` @@ -79,11 +79,11 @@ Telemetry outputs declared by the selected adapter descriptor. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -91,19 +91,19 @@ Telemetry outputs declared by the selected adapter descriptor. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for TelemetryPlan` -
PartialEq for TelemetryPlan"}} />
+
PartialEq for TelemetryPlan"}} />
#### `eq` -
eq(&self, other: &TelemetryPlan) -> bool"}} />
+
eq(&self, other: &TelemetryPlan) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryPlan` @@ -111,8 +111,8 @@ Telemetry outputs declared by the selected adapter descriptor. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for TelemetryPlan` -
StructuralPartialEq for TelemetryPlan"}} />
+
StructuralPartialEq for TelemetryPlan"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx index f683b0886..cd185c152 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Telemetry Provider Config" sidebar-title: "TelemetryProviderConfig" description: "Provider-specific telemetry configuration." -position: 65 +position: 66 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Option<Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
Option<Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Provider-specific telemetry configuration. @@ -27,31 +27,31 @@ Additive provider fields. ### `impl Clone for TelemetryProviderConfig` -
Clone for TelemetryProviderConfig"}} />
+
Clone for TelemetryProviderConfig"}} />
#### `clone` -
clone(&self) -> TelemetryProviderConfig"}} />
+
clone(&self) -> TelemetryProviderConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for TelemetryProviderConfig` -
Debug for TelemetryProviderConfig"}} />
+
Debug for TelemetryProviderConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for TelemetryProviderConfig` -
Default for TelemetryProviderConfig"}} />
+
Default for TelemetryProviderConfig"}} />
#### `default` -
default() -> TelemetryProviderConfig"}} />
+
default() -> TelemetryProviderConfig"}} />
### `impl<'de> Deserialize<'de> for TelemetryProviderConfig` @@ -59,7 +59,7 @@ Additive provider fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for TelemetryProviderConfig` @@ -67,11 +67,11 @@ Additive provider fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive provider fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for TelemetryProviderConfig` -
PartialEq for TelemetryProviderConfig"}} />
+
PartialEq for TelemetryProviderConfig"}} />
#### `eq` -
eq(&self, other: &TelemetryProviderConfig) -> bool"}} />
+
eq(&self, other: &TelemetryProviderConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for TelemetryProviderConfig` @@ -99,8 +99,8 @@ Additive provider fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for TelemetryProviderConfig` -
StructuralPartialEq for TelemetryProviderConfig"}} />
+
StructuralPartialEq for TelemetryProviderConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx index b5e0b460f..1a22133b4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Tool Definition Config" sidebar-title: "ToolDefinitionConfig" description: "One named normalized tool or tool-group definition." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub ref: String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub ref: String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
One named normalized tool or tool-group definition. @@ -35,23 +35,23 @@ Additive definition fields. ### `impl Clone for ToolDefinitionConfig` -
Clone for ToolDefinitionConfig"}} />
+
Clone for ToolDefinitionConfig"}} />
#### `clone` -
clone(&self) -> ToolDefinitionConfig"}} />
+
clone(&self) -> ToolDefinitionConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ToolDefinitionConfig` -
Debug for ToolDefinitionConfig"}} />
+
Debug for ToolDefinitionConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for ToolDefinitionConfig` @@ -59,7 +59,7 @@ Additive definition fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ToolDefinitionConfig` @@ -67,11 +67,11 @@ Additive definition fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Additive definition fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ToolDefinitionConfig` -
PartialEq for ToolDefinitionConfig"}} />
+
PartialEq for ToolDefinitionConfig"}} />
#### `eq` -
eq(&self, other: &ToolDefinitionConfig) -> bool"}} />
+
eq(&self, other: &ToolDefinitionConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ToolDefinitionConfig` @@ -99,8 +99,8 @@ Additive definition fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ToolDefinitionConfig` -
StructuralPartialEq for ToolDefinitionConfig"}} />
+
StructuralPartialEq for ToolDefinitionConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx index 64b2d6eec..655ac9d26 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Tools Config" sidebar-title: "ToolsConfig" description: "Harness-neutral tool capability configuration." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Harness-neutral tool capability configuration. @@ -35,31 +35,31 @@ Additive tool configuration fields. ### `impl Clone for ToolsConfig` -
Clone for ToolsConfig"}} />
+
Clone for ToolsConfig"}} />
#### `clone` -
clone(&self) -> ToolsConfig"}} />
+
clone(&self) -> ToolsConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ToolsConfig` -
Debug for ToolsConfig"}} />
+
Debug for ToolsConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for ToolsConfig` -
Default for ToolsConfig"}} />
+
Default for ToolsConfig"}} />
#### `default` -
default() -> ToolsConfig"}} />
+
default() -> ToolsConfig"}} />
### `impl<'de> Deserialize<'de> for ToolsConfig` @@ -67,7 +67,7 @@ Additive tool configuration fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ToolsConfig` @@ -75,11 +75,11 @@ Additive tool configuration fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Additive tool configuration fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ToolsConfig` -
PartialEq for ToolsConfig"}} />
+
PartialEq for ToolsConfig"}} />
#### `eq` -
eq(&self, other: &ToolsConfig) -> bool"}} />
+
eq(&self, other: &ToolsConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ToolsConfig` @@ -107,8 +107,8 @@ Additive tool configuration fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ToolsConfig` -
StructuralPartialEq for ToolsConfig"}} />
+
StructuralPartialEq for ToolsConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx index d2b51fe8b..f9466c489 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx @@ -2,14 +2,14 @@ title: "Struct Tools Plan" sidebar-title: "ToolsPlan" description: "Normalized tool policy for a run." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n}"}} />
+
BTreeMap<String, ToolDefinitionConfig>,\n    pub enabled: Option<Vec<String>>,\n    pub blocked: Vec<String>,\n}"}} />
Normalized tool policy for a run. @@ -31,31 +31,31 @@ Adapter-native tool names to block. ### `impl Clone for ToolsPlan` -
Clone for ToolsPlan"}} />
+
Clone for ToolsPlan"}} />
#### `clone` -
clone(&self) -> ToolsPlan"}} />
+
clone(&self) -> ToolsPlan"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for ToolsPlan` -
Debug for ToolsPlan"}} />
+
Debug for ToolsPlan"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Default for ToolsPlan` -
Default for ToolsPlan"}} />
+
Default for ToolsPlan"}} />
#### `default` -
default() -> ToolsPlan"}} />
+
default() -> ToolsPlan"}} />
### `impl<'de> Deserialize<'de> for ToolsPlan` @@ -63,7 +63,7 @@ Adapter-native tool names to block. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for ToolsPlan` @@ -71,11 +71,11 @@ Adapter-native tool names to block. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -83,19 +83,19 @@ Adapter-native tool names to block. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for ToolsPlan` -
PartialEq for ToolsPlan"}} />
+
PartialEq for ToolsPlan"}} />
#### `eq` -
eq(&self, other: &ToolsPlan) -> bool"}} />
+
eq(&self, other: &ToolsPlan) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for ToolsPlan` @@ -103,8 +103,8 @@ Adapter-native tool names to block. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for ToolsPlan` -
StructuralPartialEq for ToolsPlan"}} />
+
StructuralPartialEq for ToolsPlan"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx index 21beec0df..92f3dbaf8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Workflow Config" sidebar-title: "WorkflowConfig" description: "Registered workflow target and immutable construction settings." -position: 68 +position: 69 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub settings: Map<String, Value>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Registered workflow target and immutable construction settings. @@ -31,23 +31,23 @@ Additive workflow fields. ### `impl Clone for WorkflowConfig` -
Clone for WorkflowConfig"}} />
+
Clone for WorkflowConfig"}} />
#### `clone` -
clone(&self) -> WorkflowConfig"}} />
+
clone(&self) -> WorkflowConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for WorkflowConfig` -
Debug for WorkflowConfig"}} />
+
Debug for WorkflowConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for WorkflowConfig` @@ -55,7 +55,7 @@ Additive workflow fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for WorkflowConfig` @@ -63,11 +63,11 @@ Additive workflow fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Additive workflow fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for WorkflowConfig` -
PartialEq for WorkflowConfig"}} />
+
PartialEq for WorkflowConfig"}} />
#### `eq` -
eq(&self, other: &WorkflowConfig) -> bool"}} />
+
eq(&self, other: &WorkflowConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for WorkflowConfig` @@ -95,8 +95,8 @@ Additive workflow fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for WorkflowConfig` -
StructuralPartialEq for WorkflowConfig"}} />
+
StructuralPartialEq for WorkflowConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx index a002d8fc3..6c99169ea 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Workflow Entrypoint Config" sidebar-title: "WorkflowEntrypointConfig" description: "Adapter-owned workflow entry point supplied by an Adapter Target Descriptor." -position: 69 +position: 70 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub ref: String,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub ref: String,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Adapter-owned workflow entry point supplied by an Adapter Target Descriptor. @@ -31,23 +31,23 @@ Additive workflow entry-point fields. ### `impl Clone for WorkflowEntrypointConfig` -
Clone for WorkflowEntrypointConfig"}} />
+
Clone for WorkflowEntrypointConfig"}} />
#### `clone` -
clone(&self) -> WorkflowEntrypointConfig"}} />
+
clone(&self) -> WorkflowEntrypointConfig"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for WorkflowEntrypointConfig` -
Debug for WorkflowEntrypointConfig"}} />
+
Debug for WorkflowEntrypointConfig"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for WorkflowEntrypointConfig` @@ -55,7 +55,7 @@ Additive workflow entry-point fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for WorkflowEntrypointConfig` @@ -63,11 +63,11 @@ Additive workflow entry-point fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Additive workflow entry-point fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for WorkflowEntrypointConfig` -
PartialEq for WorkflowEntrypointConfig"}} />
+
PartialEq for WorkflowEntrypointConfig"}} />
#### `eq` -
eq(&self, other: &WorkflowEntrypointConfig) -> bool"}} />
+
eq(&self, other: &WorkflowEntrypointConfig) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for WorkflowEntrypointConfig` @@ -95,8 +95,8 @@ Additive workflow entry-point fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for WorkflowEntrypointConfig` -
StructuralPartialEq for WorkflowEntrypointConfig"}} />
+
StructuralPartialEq for WorkflowEntrypointConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx index e66041abb..3937f9f5f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx @@ -2,14 +2,14 @@ title: "Struct Workflow Target Spec" sidebar-title: "WorkflowTargetSpec" description: "Workflow-specific Adapter Target Descriptor fields." -position: 70 +position: 71 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
WorkflowEntrypointConfig,\n    pub settings_schema: Option<Map<String, Value>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
WorkflowEntrypointConfig,\n    pub settings_schema: Option<Map<String, Value>>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Workflow-specific Adapter Target Descriptor fields. @@ -31,23 +31,23 @@ Additive workflow target fields. ### `impl Clone for WorkflowTargetSpec` -
Clone for WorkflowTargetSpec"}} />
+
Clone for WorkflowTargetSpec"}} />
#### `clone` -
clone(&self) -> WorkflowTargetSpec"}} />
+
clone(&self) -> WorkflowTargetSpec"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for WorkflowTargetSpec` -
Debug for WorkflowTargetSpec"}} />
+
Debug for WorkflowTargetSpec"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for WorkflowTargetSpec` @@ -55,7 +55,7 @@ Additive workflow target fields. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for WorkflowTargetSpec` @@ -63,11 +63,11 @@ Additive workflow target fields. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Additive workflow target fields. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for WorkflowTargetSpec` -
PartialEq for WorkflowTargetSpec"}} />
+
PartialEq for WorkflowTargetSpec"}} />
#### `eq` -
eq(&self, other: &WorkflowTargetSpec) -> bool"}} />
+
eq(&self, other: &WorkflowTargetSpec) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for WorkflowTargetSpec` @@ -95,8 +95,8 @@ Additive workflow target fields. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for WorkflowTargetSpec` -
StructuralPartialEq for WorkflowTargetSpec"}} />
+
StructuralPartialEq for WorkflowTargetSpec"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx index d8c0693d4..c00ba879b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx @@ -2,7 +2,7 @@ title: "Enum Doctor Status" sidebar-title: "DoctorStatus" description: "Diagnostic status." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -43,23 +43,23 @@ Check failed. ### `impl Clone for DoctorStatus` -
Clone for DoctorStatus"}} />
+
Clone for DoctorStatus"}} />
#### `clone` -
clone(&self) -> DoctorStatus"}} />
+
clone(&self) -> DoctorStatus"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for DoctorStatus` -
Debug for DoctorStatus"}} />
+
Debug for DoctorStatus"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for DoctorStatus` @@ -67,7 +67,7 @@ Check failed. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for DoctorStatus` @@ -75,11 +75,11 @@ Check failed. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -87,19 +87,19 @@ Check failed. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for DoctorStatus` -
PartialEq for DoctorStatus"}} />
+
PartialEq for DoctorStatus"}} />
#### `eq` -
eq(&self, other: &DoctorStatus) -> bool"}} />
+
eq(&self, other: &DoctorStatus) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for DoctorStatus` @@ -107,16 +107,16 @@ Check failed. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl Copy for DoctorStatus` -
Copy for DoctorStatus"}} />
+
Copy for DoctorStatus"}} />
### `impl Eq for DoctorStatus` -
Eq for DoctorStatus"}} />
+
Eq for DoctorStatus"}} />
### `impl StructuralPartialEq for DoctorStatus` -
StructuralPartialEq for DoctorStatus"}} />
+
StructuralPartialEq for DoctorStatus"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx index bb8aa23e8..b97461157 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx @@ -2,7 +2,7 @@ title: "Function doctor_plan" sidebar-title: "doctor_plan" description: "Inspect a resolved run plan without mutating the environment." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index 6a1929704..e11cf4d12 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 146 +position: 147 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx index c64d5792e..6ce3b87f8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx @@ -2,14 +2,14 @@ title: "Struct Doctor Check" sidebar-title: "DoctorCheck" description: "Diagnostic check result." -position: 74 +position: 75 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub status: DoctorStatus,\n    pub message: String,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub status: DoctorStatus,\n    pub message: String,\n    pub metadata: BTreeMap<String, Value>,\n}"}} />
Diagnostic check result. @@ -35,23 +35,23 @@ Optional structured metadata. ### `impl Clone for DoctorCheck` -
Clone for DoctorCheck"}} />
+
Clone for DoctorCheck"}} />
#### `clone` -
clone(&self) -> DoctorCheck"}} />
+
clone(&self) -> DoctorCheck"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for DoctorCheck` -
Debug for DoctorCheck"}} />
+
Debug for DoctorCheck"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for DoctorCheck` @@ -59,7 +59,7 @@ Optional structured metadata. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for DoctorCheck` @@ -67,11 +67,11 @@ Optional structured metadata. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -79,19 +79,19 @@ Optional structured metadata. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for DoctorCheck` -
PartialEq for DoctorCheck"}} />
+
PartialEq for DoctorCheck"}} />
#### `eq` -
eq(&self, other: &DoctorCheck) -> bool"}} />
+
eq(&self, other: &DoctorCheck) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for DoctorCheck` @@ -99,8 +99,8 @@ Optional structured metadata. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for DoctorCheck` -
StructuralPartialEq for DoctorCheck"}} />
+
StructuralPartialEq for DoctorCheck"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx index c7d65e76f..a7672fc4c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx @@ -2,14 +2,14 @@ title: "Struct Doctor Report" sidebar-title: "DoctorReport" description: "Diagnostic report for a resolved run plan." -position: 75 +position: 76 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub status: DoctorStatus,\n    pub checks: Vec<DoctorCheck>,\n}"}} />
+
String,\n    pub status: DoctorStatus,\n    pub checks: Vec<DoctorCheck>,\n}"}} />
Diagnostic report for a resolved run plan. @@ -31,23 +31,23 @@ Checks. ### `impl Clone for DoctorReport` -
Clone for DoctorReport"}} />
+
Clone for DoctorReport"}} />
#### `clone` -
clone(&self) -> DoctorReport"}} />
+
clone(&self) -> DoctorReport"}} />
#### `clone_from` -
clone_from(&mut self, source: &Self)"}} />
+
clone_from(&mut self, source: &Self)"}} />
### `impl Debug for DoctorReport` -
Debug for DoctorReport"}} />
+
Debug for DoctorReport"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl<'de> Deserialize<'de> for DoctorReport` @@ -55,7 +55,7 @@ Checks. #### `deserialize` -
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
### `impl JsonSchema for DoctorReport` @@ -63,11 +63,11 @@ Checks. #### `schema_name` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `schema_id` -
Cow<'static, str>"}} />
+
Cow<'static, str>"}} />
#### `json_schema` @@ -75,19 +75,19 @@ Checks. #### `inline_schema` -
bool"}} />
+
bool"}} />
### `impl PartialEq for DoctorReport` -
PartialEq for DoctorReport"}} />
+
PartialEq for DoctorReport"}} />
#### `eq` -
eq(&self, other: &DoctorReport) -> bool"}} />
+
eq(&self, other: &DoctorReport) -> bool"}} />
#### `ne` -
ne(&self, other: &Rhs) -> bool"}} />
+
ne(&self, other: &Rhs) -> bool"}} />
### `impl Serialize for DoctorReport` @@ -95,8 +95,8 @@ Checks. #### `serialize` -
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
### `impl StructuralPartialEq for DoctorReport` -
StructuralPartialEq for DoctorReport"}} />
+
StructuralPartialEq for DoctorReport"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 0d0570533..bed56c4a4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -2,14 +2,14 @@ title: "Enum Fabric Error" sidebar-title: "FabricError" description: "Errors raised by NeMo Fabric config loading and validation." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -33,7 +33,7 @@ Underlying path-resolution error. ### `PathNotFound(PathBuf)` -
PathBuf)"}} />
+
PathBuf)"}} />
The requested path does not exist. @@ -755,36 +755,36 @@ Underlying JSON error. ### `impl Debug for FabricError` -
Debug for FabricError"}} />
+
Debug for FabricError"}} />
#### `fmt` -
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
### `impl Display for FabricError` -
Display for FabricError"}} />
+
Display for FabricError"}} />
#### `fmt` -
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
+
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
### `impl Error for FabricError` -
Error for FabricError"}} />
+
Error for FabricError"}} />
#### `source` -
source(&self) -> Option<&(dyn Error + 'static)>"}} />
+
source(&self) -> Option<&(dyn Error + 'static)>"}} />
#### `description` -
description(&self) -> &str"}} />
+
description(&self) -> &str"}} />
#### `cause` -
cause(&self) -> Option<&dyn Error>"}} />
+
cause(&self) -> Option<&dyn Error>"}} />
#### `provide` -
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
+
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index 0e2ed4c29..0cf8cb514 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 147 +position: 148 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx index 0a91b0129..aaf087962 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx @@ -2,14 +2,14 @@ title: "Type Alias Result" sidebar-title: "Result" description: "Core NeMo Fabric result type." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
Result<T, FabricError>;"}} />
+
Result<T, FabricError>;"}} />
Core NeMo Fabric result type. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index f5fedaa31..fb4cfff97 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,13 +2,13 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 155 +position: 157 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
str"}} />
+
str"}} />
Returns the crate version compiled into this build. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 0c4ece762..5a9180fa7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -66,6 +66,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use config::MetadataConfig;` - `pub use config::ModelConfig;` - `pub use config::OAuthTokenEndpointAuthMethod;` +- `pub use config::PricingConfig;` - `pub use config::ResolutionStrategy;` - `pub use config::ResolveContext;` - `pub use config::ResolvedAdapterDescriptor;` @@ -164,7 +165,8 @@ Core config and runtime contract for NeMo Fabric. - [doctor](doctor/index.mdx): Plan diagnostics for NeMo Fabric. - [error](error/index.mdx): Error types for NeMo Fabric core. - [model_usage](model-usage/index.mdx): Invocation-owned normalized counts and separately identified costs. -- [pricing](pricing/index.mdx): Persisted pricing-basis types for the stable model-usage result contract. +- [price_catalog](price-catalog/index.mdx): Model price catalogs for estimating token cost. +- [pricing](pricing/index.mdx): Catalog prices are estimates, separate from provider-reported amounts. - [resource_snapshot](resource-snapshot/index.mdx): Resource snapshots of a runtime's local adapter host and its descendants. - [runtime](runtime/index.mdx): Runtime invocation helpers. - [schema](schema/index.mdx): JSON Schema generation for the public NeMo Fabric contract. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx new file mode 100644 index 000000000..f6e285726 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/fn-normalize-model-usage.mdx @@ -0,0 +1,15 @@ +--- +title: "Function normalize_model_usage" +sidebar-title: "normalize_model_usage" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage/fn-normalize-model-usage" +description: "Normalize supported evidence. Legacy usage remains raw rather than gaining a completeness guarantee merely because a newer core read it." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
AgentUsage,\n    catalog: Option<&UserPriceCatalog>,\n    bundled_pricing_allowed: bool,\n) -> Result<Option<ModelUsage>, AccountingError>"}} />
+ +Normalize supported evidence. Legacy usage remains raw rather than gaining a completeness guarantee merely because a newer core read it. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx index 7176f9683..6685d65d7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/model-usage/index.mdx @@ -3,7 +3,7 @@ title: "Module model_usage" sidebar-title: "model_usage" slug: "/reference/api/rust-library-reference/nemo-fabric-core/model_usage" description: "Invocation-owned normalized counts and separately identified costs." -position: 148 +position: 149 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -24,3 +24,7 @@ Invocation-owned normalized counts and separately identified costs. ## Enums - [UsageDimension](enum-usagedimension.mdx): Dimensions whose complete values can be unavailable independently. + +## Functions + +- [normalize_model_usage](fn-normalize-model-usage.mdx): Normalize supported evidence. Legacy usage remains raw rather than gaining a completeness guarantee merely because a newer core read it. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx new file mode 100644 index 000000000..ded366199 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-cachereadaccounting.mdx @@ -0,0 +1,116 @@ +--- +title: "Enum Cache Read Accounting" +sidebar-title: "CacheReadAccounting" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-cachereadaccounting" +description: "Whether a provider's prompt token count includes cached reads." +position: 7 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum CacheReadAccounting { + IncludedInPromptTokens, + Separate, +} +``` + +Whether a provider's prompt token count includes cached reads. + +## Variants + +### `IncludedInPromptTokens` + +
+ +Prompt tokens include cached reads. + +### `Separate` + +
+ +Prompt tokens exclude cached reads. + +## Trait Implementations + +### `impl Clone for CacheReadAccounting` + +
Clone for CacheReadAccounting"}} />
+ +#### `clone` + +
clone(&self) -> CacheReadAccounting"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for CacheReadAccounting` + +
Debug for CacheReadAccounting"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for CacheReadAccounting` + +
Deserialize<'de> for CacheReadAccounting"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for CacheReadAccounting` + +
CacheReadAccounting"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for CacheReadAccounting` + +
PartialEq for CacheReadAccounting"}} />
+ +#### `eq` + +
eq(&self, other: &CacheReadAccounting) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for CacheReadAccounting` + +
Serialize for CacheReadAccounting"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for CacheReadAccounting` + +
Copy for CacheReadAccounting"}} />
+ +### `impl Eq for CacheReadAccounting` + +
Eq for CacheReadAccounting"}} />
+ +### `impl StructuralPartialEq for CacheReadAccounting` + +
StructuralPartialEq for CacheReadAccounting"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx new file mode 100644 index 000000000..69d57502d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingcatalogerror.mdx @@ -0,0 +1,129 @@ +--- +title: "Enum Pricing Catalog Error" +sidebar-title: "PricingCatalogError" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-pricingcatalogerror" +description: "Why a catalog is invalid. The messages match NeMo Relay's." +position: 8 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Error),\n    UnsupportedVersion {\n        version: u32,\n    },\n    DuplicateModelAlias {\n        model: String,\n    },\n    EmptyField {\n        entry_index: usize,\n        field: String,\n    },\n    InvalidRate {\n        entry_index: usize,\n        field: String,\n        value: f64,\n    },\n}"}} />
+ +Why a catalog is invalid. The messages match NeMo Relay's. + +## Variants + +### `Json(Error)` + +
Error)"}} />
+ +The catalog is not JSON of the catalog format. + +### `UnsupportedVersion` + +
+ +The catalog format version is not 1. + +#### Fields + +##### `version: u32` + +Version in the catalog. + +### `DuplicateModelAlias` + +
+ +Two entries or aliases of one provider name the same model. + +#### Fields + +##### `model: String` + +Normalized `provider/model` key. + +### `EmptyField` + +
+ +A required text field is blank. + +#### Fields + +##### `entry_index: usize` + +Zero-based entry index. + +##### `field: String` + +Field name. + +### `InvalidRate` + +
+ +A rate or a tier bound is out of range. + +#### Fields + +##### `entry_index: usize` + +Zero-based entry index. + +##### `field: String` + +Field name. + +##### `value: f64` + +Invalid value. + +## Trait Implementations + +### `impl Debug for PricingCatalogError` + +
Debug for PricingCatalogError"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Display for PricingCatalogError` + +
Display for PricingCatalogError"}} />
+ +#### `fmt` + +
fmt(&self, __formatter: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Error for PricingCatalogError` + +
Error for PricingCatalogError"}} />
+ +#### `source` + +
source(&self) -> Option<&(dyn Error + 'static)>"}} />
+ +#### `description` + +
description(&self) -> &str"}} />
+ +#### `cause` + +
cause(&self) -> Option<&dyn Error>"}} />
+ +#### `provide` + +
provide<'a>(&'a self, request: &mut Request<'a>)"}} />
+ +### `impl From< Error> for PricingCatalogError` + +
From<Error> for PricingCatalogError"}} />
+ +#### `from` + +
from(source: Error) -> Self"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx new file mode 100644 index 000000000..d4ecc5b7d --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-pricingunit.mdx @@ -0,0 +1,138 @@ +--- +title: "Enum Pricing Unit" +sidebar-title: "PricingUnit" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-pricingunit" +description: "What a model price is charged per." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum PricingUnit { + PerToken, + PerRequest, + PerSecond, + GpuHour, +} +``` + +What a model price is charged per. + +## Variants + +### `PerToken` + +
+ +Per token. + +### `PerRequest` + +
+ +Per request. + +### `PerSecond` + +
+ +Per second. + +### `GpuHour` + +
+ +Per GPU hour, for self-hosted models. + +## Trait Implementations + +### `impl Clone for PricingUnit` + +
Clone for PricingUnit"}} />
+ +#### `clone` + +
clone(&self) -> PricingUnit"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingUnit` + +
Debug for PricingUnit"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for PricingUnit` + +
Default for PricingUnit"}} />
+ +#### `default` + +
default() -> PricingUnit"}} />
+ +### `impl<'de> Deserialize<'de> for PricingUnit` + +
Deserialize<'de> for PricingUnit"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingUnit` + +
PricingUnit"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingUnit` + +
PartialEq for PricingUnit"}} />
+ +#### `eq` + +
eq(&self, other: &PricingUnit) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingUnit` + +
Serialize for PricingUnit"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for PricingUnit` + +
Copy for PricingUnit"}} />
+ +### `impl Eq for PricingUnit` + +
Eq for PricingUnit"}} />
+ +### `impl StructuralPartialEq for PricingUnit` + +
StructuralPartialEq for PricingUnit"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx new file mode 100644 index 000000000..ce7fa57bc --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-ratescheduleapplication.mdx @@ -0,0 +1,117 @@ +--- +title: "Enum Rate Schedule Application" +sidebar-title: "RateScheduleApplication" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-ratescheduleapplication" +description: "How a chosen rate tier applies to a request." +position: 10 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub enum RateScheduleApplication { + FullRequest, +} +``` + +How a chosen rate tier applies to a request. + +## Variants + +### `FullRequest` + +
+ +The tier's rates apply to every token of the request. + +## Trait Implementations + +### `impl Clone for RateScheduleApplication` + +
Clone for RateScheduleApplication"}} />
+ +#### `clone` + +
clone(&self) -> RateScheduleApplication"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RateScheduleApplication` + +
Debug for RateScheduleApplication"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for RateScheduleApplication` + +
Default for RateScheduleApplication"}} />
+ +#### `default` + +
default() -> RateScheduleApplication"}} />
+ +### `impl<'de> Deserialize<'de> for RateScheduleApplication` + +
Deserialize<'de> for RateScheduleApplication"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RateScheduleApplication` + +
RateScheduleApplication"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RateScheduleApplication` + +
PartialEq for RateScheduleApplication"}} />
+ +#### `eq` + +
eq(&self, other: &RateScheduleApplication) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RateScheduleApplication` + +
Serialize for RateScheduleApplication"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for RateScheduleApplication` + +
Copy for RateScheduleApplication"}} />
+ +### `impl Eq for RateScheduleApplication` + +
Eq for RateScheduleApplication"}} />
+ +### `impl StructuralPartialEq for RateScheduleApplication` + +
StructuralPartialEq for RateScheduleApplication"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx new file mode 100644 index 000000000..3fa7398d9 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/enum-tokenrateschedule.mdx @@ -0,0 +1,107 @@ +--- +title: "Enum Token Rate Schedule" +sidebar-title: "TokenRateSchedule" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/enum-tokenrateschedule" +description: "Rates that depend on the size of a request." +position: 11 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
RateScheduleApplication,\n        tiers: Vec<TokenRateTier>,\n    },\n}"}} />
+ +Rates that depend on the size of a request. + +## Variants + +### `PromptTokenThreshold` + +
+ +One tier's rates apply to a whole request, chosen by the request's prompt tokens. + +#### Fields + +##### `applies_to: RateScheduleApplication` + +How the chosen tier's rates apply. + +##### `tiers: Vec` + +Tiers in the order they are tried. The first tier whose bounds contain a request's prompt tokens applies to it. + +## Trait Implementations + +### `impl Clone for TokenRateSchedule` + +
Clone for TokenRateSchedule"}} />
+ +#### `clone` + +
clone(&self) -> TokenRateSchedule"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenRateSchedule` + +
Debug for TokenRateSchedule"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenRateSchedule` + +
Deserialize<'de> for TokenRateSchedule"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenRateSchedule` + +
TokenRateSchedule"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenRateSchedule` + +
PartialEq for TokenRateSchedule"}} />
+ +#### `eq` + +
eq(&self, other: &TokenRateSchedule) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenRateSchedule` + +
Serialize for TokenRateSchedule"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for TokenRateSchedule` + +
StructuralPartialEq for TokenRateSchedule"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx new file mode 100644 index 000000000..c9aa13b29 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx @@ -0,0 +1,34 @@ +--- +title: "Module price_catalog" +sidebar-title: "price_catalog" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog" +description: "Model price catalogs for estimating token cost." +position: 150 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +Model price catalogs for estimating token cost. + +A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, model-only lookup uses the first matching entry in listed order. + +NeMo Fabric ships one catalog, embedded at build time, with the list prices of models that its presets, docs, and examples use. Each entry cites the provider pricing page it was taken from and the date it was checked. A caller can add a catalog of their own, whose entries take precedence. + +## Structs + +- [ModelPricing](struct-modelpricing.mdx): Prices of one model, in `currency` per million tokens. +- [PricingCatalog](struct-pricingcatalog.mdx): A model price catalog in the NeMo Relay model pricing catalog format, version 1. +- [PromptCachePricing](struct-promptcachepricing.mdx): How a provider counts cached prompt tokens in its usage reports. +- [TokenPricingRates](struct-tokenpricingrates.mdx): Token rates in currency units per million tokens. +- [TokenRateTier](struct-tokenratetier.mdx): Rates for requests whose prompt tokens fall within inclusive bounds. +- [UserPriceCatalog](struct-userpricecatalog.mdx): A price catalog the caller supplied, read and validated when the run plan is resolved. + +## Enums + +- [CacheReadAccounting](enum-cachereadaccounting.mdx): Whether a provider's prompt token count includes cached reads. +- [PricingCatalogError](enum-pricingcatalogerror.mdx): Why a catalog is invalid. The messages match NeMo Relay's. +- [PricingUnit](enum-pricingunit.mdx): What a model price is charged per. +- [RateScheduleApplication](enum-ratescheduleapplication.mdx): How a chosen rate tier applies to a request. +- [TokenRateSchedule](enum-tokenrateschedule.mdx): Rates that depend on the size of a request. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx new file mode 100644 index 000000000..814233c4e --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx @@ -0,0 +1,131 @@ +--- +title: "Struct Model Pricing" +sidebar-title: "ModelPricing" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-modelpricing" +description: "Prices of one model, in `currency` per million tokens." +position: 1 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
String,\n    pub model_id: String,\n    pub aliases: Vec<String>,\n    pub currency: String,\n    pub unit: PricingUnit,\n    pub rates: Option<TokenPricingRates>,\n    pub rate_schedule: Option<TokenRateSchedule>,\n    pub prompt_cache: PromptCachePricing,\n    pub pricing_as_of: String,\n    pub pricing_source: String,\n}"}} />
+ +Prices of one model, in `currency` per million tokens. + +## Fields + +### `provider: String` + +Provider that sets the price, such as `anthropic` or `openai`. + +### `model_id: String` + +Canonical model identifier. + +### `aliases: Vec` + +Other identifiers of the same model, such as dated snapshots. + +### `currency: String` + +ISO 4217 currency of the rates. NeMo Fabric estimates cost only from `USD` entries. + +### `unit: PricingUnit` + +What a rate is charged per. NeMo Fabric estimates cost only from `per_token` entries. + +### `rates: Option` + +Flat rates. Ignored when `rate_schedule` is set. + +### `rate_schedule: Option` + +Rates that depend on the number of prompt tokens in a request. + +### `prompt_cache: PromptCachePricing` + +How the provider counts cached prompt tokens. NeMo Relay uses it to split provider usage; NeMo Fabric prices usage that is already normalized, so it does not read it. + +### `pricing_as_of: String` + +Date the price was taken from `pricing_source`, as `YYYY-MM-DD` in the bundled catalog. + +### `pricing_source: String` + +Where the price was taken from, such as the provider's pricing page. + +## Trait Implementations + +### `impl Clone for ModelPricing` + +
Clone for ModelPricing"}} />
+ +#### `clone` + +
clone(&self) -> ModelPricing"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for ModelPricing` + +
Debug for ModelPricing"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for ModelPricing` + +
Deserialize<'de> for ModelPricing"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for ModelPricing` + +
ModelPricing"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for ModelPricing` + +
PartialEq for ModelPricing"}} />
+ +#### `eq` + +
eq(&self, other: &ModelPricing) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for ModelPricing` + +
Serialize for ModelPricing"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for ModelPricing` + +
StructuralPartialEq for ModelPricing"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx new file mode 100644 index 000000000..01d5d4765 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx @@ -0,0 +1,113 @@ +--- +title: "Struct Pricing Catalog" +sidebar-title: "PricingCatalog" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-pricingcatalog" +description: "A model price catalog in the NeMo Relay model pricing catalog format, version 1." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
u32,\n    pub entries: Vec<ModelPricing>,\n}"}} />
+ +A model price catalog in the NeMo Relay model pricing catalog format, version 1. + +NeMo Fabric reads and writes the format unchanged, with one extra check: every rate must be below one million US dollars per million tokens. + +## Fields + +### `version: u32` + +Catalog format version. Must be `1`. + +### `entries: Vec` + +Price entries. Lookup uses the first entry that matches. + +## Implementations + +### `impl PricingCatalog` + +
PricingCatalog"}} />
+ +#### `from_json` + +
u8]) -> Result<Self, PricingCatalogError>"}} />
+ +Parse and validate a catalog. + +## Trait Implementations + +### `impl Clone for PricingCatalog` + +
Clone for PricingCatalog"}} />
+ +#### `clone` + +
clone(&self) -> PricingCatalog"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PricingCatalog` + +
Debug for PricingCatalog"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PricingCatalog` + +
Deserialize<'de> for PricingCatalog"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PricingCatalog` + +
PricingCatalog"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PricingCatalog` + +
PartialEq for PricingCatalog"}} />
+ +#### `eq` + +
eq(&self, other: &PricingCatalog) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PricingCatalog` + +
Serialize for PricingCatalog"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for PricingCatalog` + +
StructuralPartialEq for PricingCatalog"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx new file mode 100644 index 000000000..423b73286 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-promptcachepricing.mdx @@ -0,0 +1,103 @@ +--- +title: "Struct Prompt Cache Pricing" +sidebar-title: "PromptCachePricing" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-promptcachepricing" +description: "How a provider counts cached prompt tokens in its usage reports." +position: 3 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
CacheReadAccounting,\n}"}} />
+ +How a provider counts cached prompt tokens in its usage reports. + +## Fields + +### `read_accounting: CacheReadAccounting` + +Whether the provider's prompt token count includes cached reads. + +## Trait Implementations + +### `impl Clone for PromptCachePricing` + +
Clone for PromptCachePricing"}} />
+ +#### `clone` + +
clone(&self) -> PromptCachePricing"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for PromptCachePricing` + +
Debug for PromptCachePricing"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for PromptCachePricing` + +
Deserialize<'de> for PromptCachePricing"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for PromptCachePricing` + +
PromptCachePricing"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for PromptCachePricing` + +
PartialEq for PromptCachePricing"}} />
+ +#### `eq` + +
eq(&self, other: &PromptCachePricing) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for PromptCachePricing` + +
Serialize for PromptCachePricing"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for PromptCachePricing` + +
Copy for PromptCachePricing"}} />
+ +### `impl Eq for PromptCachePricing` + +
Eq for PromptCachePricing"}} />
+ +### `impl StructuralPartialEq for PromptCachePricing` + +
StructuralPartialEq for PromptCachePricing"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx new file mode 100644 index 000000000..8c6ea1f0b --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenpricingrates.mdx @@ -0,0 +1,111 @@ +--- +title: "Struct Token Pricing Rates" +sidebar-title: "TokenPricingRates" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-tokenpricingrates" +description: "Token rates in currency units per million tokens." +position: 4 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
f64,\n    pub output_per_million: f64,\n    pub cache_read_per_million: Option<f64>,\n    pub cache_write_per_million: Option<f64>,\n}"}} />
+ +Token rates in currency units per million tokens. + +## Fields + +### `input_per_million: f64` + +Rate for input tokens that were neither read from nor written to the prompt cache. + +### `output_per_million: f64` + +Rate for output tokens, including reasoning tokens. + +### `cache_read_per_million: Option` + +Rate for input tokens read from the prompt cache. Without it, cached reads cannot be priced; set `0.0` when they are free. + +### `cache_write_per_million: Option` + +Rate for input tokens written to the prompt cache. Without it, cache writes cannot be priced; set `0.0` when they are free. + +## Trait Implementations + +### `impl Clone for TokenPricingRates` + +
Clone for TokenPricingRates"}} />
+ +#### `clone` + +
clone(&self) -> TokenPricingRates"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenPricingRates` + +
Debug for TokenPricingRates"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenPricingRates` + +
Deserialize<'de> for TokenPricingRates"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenPricingRates` + +
TokenPricingRates"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenPricingRates` + +
PartialEq for TokenPricingRates"}} />
+ +#### `eq` + +
eq(&self, other: &TokenPricingRates) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenPricingRates` + +
Serialize for TokenPricingRates"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for TokenPricingRates` + +
Copy for TokenPricingRates"}} />
+ +### `impl StructuralPartialEq for TokenPricingRates` + +
StructuralPartialEq for TokenPricingRates"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx new file mode 100644 index 000000000..4ff3cf607 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-tokenratetier.mdx @@ -0,0 +1,107 @@ +--- +title: "Struct Token Rate Tier" +sidebar-title: "TokenRateTier" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-tokenratetier" +description: "Rates for requests whose prompt tokens fall within inclusive bounds." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
Option<u64>,\n    pub max_prompt_tokens: Option<u64>,\n    pub rates: TokenPricingRates,\n}"}} />
+ +Rates for requests whose prompt tokens fall within inclusive bounds. + +## Fields + +### `min_prompt_tokens: Option` + +Smallest number of prompt tokens in the tier, or no lower bound. + +### `max_prompt_tokens: Option` + +Largest number of prompt tokens in the tier, or no upper bound. + +### `rates: TokenPricingRates` + +Rates of the tier. + +## Trait Implementations + +### `impl Clone for TokenRateTier` + +
Clone for TokenRateTier"}} />
+ +#### `clone` + +
clone(&self) -> TokenRateTier"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for TokenRateTier` + +
Debug for TokenRateTier"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for TokenRateTier` + +
Deserialize<'de> for TokenRateTier"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for TokenRateTier` + +
TokenRateTier"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for TokenRateTier` + +
PartialEq for TokenRateTier"}} />
+ +#### `eq` + +
eq(&self, other: &TokenRateTier) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for TokenRateTier` + +
Serialize for TokenRateTier"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Copy for TokenRateTier` + +
Copy for TokenRateTier"}} />
+ +### `impl StructuralPartialEq for TokenRateTier` + +
StructuralPartialEq for TokenRateTier"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx new file mode 100644 index 000000000..63d5f8211 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-userpricecatalog.mdx @@ -0,0 +1,115 @@ +--- +title: "Struct User Price Catalog" +sidebar-title: "UserPriceCatalog" +slug: "/reference/api/rust-library-reference/nemo-fabric-core/price_catalog/struct-userpricecatalog" +description: "A price catalog the caller supplied, read and validated when the run plan is resolved." +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
PathBuf,\n    pub sha256: String,\n    pub catalog: PricingCatalog,\n}"}} />
+ +A price catalog the caller supplied, read and validated when the run plan is resolved. + +## Fields + +### `path: PathBuf` + +Absolute path the catalog was read from. + +### `sha256: String` + +`sha256:` followed by the lowercase hexadecimal SHA-256 of the file. + +### `catalog: PricingCatalog` + +The validated catalog. + +## Implementations + +### `impl UserPriceCatalog` + +
UserPriceCatalog"}} />
+ +#### `from_bytes` + +
PathBuf,\n    bytes: &[u8],\n) -> Result<Self, PricingCatalogError>"}} />
+ +Validate bytes and retain their exact catalog fingerprint. + +## Trait Implementations + +### `impl Clone for UserPriceCatalog` + +
Clone for UserPriceCatalog"}} />
+ +#### `clone` + +
clone(&self) -> UserPriceCatalog"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for UserPriceCatalog` + +
Debug for UserPriceCatalog"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl<'de> Deserialize<'de> for UserPriceCatalog` + +
Deserialize<'de> for UserPriceCatalog"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for UserPriceCatalog` + +
UserPriceCatalog"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for UserPriceCatalog` + +
PartialEq for UserPriceCatalog"}} />
+ +#### `eq` + +
eq(&self, other: &UserPriceCatalog) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for UserPriceCatalog` + +
Serialize for UserPriceCatalog"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl StructuralPartialEq for UserPriceCatalog` + +
StructuralPartialEq for UserPriceCatalog"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx index a62a00c14..4a1105670 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/pricing/index.mdx @@ -1,15 +1,15 @@ --- title: "Module pricing" sidebar-title: "pricing" -description: "Persisted pricing-basis types for the stable model-usage result contract." -position: 149 +description: "Catalog prices are estimates, separate from provider-reported amounts." +position: 151 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -Persisted pricing-basis types for the stable model-usage result contract. +Catalog prices are estimates, separate from provider-reported amounts. ## Structs diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index 931c9f916..c0e90786d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -3,7 +3,7 @@ title: "Module resource_snapshot" sidebar-title: "resource_snapshot" slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" description: "Resource snapshots of a runtime's local adapter host and its descendants." -position: 150 +position: 152 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 12cea418d..2ff453e3b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 151 +position: 153 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index a194d1995..08a1933e1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 152 +position: 154 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx index 9356d0466..ca7d260a5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx @@ -3,7 +3,7 @@ title: "Module system_stats" sidebar-title: "system_stats" slug: "/reference/api/rust-library-reference/nemo-fabric-core/system_stats" description: "Invocation measurements collected by NeMo Fabric in the executing runtime." -position: 153 +position: 155 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx index df06788a3..a942c30e8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/usage-accounting/index.mdx @@ -3,7 +3,7 @@ title: "Module usage_accounting" sidebar-title: "usage_accounting" slug: "/reference/api/rust-library-reference/nemo-fabric-core/usage_accounting" description: "Validated accounting evidence carried through the existing usage extension." -position: 154 +position: 156 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index d337ed45c..88b07bc25 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1155,7 +1155,7 @@ Important fields: | `status` | Terminal invocation status such as success, failure, or cancellation. | | `output` | Harness output normalized to the configured output schema. | | `error` | Structured failure metadata when available. | -| `model_usage` | Optional normalized complete counts, observations, gaps and provider-reported cost. Catalog estimates remain unavailable. | +| `model_usage` | Optional normalized complete counts, observations, gaps and separately reported/estimated cost. | | `usage` | Optional `RunUsage` with invocation-local input, cached-input, output, and total token counts, reported cost in USD, and provider metadata. | | `system_stats` | Optional `InvocationSystemStats`: monotonic adapter exchange duration, before/after observations with availability, and an optional CPU counter difference estimate. | | `resource_snapshot` | Optional `RuntimeResourceSnapshot` of the runtime, read once after the adapter returned the result. | @@ -1212,7 +1212,7 @@ if result.model_usage is not None: print(result.model_usage.input_tokens) # Complete count, or None. print(result.model_usage.observed.input_tokens) # Available observations. print(result.model_usage.gaps) # Affected dimensions/reasons. - print(result.model_usage.cost.reported.amount_nano_usd) + print(result.model_usage.cost.estimated.amount_nano_usd) summary = summarize_usage( [result], expected=[(result.runtime_id, result.invocation_id)] @@ -1231,10 +1231,16 @@ extensions still fail the adapter contract. Legacy usage without supported accounting remains available in `result.usage` and is not certified complete by `summarize_usage`. -`cost.reported` contains only provider-reported amounts. Complete and observed -amounts use integer nano-USD. Catalog estimates remain unavailable in this slice; -`cost.estimated` has no amounts and no typed pricing setting is exposed. The stable -result shape retains optional pricing-basis fields for persisted evidence. +`cost.reported` contains only provider-reported amounts; `cost.estimated` contains +catalog estimates. Both distinguish `amount_nano_usd` from `observed_nano_usd`. +Estimates retain their pricing basis and distinct selected rate snapshots. +Configure `PricingConfig(catalog_path="prices.json")` for a Relay-compatible +catalog override; planning freezes its bytes. An explicitly established provider must match the catalog entry's provider, including aliases and shorter model paths. A different provider's entry cannot price that source solely because the model name matches. When no provider identity is available, an explicit user catalog can match the model name alone. Bundled provider prices apply only +when the configured route establishes the native provider. Unknown model, +cache-price or request-tier evidence affects estimates without erasing tokens. +Nonzero Anthropic cache writes currently lack expiry-price evidence, so their +complete estimate remains unknown; independently priceable parts remain observed. +Known zero writes need no expiry evidence. Estimates are not invoices. `summarize_usage` requires an explicit expected invocation inventory. It retains missing results in coverage, keeps complete totals separate from observed sums, diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index cfd2150de..2cb00c651 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -558,6 +558,20 @@ } ] }, + "PricingConfig": { + "additionalProperties": true, + "description": "Central token-price configuration; never changes provider requests.", + "properties": { + "catalog_path": { + "description": "Relay-compatible v1 catalog, resolved relative to the config base directory.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -1611,6 +1625,17 @@ "description": "Named model roles.", "type": "object" }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/PricingConfig" + }, + { + "type": "null" + } + ], + "description": "Optional catalog override for central token-price estimates." + }, "relay": { "anyOf": [ { diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index e183bca75..ec8bcc2c7 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -894,6 +894,21 @@ ], "type": "object" }, + "CacheReadAccounting": { + "description": "Whether a provider's prompt token count includes cached reads.", + "oneOf": [ + { + "const": "included_in_prompt_tokens", + "description": "Prompt tokens include cached reads.", + "type": "string" + }, + { + "const": "separate", + "description": "Prompt tokens exclude cached reads.", + "type": "string" + } + ] + }, "CapabilityKind": { "description": "Capability kind.", "oneOf": [ @@ -1341,6 +1356,17 @@ "description": "Named model roles.", "type": "object" }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/PricingConfig" + }, + { + "type": "null" + } + ], + "description": "Optional catalog override for central token-price estimates." + }, "relay": { "anyOf": [ { @@ -1925,6 +1951,80 @@ ], "type": "object" }, + "ModelPricing": { + "additionalProperties": false, + "description": "Prices of one model, in `currency` per million tokens.", + "properties": { + "aliases": { + "default": [], + "description": "Other identifiers of the same model, such as dated snapshots.", + "items": { + "type": "string" + }, + "type": "array" + }, + "currency": { + "default": "USD", + "description": "ISO 4217 currency of the rates. NeMo Fabric estimates cost only from\n`USD` entries.", + "type": "string" + }, + "model_id": { + "description": "Canonical model identifier.", + "type": "string" + }, + "pricing_as_of": { + "description": "Date the price was taken from `pricing_source`, as `YYYY-MM-DD` in\nthe bundled catalog.", + "type": "string" + }, + "pricing_source": { + "description": "Where the price was taken from, such as the provider's pricing page.", + "type": "string" + }, + "prompt_cache": { + "$ref": "#/$defs/PromptCachePricing", + "description": "How the provider counts cached prompt tokens. NeMo Relay uses it to\nsplit provider usage; NeMo Fabric prices usage that is already\nnormalized, so it does not read it." + }, + "provider": { + "description": "Provider that sets the price, such as `anthropic` or `openai`.", + "type": "string" + }, + "rate_schedule": { + "anyOf": [ + { + "$ref": "#/$defs/TokenRateSchedule" + }, + { + "type": "null" + } + ], + "description": "Rates that depend on the number of prompt tokens in a request." + }, + "rates": { + "anyOf": [ + { + "$ref": "#/$defs/TokenPricingRates" + }, + { + "type": "null" + } + ], + "description": "Flat rates. Ignored when `rate_schedule` is set." + }, + "unit": { + "$ref": "#/$defs/PricingUnit", + "default": "per_token", + "description": "What a rate is charged per. NeMo Fabric estimates cost only from\n`per_token` entries." + } + }, + "required": [ + "provider", + "model_id", + "prompt_cache", + "pricing_as_of", + "pricing_source" + ], + "type": "object" + }, "OAuthTokenEndpointAuthMethod": { "description": "OAuth client authentication method used at the token endpoint.", "oneOf": [ @@ -1945,6 +2045,93 @@ } ] }, + "PricingCatalog": { + "additionalProperties": false, + "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", + "properties": { + "entries": { + "description": "Price entries. Lookup uses the first entry that matches.", + "items": { + "$ref": "#/$defs/ModelPricing" + }, + "type": "array" + }, + "version": { + "description": "Catalog format version. Must be `1`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "version", + "entries" + ], + "type": "object" + }, + "PricingConfig": { + "additionalProperties": true, + "description": "Central token-price configuration; never changes provider requests.", + "properties": { + "catalog_path": { + "description": "Relay-compatible v1 catalog, resolved relative to the config base directory.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "PricingUnit": { + "description": "What a model price is charged per.", + "oneOf": [ + { + "const": "per_token", + "description": "Per token.", + "type": "string" + }, + { + "const": "per_request", + "description": "Per request.", + "type": "string" + }, + { + "const": "per_second", + "description": "Per second.", + "type": "string" + }, + { + "const": "gpu_hour", + "description": "Per GPU hour, for self-hosted models.", + "type": "string" + } + ] + }, + "PromptCachePricing": { + "additionalProperties": false, + "description": "How a provider counts cached prompt tokens in its usage reports.", + "properties": { + "read_accounting": { + "$ref": "#/$defs/CacheReadAccounting", + "description": "Whether the provider's prompt token count includes cached reads." + } + }, + "required": [ + "read_accounting" + ], + "type": "object" + }, + "RateScheduleApplication": { + "description": "How a chosen rate tier applies to a request.", + "oneOf": [ + { + "const": "full_request", + "description": "The tier's rates apply to every token of the request.", + "type": "string" + } + ] + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -2985,6 +3172,107 @@ }, "type": "object" }, + "TokenPricingRates": { + "additionalProperties": false, + "description": "Token rates in currency units per million tokens.", + "properties": { + "cache_read_per_million": { + "description": "Rate for input tokens read from the prompt cache. Without it, cached\nreads cannot be priced; set `0.0` when they are free.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "cache_write_per_million": { + "description": "Rate for input tokens written to the prompt cache. Without it, cache\nwrites cannot be priced; set `0.0` when they are free.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "input_per_million": { + "description": "Rate for input tokens that were neither read from nor written to the\nprompt cache.", + "format": "double", + "type": "number" + }, + "output_per_million": { + "description": "Rate for output tokens, including reasoning tokens.", + "format": "double", + "type": "number" + } + }, + "required": [ + "input_per_million", + "output_per_million" + ], + "type": "object" + }, + "TokenRateSchedule": { + "description": "Rates that depend on the size of a request.", + "oneOf": [ + { + "additionalProperties": false, + "description": "One tier's rates apply to a whole request, chosen by the request's\nprompt tokens.", + "properties": { + "applies_to": { + "$ref": "#/$defs/RateScheduleApplication", + "default": "full_request", + "description": "How the chosen tier's rates apply." + }, + "tiers": { + "description": "Tiers in the order they are tried. The first tier whose bounds\ncontain a request's prompt tokens applies to it.", + "items": { + "$ref": "#/$defs/TokenRateTier" + }, + "type": "array" + }, + "type": { + "const": "prompt_token_threshold", + "type": "string" + } + }, + "required": [ + "type", + "tiers" + ], + "type": "object" + } + ] + }, + "TokenRateTier": { + "additionalProperties": false, + "description": "Rates for requests whose prompt tokens fall within inclusive bounds.", + "properties": { + "max_prompt_tokens": { + "description": "Largest number of prompt tokens in the tier, or no upper bound.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "min_prompt_tokens": { + "description": "Smallest number of prompt tokens in the tier, or no lower bound.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "rates": { + "$ref": "#/$defs/TokenPricingRates", + "description": "Rates of the tier." + } + }, + "required": [ + "rates" + ], + "type": "object" + }, "ToolDefinitionConfig": { "additionalProperties": true, "description": "One named normalized tool or tool-group definition.", @@ -3074,6 +3362,29 @@ }, "type": "object" }, + "UserPriceCatalog": { + "description": "A price catalog the caller supplied, read and validated when the run plan\nis resolved.", + "properties": { + "catalog": { + "$ref": "#/$defs/PricingCatalog", + "description": "The validated catalog." + }, + "path": { + "description": "Absolute path the catalog was read from.", + "type": "string" + }, + "sha256": { + "description": "`sha256:` followed by the lowercase hexadecimal SHA-256 of the file.", + "type": "string" + } + }, + "required": [ + "path", + "sha256", + "catalog" + ], + "type": "object" + }, "WorkflowConfig": { "additionalProperties": true, "description": "Registered workflow target and immutable construction settings.", @@ -3235,6 +3546,17 @@ } ], "description": "Resolved telemetry pass-through plan." + }, + "user_price_catalog": { + "anyOf": [ + { + "$ref": "#/$defs/UserPriceCatalog" + }, + { + "type": "null" + } + ], + "description": "Catalog bytes are frozen during planning." } }, "required": [ diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index 9d817384f..f19f256ee 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -4,6 +4,7 @@ """Python SDK surface for NeMo Fabric.""" from nemo_fabric.accounting import summarize_usage +from nemo_fabric.models import PricingConfig from nemo_fabric.types import ModelUsage from nemo_fabric.types import PerModelUsage from nemo_fabric.types import PricingBasis @@ -93,6 +94,7 @@ __all__ = [ "summarize_usage", + "PricingConfig", "ModelUsage", "PerModelUsage", "PricingBasis", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index 7a9bbd627..fa3f6c6ce 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -1118,6 +1118,19 @@ def _validate_collect(cls, value: list[str]) -> list[str]: return value +class PricingConfig(FabricBaseModel): + """Frozen Relay-compatible price-catalog override, read during planning.""" + + catalog_path: str | Path | None = None + + @field_validator("catalog_path") + @classmethod + def _validate_catalog_path(cls, value: str | Path | None) -> str | Path | None: + if value is not None and not str(value).strip(): + raise ValueError("pricing catalog path must not be empty") + return value + + class FabricConfig(FabricBaseModel): """SDK-facing typed NeMo Fabric agent configuration. @@ -1141,6 +1154,7 @@ class FabricConfig(FabricBaseModel): system_stats: SystemStatsConfig | None = Field(default_factory=SystemStatsConfig) telemetry: TelemetryConfig | None = None relay: RelayConfig | None = None + pricing: PricingConfig | None = None tools: ToolsConfig | None = None @model_serializer(mode="wrap") diff --git a/tests/python/conftest.py b/tests/python/conftest.py index ba0e484c1..40a104a58 100644 --- a/tests/python/conftest.py +++ b/tests/python/conftest.py @@ -6,11 +6,16 @@ from pathlib import Path import pytest -from nemo_fabric import Fabric, FabricConfig +from nemo_fabric import Fabric, FabricConfig, PricingConfig + + +@pytest.fixture(name="accounting_catalog") +def accounting_catalog_fixture(): + return None @pytest.fixture(name="accounting_runtime") -async def accounting_runtime_fixture(tmp_path): +async def accounting_runtime_fixture(tmp_path, accounting_catalog): """Normalize controlled adapter evidence through the native public runtime.""" root = Path(__file__).resolve().parents[2] accounting_schema = json.loads( @@ -68,6 +73,10 @@ async def accounting_runtime_fixture(tmp_path): "runtime": {"artifacts": "./artifacts"}, } ) + if accounting_catalog is not None: + catalog_path = tmp_path / "catalog.json" + catalog_path.write_text(json.dumps(accounting_catalog)) + config.pricing = PricingConfig(catalog_path=catalog_path) async with await Fabric().start_runtime(config, base_dir=tmp_path) as runtime: yield runtime diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py index c305b4b33..dca8697bb 100644 --- a/tests/python/test_usage_reducer_parity.py +++ b/tests/python/test_usage_reducer_parity.py @@ -104,8 +104,39 @@ async def test_actual_calls_with_mixed_conventions_recover_complete_counts_in_co @pytest.mark.integration -async def test_scoped_request_gap_preserves_cache_uncertainty_in_core( - reduced, accounting_runtime +@pytest.mark.parametrize( + "accounting_catalog, complete_cost", + [ + ( + { + "version": 1, + "entries": [ + { + "provider": "openai", + "model_id": "model", + "currency": "USD", + "unit": "per_token", + "rates": { + "input_per_million": 1.0, + "output_per_million": 2.0, + "cache_read_per_million": cache_rate, + "cache_write_per_million": 1.25, + }, + "prompt_cache": { + "read_accounting": "included_in_prompt_tokens" + }, + "pricing_as_of": "2026-10-21", + "pricing_source": "test fixture", + } + ], + }, + complete_cost, + ) + for cache_rate, complete_cost in [(0.1, None), (1.0, 110_000)] + ], +) +async def test_scoped_request_gap_preserves_cache_uncertainty_and_independent_pricing( + reduced, accounting_runtime, complete_cost ): index = next( i @@ -132,3 +163,4 @@ async def test_scoped_request_gap_preserves_cache_uncertainty_in_core( assert usage.cached_input_tokens is None assert usage.observed.cached_input_tokens == 20 assert usage.models[0].cached_input_tokens is None + assert usage.cost.estimated.amount_nano_usd == complete_cost From cbb06993ec9db38d0b7e2dfdd072c94eecfd16ae Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 14:59:52 -0700 Subject: [PATCH 114/159] fix: validate recovered usage and optional pricing rates Signed-off-by: mnajafian-nv --- .../generated/usage-accounting-vectors.ts | 180 ++++++++++++++++++ crates/fabric-core/src/runtime.rs | 7 +- crates/fabric-core/src/usage_normalization.rs | 23 +++ docs/adapter-contract/tutorials/results.md | 2 +- .../nemo_fabric.types.md | 1 + docs/sdk/python.mdx | 3 - schemas/SCHEMA.md | 9 +- .../src/nemo_fabric/types.py | 7 +- tests/adapters/test_deepagents_accounting.py | 4 +- tests/docs/test_python_api_docs.py | 4 +- .../usage_accounting/conformance.json | 180 ++++++++++++++++++ tests/python/test_model_usage.py | 22 ++- 12 files changed, 426 insertions(+), 16 deletions(-) diff --git a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts index 05a913834..cbc90c193 100644 --- a/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts +++ b/adapter-contract/typescript/test/generated/usage-accounting-vectors.ts @@ -493,6 +493,186 @@ export const accountingVectors = [ "observed": 20 } } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } } ] satisfies { name: string; diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 2b7a780b4..b03484eee 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -2324,7 +2324,7 @@ fn run_local_host_invocation_with_timeout( let model_usage = match agent_result .usage .as_ref() - .map(|raw| crate::model_usage::normalize_model_usage(raw)) + .map(crate::model_usage::normalize_model_usage) .transpose() { Ok(usage) => usage.flatten(), @@ -6114,7 +6114,7 @@ for line in sys.stdin: #[test] fn local_host_returns_flat_supported_accounting_with_the_normalized_result() { - let (_root, plan) = local_host_plan("accounted_usage_result"); + let (root, plan) = local_host_plan("accounted_usage_result"); let result = run_plan(&plan, RunRequest::text("accounted")).expect("normalized result"); assert_eq!(result.status, RunStatus::Succeeded); let usage = result.model_usage.expect("supported accounting"); @@ -6129,6 +6129,7 @@ for line in sys.stdin: assert_eq!(usage.cost.estimated.observed_nano_usd, None); assert!(usage.cost.pricing.is_none()); assert!(usage.cost.applied_rates.is_empty()); + let _ = fs::remove_dir_all(root); } #[test] @@ -6688,7 +6689,7 @@ for line in sys.stdin: let diagnostic = serde_json::to_string(events[0]).unwrap(); assert!(!diagnostic.contains("private-accounting-identity")); assert!(!diagnostic.contains("request totals")); - let _ = root; + let _ = fs::remove_dir_all(root); } } } diff --git a/crates/fabric-core/src/usage_normalization.rs b/crates/fabric-core/src/usage_normalization.rs index f236d7623..f8924207e 100644 --- a/crates/fabric-core/src/usage_normalization.rs +++ b/crates/fabric-core/src/usage_normalization.rs @@ -348,6 +348,7 @@ pub fn describe_model_usage(usage: &AgentUsage) -> Result Result Result input) || peak.is_ok_and(|peak| peak > input) + }) { + counts[INPUT] = Err("inconsistent_usage"); + observed[INPUT] = Err("inconsistent_usage"); + } + let reasoning = counts[UsageDimension::ReasoningTokens as usize]; + if counts[OUTPUT].is_ok_and(|output| reasoning.is_ok_and(|reasoning| reasoning > output)) { + counts[OUTPUT] = Err("inconsistent_usage"); + observed[OUTPUT] = Err("inconsistent_usage"); + } +} + // An unconvertible aggregate must not erase a qualified model/request subset. // Contradictions and overflow stay errors; these reasons only lack a usable // normalized observation. Recovery never certifies a complete invocation. diff --git a/docs/adapter-contract/tutorials/results.md b/docs/adapter-contract/tutorials/results.md index 42cd74b99..f361e00ba 100644 --- a/docs/adapter-contract/tutorials/results.md +++ b/docs/adapter-contract/tutorials/results.md @@ -108,7 +108,7 @@ A model aggregate is not a single request. No adapter should fabricate call boundaries, hide execution features or change retry settings to make usage look complete. Costs computed from a price table never become `cost_usd`. -## 2. Separate Failure Classes +### 2. Separate Failure Classes Use these failure classes consistently: diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 57ba50589..83cdfab2b 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -2634,6 +2634,7 @@ The model is both attribute-accessible and mapping-compatible. A harness failure - `output`: Object-shaped adapter output as ``RunOutput``; non-object values are preserved as-is. - `error`: Structured failure, or ``None`` on success. - `usage`: Normalized invocation usage, or ``None`` when unavailable. + - `model_usage`: Complete counts, observations and accounting gaps, or ``None`` when supported accounting evidence is unavailable. - `resource_snapshot`: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. - `system_stats`: Invocation duration and endpoint observations, or None for older results or execution backends without collection. - `artifacts`: Normalized artifact manifest. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index d337ed45c..f55fe8ba9 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1194,9 +1194,6 @@ the SDK raises a `FabricError` subclass instead of returning a partial ### Complete counts and useful observations -The bundled Deep Agents adapter supplies this evidence for its documented collection scope. - - `result.model_usage` is present when the adapter supplies the declared `nemo_fabric.usage_accounting.v1` evidence. Its token fields are independently nullable counts: input includes cache reads and writes, output includes diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 6e90b6cec..8f43fc0a8 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -26,6 +26,8 @@ schemas/ │ ├── agent.schema.json │ ├── run-plan.schema.json │ └── ... +├── extensions/ # Descriptor-declared extension payloads +│ └── usage-accounting.schema.json └── adapter-contract/ # Southbound adapter-facing contract ├── adapter-descriptor.schema.json ├── adapter-target-descriptor.schema.json @@ -38,9 +40,10 @@ schemas/ └── openai-stream-record.schema.json ``` -An adapter author can treat `adapter-contract/` as the complete schema entry -point. Contract versions remain explicit fields in the wire format rather than -directory names. +`adapter-contract/` defines the base adapter wire objects. Adapters returning +supported extension payloads must also declare the applicable schema from +`extensions/`. Contract versions remain explicit fields in the wire format +rather than directory names. Start with the [adapter contract overview](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/README.md) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 72b8ed64d..bea167f4a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2587,8 +2587,11 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if re.fullmatch(r"sha256:[0-9a-f]{64}", data["catalog_sha256"]) is None: raise FabricConfigError("invalid applied pricing fingerprint") rates = _mapping(data["rates"], "pricing rates") - if set(rates) != {"input", "output", "cache_read", "cache_write"}: + allowed = {"input", "output", "cache_read", "cache_write"} + if not {"input", "output"} <= set(rates) <= allowed: raise FabricConfigError("incomplete pricing rates") + rates.setdefault("cache_read", None) + rates.setdefault("cache_write", None) for key, value in rates.items(): if value is not None or key in {"input", "output"}: _usage_counter(value, "pricing rate") @@ -2902,6 +2905,8 @@ class RunResult(FabricMapping): are preserved as-is. error: Structured failure, or ``None`` on success. usage: Normalized invocation usage, or ``None`` when unavailable. + model_usage: Complete counts, observations and accounting gaps, or + ``None`` when supported accounting evidence is unavailable. resource_snapshot: Runtime resource snapshot read once after the adapter returned this result, or ``None`` when the runtime's resources could not be observed. diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py index 5ba450330..dccaf29a3 100644 --- a/tests/adapters/test_deepagents_accounting.py +++ b/tests/adapters/test_deepagents_accounting.py @@ -365,8 +365,8 @@ async def test_non_model_failure_keeps_completed_call_usage(make_model): @pytest.fixture(name="make_runtime") async def make_runtime_fixture(tmp_path, monkeypatch): runtimes = [] - os.environ["SCRIPTED_MODEL_KEY"] = "unused-scripted" - os.environ.pop("OPENAI_PROXY", None) + monkeypatch.setenv("SCRIPTED_MODEL_KEY", "unused-scripted") + monkeypatch.delenv("OPENAI_PROXY", raising=False) async def make(replies, *, identity="runtime", profile=None, barrier=None): ledger = [] diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index 73a2c51ac..6fbcbfec2 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -177,7 +177,7 @@ def test_python_reference_exactly_covers_exported_sdk_classes() -> None: for module, names in functions.items(): page = REFERENCE_DIR / f"{module}.md" documented = { - match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text()) + match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } assert names <= documented assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") @@ -274,7 +274,7 @@ def test_generated_signatures_preserve_python_callable_semantics(): page = REFERENCE_DIR / f"{module_name}.md" rendered = { match.group(1): match.group(2) - for match in FUNCTION_SIGNATURE.finditer(page.read_text()) + for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } for function_name in function_names: function = getattr(module, function_name) diff --git a/tests/fixtures/usage_accounting/conformance.json b/tests/fixtures/usage_accounting/conformance.json index dc9d5cf47..b5e026d61 100644 --- a/tests/fixtures/usage_accounting/conformance.json +++ b/tests/fixtures/usage_accounting/conformance.json @@ -484,5 +484,185 @@ "observed": 20 } } + }, + { + "name": "aggregate_recovered_cache_cannot_exceed_complete_input", + "usage": { + "input_tokens": 100, + "input_tokens_include_cache": true + }, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "output_tokens_include_reasoning": true + } + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_cache_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "cache-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "cached_input_tokens": 200, + "cache_write_input_tokens": 0 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "cached_input_tokens": { + "complete": 200, + "observed": 200 + }, + "cache_write_input_tokens": { + "complete": 0, + "observed": 0 + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_peak_cannot_exceed_complete_input", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "peak-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "peak_request_input_tokens": 200 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": null, + "observed": null + }, + "output_tokens": { + "complete": 5, + "observed": 5 + }, + "total_tokens": { + "complete": null, + "observed": null + } + } + }, + { + "name": "model_recovered_request_reasoning_cannot_exceed_complete_output", + "usage": {}, + "accounting": { + "models_exhaustive": true, + "gaps": [], + "models": [ + { + "model": "reasoning-model", + "gaps": [], + "reported": { + "input_tokens": 100, + "input_tokens_include_cache": true, + "output_tokens": 5, + "output_tokens_include_reasoning": true + }, + "requests_exhaustive": true, + "requests": [ + { + "usage": { + "reasoning_tokens": 10 + } + } + ] + } + ] + }, + "expected": { + "input_tokens": { + "complete": 100, + "observed": 100 + }, + "output_tokens": { + "complete": null, + "observed": null + }, + "total_tokens": { + "complete": null, + "observed": null + } + } } ] diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 93d31dd63..fb018ffaa 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -10,7 +10,7 @@ from nemo_fabric.accounting import summarize_usage from nemo_fabric.errors import FabricConfigError -from nemo_fabric.types import ModelUsage, PerModelUsage, TokenCounts +from nemo_fabric.types import AppliedPricing, ModelUsage, PerModelUsage, TokenCounts def test_per_model_complete_counts_have_the_public_token_types(): @@ -472,3 +472,23 @@ def test_independently_rounded_provider_cost_is_not_a_token_reconciliation_error per_model_usage["cost"]["reported"] = {"amount_nano_usd": 9} usage["models"] = [per_model_usage] assert ModelUsage.from_mapping(usage).cost.reported.amount_nano_usd == 10 + + +@pytest.mark.parametrize("cache_rates", [{}, {"cache_read": 0}, {"cache_write": 0}]) +def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): + parsed = AppliedPricing.from_mapping( + { + "catalog_sha256": "sha256:" + "a" * 64, + "provider": "fixture-provider", + "model_id": "fixture-model", + "pricing_as_of": "2026-10-22", + "pricing_source": "fixture-catalog", + "rates": {"input": 10, "output": 20, **cache_rates}, + } + ) + assert parsed.rates == { + "input": 10, + "output": 20, + "cache_read": cache_rates.get("cache_read"), + "cache_write": cache_rates.get("cache_write"), + } From 559ce0a558bb9faaa258b7ec17b1ac870b9d3248 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 15:05:43 -0700 Subject: [PATCH 115/159] fix: preserve fractional cost precision across usage reducers Signed-off-by: mnajafian-nv --- adapter-contract/typescript/src/usage.ts | 38 ++++++++++++++++++- .../usage_accounting/reducer-cases.json | 36 ++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index 726b5aff0..cb8f0dd9f 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -44,6 +44,42 @@ function validate(record: UsageObservation): void { } } +// Keep low-order cost contributions until the final round, matching math.fsum. +// Inputs have already passed the finite, nonnegative cost validation. +function sumCosts(values: readonly number[]): number { + const partials: number[] = []; + for (let high of values) { + let retained = 0; + for (let low of partials) { + if (Math.abs(high) < Math.abs(low)) [high, low] = [low, high]; + const sum = high + low; + const residual = low - (sum - high); + if (residual !== 0) partials[retained++] = residual; + high = sum; + } + partials.length = retained; + partials.push(high); + } + let index = partials.length; + let total = index === 0 ? 0 : partials[--index]!; + let residual = 0; + while (index > 0) { + const high = total; + const low = partials[--index]!; + total = high + low; + residual = low - (total - high); + if (residual !== 0) break; + } + // Preserve round-to-even when the remaining low-order terms share a sign. + if (index > 0 && ((residual < 0 && partials[index - 1]! < 0) || + (residual > 0 && partials[index - 1]! > 0))) { + const correction = residual * 2; + const rounded = total + correction; + if (correction === rounded - total) total = rounded; + } + return total; +} + function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage; observed: ReportedUsage; gaps: UsageGap[] } { const reported: ReportedUsage = {}; const observed: ReportedUsage = {}; @@ -58,7 +94,7 @@ function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage let value: number | undefined; if (values.length) { if (key === "cost_usd") { - const total = values.reduce((sum, item) => sum + item, 0); + const total = sumCosts(values); if (Number.isFinite(total) && total * 1e9 < 2 ** 64) value = total; } else if (key === "peak_request_input_tokens") { value = values.reduce((maximum, item) => Math.max(maximum, item)); diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json index e79697c63..cba63db2e 100644 --- a/tests/fixtures/usage_accounting/reducer-cases.json +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -437,5 +437,41 @@ "observed_input": 100, "exhaustive": true } + }, + { + "name": "reported_cost_keeps_fractional_sum_precision", + "records": [ + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.2000000005 + } + }, + { + "model": "fixture", + "provider": "fixture", + "tokens_aligned": true, + "reported": { + "cost_usd": 0.1 + } + } + ], + "expected": { + "input_tokens": null, + "cost_usd": 0.4000000005, + "observed_input": null, + "observed_output": null, + "exhaustive": true + } } ] From 2ae6b9ebfd847ad47ffa05695d7a33c6630ece23 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 15:32:12 -0700 Subject: [PATCH 116/159] test: respect NOOA Python support in provenance cases Signed-off-by: mnajafian-nv --- tests/adapters/test_adapter_provenance.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/adapters/test_adapter_provenance.py b/tests/adapters/test_adapter_provenance.py index a210150e1..74e2537c8 100644 --- a/tests/adapters/test_adapter_provenance.py +++ b/tests/adapters/test_adapter_provenance.py @@ -7,6 +7,7 @@ import asyncio import importlib +import sys from types import SimpleNamespace from unittest.mock import AsyncMock @@ -22,6 +23,12 @@ from nemo_fabric_adapters.common import provenance +_NOOA_PYTHON = pytest.mark.skipif( + not ((3, 12) <= sys.version_info[:2] < (3, 14)), + reason="NOOA requires Python 3.12 or 3.13", +) + + @pytest.mark.parametrize( ("module", "runtime_type"), [ @@ -30,8 +37,8 @@ ("deepagents", "DeepAgentsRuntime"), ("hermes", "HermesRuntime"), ("mini_swe_agent", "MiniSweAgentRuntime"), - ("nooa", "NooaRuntime"), - ("nooa.bench", "BenchRuntime"), + pytest.param("nooa", "NooaRuntime", marks=_NOOA_PYTHON), + pytest.param("nooa.bench", "BenchRuntime", marks=_NOOA_PYTHON), ("openhands", "OpenHandsRuntime"), ("remote_agent", "RemoteAgentRuntime"), ], From 11bc1d6b889370a4a6d92c02dce16cd9dc5b58b0 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:14:31 -0700 Subject: [PATCH 117/159] feat: preserve execution facts in saved run records Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 206 ++++++- crates/fabric-core/src/record/lines.rs | 3 + crates/fabric-core/src/record/writer.rs | 3 +- crates/fabric-core/src/what_ran.rs | 8 +- .../nemo_fabric.reporting.md | 3 +- .../what-ran/struct-whatran.mdx | 2 +- docs/sdk/python.mdx | 8 +- schemas/sdk/run-record-line.schema.json | 557 ++++++++++++++++++ schemas/sdk/run-result.schema.json | 4 +- .../src/nemo_fabric/reporting.py | 76 ++- tests/python/test_invocation_reporting.py | 69 +++ 11 files changed, 922 insertions(+), 17 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 11df9c900..1543ca278 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -20,7 +20,7 @@ use super::lines::{ ContextValue, EndArtifact, EndError, EndLine, EndUsage, LineBody, ModelName, NotRecorded, RecordLine, RecordSchema, RecordVersion, StartLine, cut_text, recorded_code, }; -use super::writer::{RECORD_FILE_NAME, RecordWriter}; +use super::writer::{MAX_LINE_BYTES, RECORD_FILE_NAME, RecordWriter}; use crate::config::RunPlan; use crate::error::{FabricError, Result}; use crate::runtime::{ @@ -159,7 +159,7 @@ impl InvocationRecorder { let Some(writer) = self.writer.as_mut() else { return; }; - let line = RecordLine { + let mut line = RecordLine { schema: RecordSchema::RunRecord, version: RecordVersion::V1Alpha1, runtime_id: self.runtime_id.clone(), @@ -168,13 +168,40 @@ impl InvocationRecorder { time: rfc3339_millis(SystemTime::now()), body, }; - if let Err(error) = writer.append(&line) { + if let Err(error) = + fit_execution_facts_to_line_budget(&mut line).and_then(|()| writer.append(&line)) + { self.failure = Some(error); self.writer = None; } } } +/// Optional facts must not discard an otherwise valid accounting end line. +/// The full result keeps the complete bounded object and remains authoritative. +fn fit_execution_facts_to_line_budget(line: &mut RecordLine) -> io::Result<()> { + if !matches!(&line.body, LineBody::End(end) if end.what_ran.is_some()) { + return Ok(()); + } + let serialized_len = |line: &RecordLine| { + serde_json::to_vec(line) + .map(|bytes| bytes.len()) + .map_err(io::Error::other) + }; + if serialized_len(line)? <= MAX_LINE_BYTES { + return Ok(()); + } + if let LineBody::End(end) = &mut line.body { + end.what_ran = Some(crate::what_ran::WhatRan::minimal()); + } + if serialized_len(line)? > MAX_LINE_BYTES + && let LineBody::End(end) = &mut line.body + { + end.what_ran = None; + } + Ok(()) +} + fn start_line(start: &RecordStart<'_>) -> StartLine { StartLine { agent_name: cut_text(&start.runtime.agent_name), @@ -243,6 +270,7 @@ fn result_end_line(result: &RunResult, elapsed_ms: u64) -> EndLine { error: result.error.as_ref().map(result_error), usage: result.usage.as_ref().map(end_usage), model_usage: result.model_usage.clone(), + what_ran: result.what_ran.clone(), artifacts: end_artifacts(&result.artifacts), } } @@ -272,6 +300,7 @@ fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { }), usage: None, model_usage: None, + what_ran: None, artifacts: Vec::new(), } } @@ -493,6 +522,177 @@ mod tests { .expect("result") } + #[test] + fn end_line_keeps_bounded_execution_facts_without_changing_status() { + let (_, _, invocation) = record_test_fixture(); + let vectors: Value = serde_json::from_str(include_str!( + "../../../../tests/fixtures/what_ran/reader-vectors.json" + )) + .expect("reader vectors"); + let mut result = succeeded_result(&invocation); + result.what_ran = serde_json::from_value(vectors["base"].clone()).expect("what_ran"); + let end = serde_json::to_value(result_end_line(&result, 25)).expect("end line"); + assert_eq!(end["what_ran"], vectors["base"]); + assert_eq!(end["status"], "succeeded"); + assert_eq!(end["elapsed_ms"], 25); + assert_eq!(result.status, RunStatus::Succeeded); + } + + #[test] + fn oversized_execution_facts_keep_the_end_line_and_full_result() { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder("large-execution-facts"); + let request = RunRequest::text("input"); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + // Character bounds differ from byte bounds for valid Unicode values. + let (mut facts_plan, _, _) = record_test_fixture(); + let model = facts_plan.config.models["default"].clone(); + facts_plan.config.models = (0..16) + .map(|index| { + let role = format!("role-{index:02}-{}", "🧪".repeat(120)); + let mut model = model.clone(); + model.model = "🧪".repeat(256); + (role, model) + }) + .collect(); + facts_plan.capability_plan.mcp_servers = (0..32) + .map(|index| { + let name = format!("mcp-{index:02}-{}", "🧪".repeat(121)); + let server = serde_json::from_value(serde_json::json!({ + "transport": "stdio", "url": "🧪".repeat(256), "exposure": "harness_native", + })) + .expect("MCP server plan"); + (name, server) + }) + .collect(); + let what_ran = crate::what_ran::collect_before_request( + &facts_plan, + Some("nvidia.fabric.hermes".to_string()), + "local", + ); + let facts = serde_json::to_value(&what_ran).expect("collected facts"); + assert_eq!(facts["models"].as_object().expect("models").len(), 16); + assert_eq!( + facts["mcp"]["servers"].as_object().expect("servers").len(), + 32 + ); + assert!(facts.get("models_truncated").is_none()); + assert!(facts["mcp"].get("servers_truncated").is_none()); + let facts_bytes = serde_json::to_vec(&facts).expect("facts JSON").len(); + assert!( + facts_bytes > MAX_LINE_BYTES && facts_bytes <= 128 * 1024, + "{facts_bytes}" + ); + let mut result = succeeded_result(&invocation); + result.what_ran = Some(what_ran); + result.usage = Some(RunUsage { + input_tokens: Some(3), + ..RunUsage::default() + }); + let source: crate::AgentUsage = serde_json::from_value(serde_json::json!({ + "input_tokens": 3, "input_tokens_include_cache": true, + "extensions": {"nemo_fabric.usage_accounting.v1": {"gaps": [], "models_exhaustive": false}}, + })).expect("valid accounting source"); + result.model_usage = crate::model_usage::normalize_model_usage(&source, None, false) + .expect("normalize accounting"); + let accounting = serde_json::to_value(&result.model_usage).expect("accounting JSON"); + assert_eq!(accounting["input_tokens"], 3); + let finished = recorder.finish(Ok(result)).expect("finished result"); + let record = std::fs::read_to_string(folder.join(RECORD_FILE_NAME)).expect("record"); + let rows: Vec = record + .lines() + .map(|line| serde_json::from_str(line).expect("line")) + .collect(); + assert_eq!( + rows.len(), + 2, + "large execution facts must not discard the end line" + ); + assert_eq!(rows[1]["status"], "succeeded"); + assert_eq!(rows[1]["usage"]["input_tokens"], 3); + assert_eq!(rows[1]["model_usage"], accounting); + assert_eq!( + serde_json::to_value(&finished.model_usage).expect("returned accounting"), + accounting + ); + assert_eq!( + rows[1]["what_ran"]["what_ran_truncated"], + serde_json::json!({"reason": "byte_budget", "dropped": 1}) + ); + assert_eq!( + serde_json::to_value(finished.what_ran).expect("full facts"), + facts + ); + assert!( + !finished + .events + .iter() + .any(|event| event.kind == RECORD_WRITE_FAILED) + ); + } + + #[test] + fn record_facts_respect_exact_budget_and_never_hide_base_overflow() { + let (_, _, invocation) = record_test_fixture(); + let result = succeeded_result(&invocation); + let mut line = RecordLine { + schema: RecordSchema::RunRecord, + version: RecordVersion::V1Alpha1, + runtime_id: invocation.runtime_id.clone(), + invocation_id: invocation.invocation_id.clone(), + request_id: invocation.request_id.clone(), + time: "2026-10-21T00:00:00.000Z".to_string(), + body: LineBody::End(Box::new(result_end_line(&result, 0))), + }; + if let LineBody::End(end) = &mut line.body { + end.what_ran = Some(crate::what_ran::WhatRan::minimal()); + } + let overhead = serde_json::to_vec(&line).expect("line JSON").len(); + line.request_id + .push_str(&"a".repeat(MAX_LINE_BYTES - overhead)); + let exact = line.clone(); + fit_execution_facts_to_line_budget(&mut line).expect("exact line"); + assert_eq!(line, exact, "a line exactly at the limit keeps its facts"); + + if let LineBody::End(end) = &mut line.body { + end.what_ran = None; + } + let overhead = serde_json::to_vec(&line).expect("base JSON").len(); + line.request_id + .push_str(&"b".repeat(MAX_LINE_BYTES - overhead)); + let exact_base = line.clone(); + if let LineBody::End(end) = &mut line.body { + end.what_ran = Some(crate::what_ran::WhatRan::minimal()); + } + fit_execution_facts_to_line_budget(&mut line).expect("omit optional facts"); + assert_eq!( + line, exact_base, + "even a marker must not discard an otherwise valid line" + ); + line.request_id.push('x'); + if let LineBody::End(end) = &mut line.body { + end.what_ran = Some(crate::what_ran::WhatRan::minimal()); + } + fit_execution_facts_to_line_budget(&mut line).expect("base remains oversized"); + let folder = temp_folder("base-overflow"); + let mut writer = RecordWriter::create(&folder.join(RECORD_FILE_NAME)).expect("writer"); + assert!( + writer.append(&line).is_err(), + "existing base overflow must remain an error" + ); + assert_eq!( + serde_json::to_vec(&line).expect("overflow JSON").len(), + MAX_LINE_BYTES + 1 + ); + } + #[test] fn finish_names_the_record_artifact_apart_from_an_adapter_artifact_named_record() { let (plan, runtime, invocation) = record_test_fixture(); diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index 9ba4aba10..88f8a9f6c 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -154,6 +154,9 @@ pub(crate) struct EndLine { /// Validated normalized accounting frozen at the invocation boundary. #[serde(default, skip_serializing_if = "Option::is_none")] pub model_usage: Option, + /// Bounded execution facts frozen with the finalized invocation result. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub what_ran: Option, /// Result artifacts, with paths relative to the artifact root. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub artifacts: Vec, diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs index a1ece00d5..85947481d 100644 --- a/crates/fabric-core/src/record/writer.rs +++ b/crates/fabric-core/src/record/writer.rs @@ -17,7 +17,7 @@ use super::lines::RecordLine; /// File name of the record inside the invocation folder. pub(crate) const RECORD_FILE_NAME: &str = "run-record.jsonl"; /// Maximum serialized JSON size of one line, excluding its newline. -const MAX_LINE_BYTES: usize = 64 * 1024; +pub(super) const MAX_LINE_BYTES: usize = 64 * 1024; /// Appends lines to one invocation's record. #[derive(Debug)] @@ -179,6 +179,7 @@ mod tests { error: None, usage: None, model_usage: None, + what_ran: None, artifacts: Vec::new(), })), }; diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index e7fa118fc..f246b07f4 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -316,9 +316,9 @@ pub struct WhatRan { )] #[schemars(with = "EnvironmentFacts")] pub environment: Option, - /// Present only on a minimal object: every section was dropped to keep - /// the object within 128 KiB, with reason `byte_budget` and a dropped - /// count of 1. + /// Present only on a minimal object: every section was dropped to fit + /// its containing record's byte budget, with reason `byte_budget` and + /// a dropped count of 1. The full-result object uses a 128 KiB budget. #[serde( default, deserialize_with = "present", @@ -622,7 +622,7 @@ impl WhatRan { /// The object left when every section had to go: the schema name, the /// version and a `what_ran_truncated` marker. - fn minimal() -> Self { + pub(crate) fn minimal() -> Self { Self { schema: WHAT_RAN_SCHEMA.to_string(), version: WHAT_RAN_VERSION.to_string(), diff --git a/docs/reference/api/python-library-reference/nemo_fabric.reporting.md b/docs/reference/api/python-library-reference/nemo_fabric.reporting.md index 5dd79118f..c88c9a077 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.reporting.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.reporting.md @@ -13,6 +13,7 @@ Inspect saved invocation evidence without a benchmark manifest or trace import. **Global Variables** --------------- - **MAX_RESULT_BYTES** +- **RESOURCE_SCOPE** --- @@ -85,7 +86,7 @@ Describe the invocation interval without assuming whole-runtime success. def render_result_summary(summary: Mapping[str, Any], *, details: bool = False) -> str ``` -Render a concise overview; details add resource and pricing evidence. +Render a concise overview; details add resource evidence. Token and cost completeness remain separate. JSON quantities and accounting semantics are unchanged by this presentation. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx index a7d5f6413..1687ae0a1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/what-ran/struct-whatran.mdx @@ -58,7 +58,7 @@ Environment the runtime ran in. ### `what_ran_truncated: Option` -Present only on a minimal object: every section was dropped to keep the object within 128 KiB, with reason `byte_budget` and a dropped count of 1. +Present only on a minimal object: every section was dropped to fit its containing record's byte budget, with reason `byte_budget` and a dropped count of 1. The full-result object uses a 128 KiB budget. ## Trait Implementations diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 6eff14d9e..485b26a4c 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1596,8 +1596,12 @@ The start/end record's elapsed time covers its invocation window, including star record I/O and core preparation. It excludes end-record writes, full-result persistence and one-shot runtime startup/shutdown. It is not pure model time or the caller's task schedule interval. Record I/O is synchronous and best effort. -Each record line is limited to 64 KiB; oversized lines stop that record without -replacing the available execution result. +Each record line is limited to 64 KiB. The end line copies bounded `what_ran` +from the finalized invocation result. If those optional facts would exceed the +line budget, the record keeps a minimal `what_ran_truncated` marker with reason +`byte_budget` and `dropped: 1`. If even that marker does not fit, the record omits +`what_ran`. The separate full result keeps its original facts. Other oversized +lines stop that record without replacing the available execution result. ### Inspect a saved run diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index c35c9a6c9..76aaeb9b0 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -1,5 +1,25 @@ { "$defs": { + "AdapterFacts": { + "description": "Adapter facts.", + "properties": { + "id": { + "description": "Adapter ID from the plan, when an adapter was resolved.", + "type": [ + "string", + "null" + ] + }, + "package_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the adapter package, as the adapter reports it." + } + }, + "required": [ + "package_version" + ], + "type": "object" + }, "AppliedPricing": { "additionalProperties": false, "description": "Persisted entry and selected rates, so saved prices remain reproducible.", @@ -203,6 +223,17 @@ } ], "description": "Numeric usage counters, without adapter metadata." + }, + "what_ran": { + "anyOf": [ + { + "$ref": "#/$defs/WhatRan" + }, + { + "type": "null" + } + ], + "description": "Bounded execution facts frozen with the finalized invocation result." } }, "required": [ @@ -270,6 +301,19 @@ }, "type": "object" }, + "EnvironmentFacts": { + "description": "Environment facts.", + "properties": { + "provider": { + "description": "Environment provider, such as `local`.", + "type": "string" + } + }, + "required": [ + "provider" + ], + "type": "object" + }, "ErrorStage": { "description": "NeMo Fabric lifecycle stage associated with an error.", "oneOf": [ @@ -315,6 +359,108 @@ } ] }, + "FabricFacts": { + "description": "NeMo Fabric core facts.", + "properties": { + "core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the core that ran the adapter." + }, + "host_core_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the host core that relayed the result from a task core\nin another environment. `not_applicable` for a local run." + } + }, + "required": [ + "core_version", + "host_core_version" + ], + "type": "object" + }, + "FactSource": { + "description": "How NeMo Fabric learned a fact.", + "oneOf": [ + { + "const": "observed", + "description": "NeMo Fabric core saw it itself.", + "type": "string" + }, + { + "const": "reported", + "description": "The adapter, the harness, an MCP server or a remote service said so.", + "type": "string" + }, + { + "const": "configured", + "description": "It comes from the resolved run plan.", + "type": "string" + } + ] + }, + "FactState": { + "description": "Why a fact has no value. A collector picks the first state that matches,\nin declaration order.", + "oneOf": [ + { + "const": "not_applicable", + "description": "The fact cannot exist for this run.", + "type": "string" + }, + { + "const": "unsupported", + "description": "The adapter or environment provider has no source for the fact.", + "type": "string" + }, + { + "const": "collection_failed", + "description": "A source exists, but it failed, timed out, returned an invalid report\nor never returned.", + "type": "string" + }, + { + "const": "unavailable", + "description": "NeMo Fabric holds no value and no evidence for the other states.", + "type": "string" + } + ] + }, + "Fact_for_Array_of_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_Array_of_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_boolean": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_boolean", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, + "Fact_for_string": { + "anyOf": [ + { + "$ref": "#/$defs/KnownFact_for_string", + "description": "The fact has a value." + }, + { + "$ref": "#/$defs/MissingFact", + "description": "The fact has no value." + } + ], + "description": "One fact: a `value`, or a `state` with a `reason`, never both." + }, "GapReason": { "description": "Categories that affect whether a consumer can treat a value as complete.", "oneOf": [ @@ -335,6 +481,323 @@ } ] }, + "HarnessFacts": { + "description": "Harness facts.", + "properties": { + "name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name of the harness program that ran, as the adapter reports it.\nCore never derives it from the adapter ID." + }, + "version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version of the harness program that ran, as the adapter reports it." + } + }, + "required": [ + "name", + "version" + ], + "type": "object" + }, + "KnownFact_for_Array_of_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_boolean": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "boolean" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "KnownFact_for_string": { + "description": "A fact with a value.", + "not": { + "anyOf": [ + { + "required": [ + "state" + ] + }, + { + "required": [ + "reason" + ] + } + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "value": { + "description": "The fact itself: a string, a boolean or a list of strings.", + "type": "string" + } + }, + "required": [ + "value", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "McpFacts": { + "description": "MCP facts.", + "properties": { + "servers": { + "additionalProperties": { + "$ref": "#/$defs/McpServerFacts" + }, + "description": "Servers keyed by name. Empty when the plan configures no MCP server\nand the harness reports none. At most 32 servers, in sorted key\norder, each key at most 128 characters.", + "type": "object" + }, + "servers_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Servers dropped from `servers`. Absent when none was dropped, never\n`null`." + } + }, + "required": [ + "servers" + ], + "type": "object" + }, + "McpRuntimeFacts": { + "description": "What the harness learned about one MCP server when it connected.", + "properties": { + "connected": { + "$ref": "#/$defs/Fact_for_boolean", + "description": "Whether the harness connected to the server." + }, + "protocol_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "MCP protocol version negotiated with the server." + }, + "server_name": { + "$ref": "#/$defs/Fact_for_string", + "description": "Name the server reports for itself." + }, + "server_version": { + "$ref": "#/$defs/Fact_for_string", + "description": "Version the server reports for itself." + }, + "tool_names": { + "$ref": "#/$defs/Fact_for_Array_of_string", + "description": "Raw tool names the server lists, each bounded by the string length\nrule." + } + }, + "required": [ + "connected", + "protocol_version", + "server_name", + "server_version", + "tool_names" + ], + "type": "object" + }, + "McpServerFacts": { + "description": "Facts about one MCP server.\n\nAn entry with origin `harness` is a server the harness reports and the\nplan does not configure. It has the same shape: its `endpoint` reads\n`not_applicable` with reason `not_configured`, and runtime facts the\nadapter did not collect read `unavailable` with reason `not_collected`.\nOnly a `plan` entry must name its transport. An entry of any other\norigin may leave it out.", + "if": { + "properties": { + "origin": { + "const": "plan" + } + } + }, + "properties": { + "endpoint": { + "$ref": "#/$defs/Fact_for_string", + "description": "Configured endpoint without secrets: the lowercase scheme and host\nand the port for a network server, or the executable's file name for\na `stdio` server." + }, + "origin": { + "description": "Where the entry comes from: `plan` for a server the plan configures,\n`harness` for one only the harness reports.", + "type": "string" + }, + "runtime": { + "$ref": "#/$defs/McpRuntimeFacts", + "description": "What the harness learned when it connected, as the adapter reports it." + }, + "transport": { + "description": "Transport, such as `stdio` or `streamable-http`. Required for a\n`plan` entry, and optional for any other origin.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "origin", + "endpoint", + "runtime" + ], + "then": { + "properties": { + "transport": { + "type": "string" + } + }, + "required": [ + "transport" + ] + }, + "type": "object" + }, + "MissingFact": { + "description": "A fact without a value.", + "not": { + "required": [ + "value" + ] + }, + "properties": { + "collected_by": { + "description": "Component that collected the fact, such as `host_core`, `task_core`\nor `adapter`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "method": { + "description": "How the collector got the fact, such as `crate_version`,\n`resolved_plan` or `adapter_report`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "reason": { + "description": "Short code that explains the state, such as `key_not_declared`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FactSource", + "description": "How NeMo Fabric learned the fact." + }, + "state": { + "$ref": "#/$defs/FactState", + "description": "Why the fact has no value." + } + }, + "required": [ + "state", + "reason", + "source", + "collected_by", + "method" + ], + "type": "object" + }, + "ModelFacts": { + "description": "Facts about one model role.", + "properties": { + "provider": { + "description": "Provider from the plan.", + "type": "string" + }, + "requested": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model the plan requested." + }, + "served": { + "$ref": "#/$defs/Fact_for_string", + "description": "Model that served, as the harness or provider reports it." + } + }, + "required": [ + "provider", + "requested", + "served" + ], + "type": "object" + }, "ModelName": { "description": "One configured model.", "properties": { @@ -819,6 +1282,28 @@ }, "type": "object" }, + "TruncatedEntries": { + "description": "Entries core dropped to keep `what_ran` within its bounds: next to a\nmap for the entries dropped from it, or as `what_ran_truncated` for the\nwhole object.", + "properties": { + "dropped": { + "description": "How many entries were dropped. A JSON integer: the Rust and Python\nreaders reject `1.0`, although a JSON Schema validator may accept it.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 1, + "type": "integer" + }, + "reason": { + "description": "Why the entries were dropped: `key_too_long` for a key longer than\n128 characters, `entry_cap` for an entry past the map's cap, or\n`byte_budget` for an entry dropped to keep the whole object within\n128 KiB. One marker covers a map: when more than one rule dropped\nentries from it, `reason` names the last rule applied, in that\norder, and `dropped` counts them all. An open code like a fact's\n`reason`.", + "pattern": "^[a-z0-9_.:#+/-]{1,128}$", + "type": "string" + } + }, + "required": [ + "reason", + "dropped" + ], + "type": "object" + }, "UsageDimension": { "description": "Dimensions whose complete values can be unavailable independently.", "oneOf": [ @@ -889,6 +1374,78 @@ "reason" ], "type": "object" + }, + "WhatRan": { + "anyOf": [ + { + "required": [ + "what_ran_truncated" + ] + }, + { + "required": [ + "fabric", + "adapter", + "harness", + "models", + "mcp", + "environment" + ] + } + ], + "description": "What executed for one invocation.\n\nEvery section is present, unless `what_ran_truncated` is present: then\nthe object is a minimal one that the byte budget left with only its\nschema name and version, and every section may be absent.\n\nDecoding a `WhatRan` directly checks the shape of every section and\nfact, and that no section is `null`. `RunResult` also checks the schema\nname and version, that every section is present unless\n`what_ran_truncated` is, and that every `plan` MCP entry names its\ntransport.", + "properties": { + "adapter": { + "$ref": "#/$defs/AdapterFacts", + "description": "Adapter that ran." + }, + "environment": { + "$ref": "#/$defs/EnvironmentFacts", + "description": "Environment the runtime ran in." + }, + "fabric": { + "$ref": "#/$defs/FabricFacts", + "description": "NeMo Fabric core that ran the adapter." + }, + "harness": { + "$ref": "#/$defs/HarnessFacts", + "description": "Harness behind the adapter." + }, + "mcp": { + "$ref": "#/$defs/McpFacts", + "description": "MCP server facts." + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelFacts" + }, + "description": "Model facts keyed by the role names of the plan's `models` settings.\nAt most 16 roles, in sorted key order, each key at most 128\ncharacters.", + "type": "object" + }, + "models_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Model roles dropped from `models`. Absent when none was dropped,\nnever `null`." + }, + "schema": { + "const": "nemo.fabric.what_ran", + "description": "Schema name, `nemo.fabric.what_ran`.", + "type": "string" + }, + "version": { + "const": "v1alpha1", + "description": "Contract version, `v1alpha1`.", + "type": "string" + }, + "what_ran_truncated": { + "$ref": "#/$defs/TruncatedEntries", + "description": "Present only on a minimal object: every section was dropped to fit\nits containing record's byte budget, with reason `byte_budget` and\na dropped count of 1. The full-result object uses a 128 KiB budget." + } + }, + "required": [ + "schema", + "version" + ], + "type": "object" } }, "$schema": "https://json-schema.org/draft/2020-12/schema", diff --git a/schemas/sdk/run-result.schema.json b/schemas/sdk/run-result.schema.json index 7466ade84..90ce14b22 100644 --- a/schemas/sdk/run-result.schema.json +++ b/schemas/sdk/run-result.schema.json @@ -2113,7 +2113,7 @@ }, "what_ran_truncated": { "$ref": "#/$defs/TruncatedEntries", - "description": "Present only on a minimal object: every section was dropped to keep\nthe object within 128 KiB, with reason `byte_budget` and a dropped\ncount of 1." + "description": "Present only on a minimal object: every section was dropped to fit\nits containing record's byte budget, with reason `byte_budget` and\na dropped count of 1. The full-result object uses a 128 KiB budget." } }, "required": [ @@ -2267,4 +2267,4 @@ ], "title": "RunResult", "type": "object" -} +} \ No newline at end of file diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py index 9c6cf6948..7b5fb9b43 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py @@ -22,6 +22,7 @@ ResourceObservation, ModelUsage, RuntimeResourceSnapshot, + _readable_what_ran, ) MAX_RESULT_BYTES = 16 * 1024 * 1024 @@ -144,6 +145,7 @@ def summarize_result(result: Mapping[str, Any]) -> dict[str, Any]: except (FabricConfigError, TypeError, ValueError): stats = None stats_error = "invalid_system_stats" + what_ran = _readable_what_ran(document.get("what_ran")) refs = document.get("artifacts") refs = refs.get("artifacts", []) if isinstance(refs, Mapping) else [] artifacts = ( @@ -202,8 +204,10 @@ def summarize_result(result: Mapping[str, Any]) -> dict[str, Any]: ) if stats is not None else None, - "what_ran": None, - "provenance_error": None, + "what_ran": what_ran, + "provenance_error": "unreadable_what_ran" + if document.get("what_ran") is not None and what_ran is None + else None, "artifacts": artifacts, "telemetry": document["telemetry"] if isinstance(document.get("telemetry"), list) @@ -459,7 +463,12 @@ def read_invocation_record(path: Path) -> dict[str, Any]: "usage": usage, "accounting_error": accounting_error, "legacy_usage": end.get("usage") if end else None, - "what_ran": None, + "what_ran": _readable_what_ran(end.get("what_ran")) if end else None, + "provenance_error": "unreadable_what_ran" + if end + and end.get("what_ran") is not None + and _readable_what_ran(end["what_ran"]) is None + else None, "elapsed_ms": end["elapsed_ms"] if end else None, "exchange_elapsed_ms": end.get("exchange_elapsed_ms") if end else None, "resource_window": None, @@ -499,6 +508,7 @@ def render_invocation_record_summary( "Invocation outcome before runtime shutdown: " + (summary["status"] or "unavailable"), ] + lines.extend(_render_provenance(summary.get("what_ran"), details=details)) if summary.get("provenance_error"): lines.append( "Saved execution facts were unreadable; status and accounting are retained." @@ -528,6 +538,61 @@ def render_invocation_record_summary( return "\n".join(lines) +def _render_provenance( + what_ran: Mapping[str, Any] | None, *, details: bool = False +) -> list[str]: + """Render only bounded public identities, keeping requested and served separate.""" + if what_ran is None: + return [] + if what_ran.get("what_ran_truncated") is not None: + return ["Execution facts: record exceeded its byte budget."] + lines = [] + + def value(fact: Any) -> str | None: + return ( + fact.get("value") + if isinstance(fact, Mapping) and isinstance(fact.get("value"), str) + else None + ) + + harness = what_ran.get("harness", {}) + version = value(harness.get("version")) + name = value(harness.get("name")) + if version is not None: + # JSON quoting keeps terminal control characters from escaping the value. + lines.append( + "Harness identity (adapter reported): " + + json.dumps({"name": name, "version": version}, ensure_ascii=True) + ) + models = what_ran.get("models", {}) + if not details and len(models) > 1: + label = "Additional model roles" if "default" in models else "Model roles" + lines.append( + f"{label}: {len(models) - (1 if 'default' in models else 0)} (see --details)." + ) + for role, facts in models.items(): + if not details and len(models) > 1 and role != "default": + continue + requested, served = value(facts.get("requested")), value(facts.get("served")) + if requested is not None or served is not None: + lines.append( + "Model role " + + json.dumps(role, ensure_ascii=True) + + ": " + + json.dumps( + {"requested": requested, "served": served}, ensure_ascii=True + ) + ) + if ( + what_ran.get("models_truncated") is not None + or what_ran.get("mcp", {}).get("servers_truncated") is not None + ): + lines.append( + "Execution facts: entries were omitted; inspect truncation markers in JSON." + ) + return lines + + def _render_tokens(dimensions: Mapping[str, Any]) -> list[str]: totals = ("input_tokens", "output_tokens", "total_tokens") caches = ("cached_input_tokens", "cache_write_input_tokens") @@ -623,6 +688,11 @@ def render_result_summary(summary: Mapping[str, Any], *, details: bool = False) f"Runtime process-tree snapshot: CPU {resource['cpu_time_seconds']:.3f} s since host start; " f"RSS {format_bytes(resource['resident_memory_bytes'])}; live processes {resource['process_count']}" ) + lines.extend(_render_provenance(summary.get("what_ran"), details=details)) + if summary.get("provenance_error"): + lines.append( + "Saved execution facts were unreadable; status and accounting are retained." + ) normalized = summary.get("model_usage") models = normalized.get("models", []) if normalized is not None else [] if models: diff --git a/tests/python/test_invocation_reporting.py b/tests/python/test_invocation_reporting.py index a1e640545..27b363464 100644 --- a/tests/python/test_invocation_reporting.py +++ b/tests/python/test_invocation_reporting.py @@ -3,6 +3,7 @@ from copy import deepcopy import json +from pathlib import Path import pytest @@ -568,3 +569,71 @@ def test_details_retain_resource_scope_while_overview_stays_compact(result): assert "Invocation:" in detail assert "System stats collected groups" in detail assert "bytes" not in detail # Human-readable memory uses binary units. + + +def _execution_facts(): + path = Path(__file__).parents[1] / "fixtures/what_ran/reader-vectors.json" + return json.loads(path.read_text())["base"] + + +def test_full_result_keeps_execution_facts_without_recording(result): + result["what_ran"] = _execution_facts() + summary = summarize_result(result) + assert summary["what_ran"] == result["what_ran"] + assert summary["provenance_error"] is None + text = render_result_summary(summary) + assert '"requested": "claude-sonnet-4-5", "served": null' in text + assert "Accounted models" not in text + + +def test_record_only_keeps_bounded_execution_facts(tmp_path, record_lines): + record_lines[1]["what_ran"] = _execution_facts() + path = tmp_path / "run-record.jsonl" + write_record(path, record_lines) + summary = read_evidence(path) + assert summary["what_ran"] == record_lines[1]["what_ran"] + assert summary["status"] == "succeeded" + assert summary["elapsed_ms"] == 123 + + +def test_unreadable_execution_facts_keep_execution_and_accounting(result): + result["what_ran"] = {"schema": "nemo.fabric.what_ran", "version": "future"} + summary = summarize_result(result) + assert summary["what_ran"] is None + assert summary["provenance_error"] == "unreadable_what_ran" + assert summary["status"] == "succeeded" + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + assert "facts were unreadable" in render_result_summary(summary) + + +def test_full_result_facts_remain_authoritative_over_sibling_record( + tmp_path, record_lines, result +): + record_lines[1]["what_ran"] = _execution_facts() + write_record(tmp_path / "run-record.jsonl", record_lines) + result["what_ran"] = {"schema": "unreadable"} + path = tmp_path / "run-result.json" + path.write_text(json.dumps(result)) + summary = read_evidence(path) + assert summary["what_ran"] is None + assert summary["provenance_error"] == "unreadable_what_ran" + + +def test_minimal_execution_facts_show_byte_budget_omission(result): + result["what_ran"] = { + "schema": "nemo.fabric.what_ran", + "version": "v1alpha1", + "what_ran_truncated": {"reason": "byte_budget", "dropped": 1}, + } + summary = summarize_result(result) + assert summary["what_ran"] == result["what_ran"] + assert "Execution facts: record exceeded its byte budget." in render_result_summary( + summary + ) + + +def test_absent_execution_facts_are_not_malformed(result): + summary = summarize_result(result) + assert summary["what_ran"] is None + assert summary["provenance_error"] is None + assert "facts were unreadable" not in render_result_summary(summary) From a6318d4dede15289bbf31485296c6bec8bc524b0 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:24:17 -0700 Subject: [PATCH 118/159] chore: check packaged usage schema declarations Keep packaged adapter declarations in the canonical usage-schema synchronization and drift check. Prevent the check from passing with a stale packaged schema. Signed-off-by: mnajafian-nv --- scripts/contracts/generate_usage_accounting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/contracts/generate_usage_accounting.py b/scripts/contracts/generate_usage_accounting.py index 49bb52bef..3b5cbd19c 100644 --- a/scripts/contracts/generate_usage_accounting.py +++ b/scripts/contracts/generate_usage_accounting.py @@ -22,6 +22,7 @@ def synchronize(*, check: bool) -> None: descriptors = [ *ROOT.glob("adapters/python/*/*.fabric-adapter.json"), *ROOT.glob("adapters/typescript/*/*.fabric-adapter.json"), + *ROOT.glob("crates/fabric-cli/assets/adapters/*/*.fabric-adapter.json"), ] for path in descriptors: original = path.read_text() From d35ebb3fa8b055d80d69375ccf5b42e896401092 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:38:47 -0700 Subject: [PATCH 119/159] fix: qualify child model pricing by provider endpoint Check the source-qualified child provider against its own inherited endpoint before applying bundled prices. Keep native served aliases and missing-provider behavior unchanged. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/pricing.rs | 56 ++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/pricing.rs b/crates/fabric-core/src/pricing.rs index 191c1f965..008c576d6 100644 --- a/crates/fabric-core/src/pricing.rs +++ b/crates/fabric-core/src/pricing.rs @@ -27,7 +27,21 @@ pub(crate) fn native_catalog_route( forwarded: &std::collections::BTreeMap, inherited: impl Fn(&str) -> Option, ) -> bool { - let (keys, urls): (&[&str], &[&str]) = match model.provider.as_str() { + native_catalog_endpoint( + &model.provider, + model.base_url.as_deref(), + forwarded, + inherited, + ) +} + +fn native_catalog_endpoint( + provider: &str, + base_url: Option<&str>, + forwarded: &std::collections::BTreeMap, + inherited: impl Fn(&str) -> Option, +) -> bool { + let (keys, urls): (&[&str], &[&str]) = match provider { "openai" => (&OPENAI_ENDPOINT_ENV_KEYS, &["https://api.openai.com/v1"]), "anthropic" => ( &ANTHROPIC_ENDPOINT_ENV_KEYS, @@ -36,7 +50,7 @@ pub(crate) fn native_catalog_route( _ => return false, }; let native = |url: &str| urls.contains(&url.trim_end_matches('/')); - if let Some(url) = &model.base_url { + if let Some(url) = base_url { return native(url); } keys.iter().all(|key| { @@ -53,6 +67,7 @@ pub(crate) fn native_catalog_route( #[derive(Debug, Clone)] pub(crate) struct NativePricingRoutes { models: Vec, + endpoints: std::collections::BTreeMap, } #[derive(Debug, Clone)] @@ -76,6 +91,7 @@ impl NativePricingRoutes { native: native_catalog_route(model, endpoints, |_| None), }) .collect(), + endpoints: endpoints.clone(), } } @@ -98,8 +114,13 @@ impl NativePricingRoutes { if provider_routes.peek().is_some() { return provider_routes.all(|route| route.native); } - // Preserve the prior all-native case for source-qualified child models. - !self.models.is_empty() && self.models.iter().all(|route| route.native) + // A child provider's inherited endpoint can differ from configured roles. + let accounted_endpoint_native = provider.is_none_or(|provider| { + native_catalog_endpoint(provider, None, &self.endpoints, |_| None) + }); + accounted_endpoint_native + && !self.models.is_empty() + && self.models.iter().all(|route| route.native) } } @@ -712,6 +733,33 @@ mod tests { assert!(!empty.allows(Some("openai"), "gpt-5.4")); } + #[test] + fn child_provider_pricing_checks_its_own_endpoint() { + let models = serde_json::from_value(json!({ + "default": {"provider": "openai", "model": "gpt-5.4"} + })) + .unwrap(); + for (endpoint, expected) in [ + (None, true), + (Some("https://api.anthropic.com/"), true), + (Some("https://proxy.invalid"), false), + ] { + let endpoints = endpoint + .map(|url| BTreeMap::from([("ANTHROPIC_BASE_URL".into(), url.into())])) + .unwrap_or_default(); + let routes = super::NativePricingRoutes::from_models(&models, &endpoints); + assert_eq!( + routes.allows(Some("anthropic"), "source-qualified-child"), + expected, + "child endpoint: {endpoint:?}" + ); + assert!(routes.allows(Some("openai"), "served-alias")); + assert!(routes.allows(None, "source-qualified-child")); + } + let routes = super::NativePricingRoutes::from_models(&models, &BTreeMap::new()); + assert!(!routes.allows(Some("unknown-provider"), "source-qualified-child")); + } + #[test] fn inherited_and_forwarded_custom_routes_require_explicit_prices() { for (configured, inherited, forwarded, expected) in [ From c198472ce1ab461c90ccfc20402ba2252942e614 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:40:15 -0700 Subject: [PATCH 120/159] fix: return detached applied pricing rates in Python Return the documented JSON-compatible rate snapshot instead of exposing a frozen internal mapping. Preserve optional cache rates and snapshot isolation. Signed-off-by: mnajafian-nv --- sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py | 1 + tests/python/test_model_usage.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index bea167f4a..f5a52cc4c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2577,6 +2577,7 @@ class AppliedPricing(FabricMapping): "rates", } ) + _json_fields = frozenset({"rates"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index fb018ffaa..805263033 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -486,9 +486,13 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): "rates": {"input": 10, "output": 20, **cache_rates}, } ) - assert parsed.rates == { + expected = { "input": 10, "output": 20, "cache_read": cache_rates.get("cache_read"), "cache_write": cache_rates.get("cache_write"), } + rates = parsed.rates + assert json.loads(json.dumps(rates)) == expected + rates["input"] = 999 + assert parsed.rates == expected From 06e2396de292a90c375e25a9d0dd623c9a5688f6 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:41:47 -0700 Subject: [PATCH 121/159] fix: preserve frozen catalog pricing for OpenShell results Keep host catalog paths out of task planning and apply the frozen explicit catalog at the host boundary. Preserve validated task-native bundled estimates when no override is configured. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/pricing.rs | 2 +- crates/fabric-core/src/runtime/openshell.rs | 266 ++++++++++++++++++++ docs/sdk/python.mdx | 4 +- 3 files changed, 269 insertions(+), 3 deletions(-) diff --git a/crates/fabric-core/src/pricing.rs b/crates/fabric-core/src/pricing.rs index 008c576d6..f3eb8fe12 100644 --- a/crates/fabric-core/src/pricing.rs +++ b/crates/fabric-core/src/pricing.rs @@ -180,7 +180,7 @@ pub struct AppliedPricing { } impl RateSnapshot { - fn from_rates(rates: TokenPricingRates) -> Result { + pub(crate) fn from_rates(rates: TokenPricingRates) -> Result { let convert = |rate| { nano_usd_per_million_tokens(rate) .and_then(|value| u64::try_from(value).ok()) diff --git a/crates/fabric-core/src/runtime/openshell.rs b/crates/fabric-core/src/runtime/openshell.rs index 75dc78bc3..38e3403c1 100644 --- a/crates/fabric-core/src/runtime/openshell.rs +++ b/crates/fabric-core/src/runtime/openshell.rs @@ -129,6 +129,11 @@ fn task_config(plan: &RunPlan, environment: &EnvironmentHandle) -> Result Result, + provider: Option<&str>, + model: &str, + prices: &crate::price_catalog::PriceCatalogs<'_>, +) -> bool { + let Some(task) = task_usage else { + return false; + }; + let Some((bundled, _)) = prices.find(provider, model, false) else { + return false; + }; + let rates = match &bundled.rate_schedule { + Some(crate::price_catalog::TokenRateSchedule::PromptTokenThreshold { tiers, .. }) => { + tiers.iter().map(|tier| tier.rates).collect::>() + } + None => bundled.rates.into_iter().collect(), + }; + let canonical_rates = rates + .into_iter() + .filter_map(|rates| crate::pricing::RateSnapshot::from_rates(rates).ok()) + .collect::>(); + // A task-core tariff decision supplies sandbox-local route evidence. Host + // endpoint inheritance cannot establish which route the sandbox used. + task.models.iter().any(|entry| { + entry.provider.as_deref() == provider + && entry.model == model + && entry.cost.pricing.as_ref().is_some_and(|basis| { + basis.user_catalog_sha256.is_none() + && basis.bundled_catalog_sha256 == prices.bundled_sha256() + }) + && !entry.cost.applied_rates.is_empty() + && entry.cost.applied_rates.iter().all(|applied| { + applied.catalog_sha256 == prices.bundled_sha256() + && applied.provider == bundled.provider + && applied.model_id == bundled.model_id + && applied.pricing_as_of == bundled.pricing_as_of + && applied.pricing_source == bundled.pricing_source + && canonical_rates.contains(&applied.rates) + }) + }) +} +fn apply_host_catalog( + result: &mut RunResult, + catalog: Option<&crate::price_catalog::UserPriceCatalog>, +) { + let Some(catalog) = catalog else { + // The task core observes the sandbox's actual provider endpoints. Keep + // its independently fingerprinted bundled estimates when no override + // was supplied, rather than inferring routes from host environment. + return; + }; + let raw = result.usage.as_ref().map(|usage| AgentUsage { + input_tokens: usage.input_tokens, + cached_input_tokens: usage.cached_input_tokens, + input_tokens_include_cache: usage.input_tokens_include_cache, + output_tokens: usage.output_tokens, + total_tokens: usage.total_tokens, + cost_usd: usage.cost_usd, + extensions: usage.metadata.clone(), + }); + let task_usage = result.model_usage.as_ref(); + let prices = crate::price_catalog::PriceCatalogs::with_user_catalog(None); + result.model_usage = match raw + .as_ref() + .map(|usage| { + crate::model_usage::normalize_model_usage_for_routes( + usage, + Some(catalog), + |provider, model| task_has_bundled_tariff(task_usage, provider, model, &prices), + ) + }) + .transpose() + { + Ok(usage) => usage.flatten(), + Err(_) => { + if !result + .events + .iter() + .any(|event| event.kind == "model_usage_invalid") + { + result.events.push(event_with_metadata( + "model_usage_invalid", + "model accounting could not be normalized; execution outcome and raw usage were retained".to_string(), + BTreeMap::new(), + )); + } + None + } + }; +} fn set_state(key: &str, runtime_id: &str, state: State) { BINDINGS .lock() @@ -433,6 +529,7 @@ impl RuntimeAdapter for OpenShellRuntimeAdapter { "", )); } + apply_host_catalog(&mut result, plan.user_price_catalog.as_ref()); for telemetry in &mut result.telemetry { telemetry.relay_enabled = telemetry .metadata @@ -847,6 +944,175 @@ mod tests { } } + fn catalog_for_task_usage() -> crate::price_catalog::UserPriceCatalog { + crate::price_catalog::UserPriceCatalog::from_bytes( + PathBuf::from("host-only-catalog.json"), + br#"{"version":1,"entries":[{"provider":"fixture","model_id":"model","currency":"USD","unit":"per_token","rates":{"input_per_million":1.0,"output_per_million":2.0,"cache_read_per_million":0.1,"cache_write_per_million":1.25},"prompt_cache":{"read_accounting":"included_in_prompt_tokens"},"pricing_as_of":"2026-10-22","pricing_source":"test fixture"}]}"#, + ).expect("frozen test catalog") + } + + fn task_usage_result() -> RunResult { + let raw: AgentUsage = serde_json::from_value(serde_json::json!({ + "extensions": {"nemo_fabric.usage_accounting.v1": { + "models_exhaustive": true, "gaps": [], "models": [{ + "model": "model", "provider": "fixture", "gaps": [], + "reported": {"input_tokens": 100, "input_tokens_include_cache": true, + "cached_input_tokens": 0, "cache_write_input_tokens": 0, + "output_tokens": 5, "output_tokens_include_reasoning": true, + "peak_request_input_tokens": 100} + }] + }} + })) + .expect("raw task accounting"); + let mut result: RunResult = serde_json::from_value(serde_json::json!({ + "agent_name": "agent", "harness": "fixture", "adapter_kind": "process", + "runtime_id": "runtime-task", "invocation_id": "invocation-1", + "request_id": "request-1", "status": "failed", "output": "partial output", + "error": {"stage": "invoke", "code": "fixture_failure", "message": "fixture failure", "retryable": false} + })).expect("task result"); + result.usage = Some(run_usage(&raw)); + result.model_usage = crate::model_usage::normalize_model_usage(&raw, None, false) + .expect("normalized task accounting"); + result + } + + #[test] + fn task_projection_does_not_send_a_host_catalog_path_to_the_sandbox() { + let plan: RunPlan = serde_json::from_value(serde_json::json!({ + "agent_name": "agent", "base_dir": "/host", "agent_config": {}, "capabilities": {}, + "config": {"schema_version": "fabric.agent/v1alpha1", "metadata": {"name": "agent"}, + "runtime": {}, "pricing": {"catalog_path": "/host-only/catalog.json"}} + })) + .expect("projection plan"); + let projected = task_config(&plan, &environment()).expect("task configuration"); + assert!(projected.pricing.unwrap().catalog_path.is_none()); + assert_eq!( + plan.config.pricing.unwrap().catalog_path, + Some(PathBuf::from("/host-only/catalog.json")) + ); + } + + #[test] + fn task_usage_uses_the_host_frozen_catalog_without_changing_the_outcome() { + let mut result = task_usage_result(); + let original = result.clone(); + assert!( + result + .model_usage + .as_ref() + .unwrap() + .cost + .estimated + .amount_nano_usd + .is_none() + ); + let catalog = catalog_for_task_usage(); + apply_host_catalog(&mut result, Some(&catalog)); + let usage = result + .model_usage + .as_ref() + .expect("host-normalized accounting"); + assert_eq!(usage.cost.estimated.amount_nano_usd, Some(110_000)); + assert_eq!( + usage + .cost + .pricing + .as_ref() + .unwrap() + .user_catalog_sha256 + .as_deref(), + Some(catalog.sha256.as_str()) + ); + assert_eq!(result.status, original.status); + assert_eq!(result.error, original.error); + assert_eq!(result.output, original.output); + assert_eq!(result.usage, original.usage); + } + + #[test] + fn task_usage_without_a_host_override_keeps_the_task_core_route_evidence() { + let mut result = task_usage_result(); + let mut raw = AgentUsage::default(); + raw.extensions = result.usage.as_ref().unwrap().metadata.clone(); + let model = &mut raw + .extensions + .get_mut("nemo_fabric.usage_accounting.v1") + .unwrap()["models"][0]; + model["provider"] = Value::from("openai"); + model["model"] = Value::from("gpt-5.4"); + result.usage = Some(run_usage(&raw)); + result.model_usage = crate::model_usage::normalize_model_usage(&raw, None, true).unwrap(); + assert!( + result + .model_usage + .as_ref() + .unwrap() + .cost + .estimated + .amount_nano_usd + .is_some() + ); + let original = result.clone(); + apply_host_catalog(&mut result, None); + assert_eq!(result, original); + apply_host_catalog(&mut result, Some(&catalog_for_task_usage())); + assert_eq!( + result.model_usage.as_ref().unwrap().cost.estimated, + original.model_usage.as_ref().unwrap().cost.estimated, + "a partial override keeps the task-qualified bundled tariff for other models" + ); + for invalid_proof in ["provider", "fingerprint", "rates", "user_catalog"] { + let mut unqualified = original.clone(); + let entry = &mut unqualified.model_usage.as_mut().unwrap().models[0]; + match invalid_proof { + "provider" => entry.provider = Some("another-provider".into()), + "fingerprint" => { + entry.cost.pricing.as_mut().unwrap().bundled_catalog_sha256 = + "sha256:another-catalog".into() + } + "rates" => entry.cost.applied_rates[0].rates.input += 1, + "user_catalog" => { + entry.cost.pricing.as_mut().unwrap().user_catalog_sha256 = + Some("sha256:user-catalog".into()) + } + _ => unreachable!(), + } + apply_host_catalog(&mut unqualified, Some(&catalog_for_task_usage())); + assert!( + unqualified + .model_usage + .as_ref() + .unwrap() + .cost + .estimated + .amount_nano_usd + .is_none(), + "{invalid_proof}" + ); + } + } + + #[test] + fn invalid_task_accounting_with_a_host_catalog_keeps_the_execution_and_raw_evidence() { + let mut result = task_usage_result(); + result.usage.as_mut().unwrap().metadata.insert( + "nemo_fabric.usage_accounting.v1".into(), + serde_json::json!({"models_exhaustive": "invalid"}), + ); + let original = result.clone(); + apply_host_catalog(&mut result, Some(&catalog_for_task_usage())); + assert!(result.model_usage.is_none()); + assert_eq!(result.status, original.status); + assert_eq!(result.error, original.error); + assert_eq!(result.usage, original.usage); + assert!( + result + .events + .iter() + .any(|event| event.kind == "model_usage_invalid") + ); + } + #[test] fn unknown_binding_survives_a_new_host_environment_handle() { let original = environment(); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index ae64c7989..c6cb2bfeb 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1231,8 +1231,8 @@ accounting remains available in `result.usage` and is not certified complete by `cost.reported` contains only provider-reported amounts; `cost.estimated` contains catalog estimates. Both distinguish `amount_nano_usd` from `observed_nano_usd`. Estimates retain their pricing basis and distinct selected rate snapshots. -Configure `PricingConfig(catalog_path="prices.json")` for a Relay-compatible -catalog override; planning freezes its bytes. An explicitly established provider must match the catalog entry's provider, including aliases and shorter model paths. A different provider's entry cannot price that source solely because the model name matches. When no provider identity is available, an explicit user catalog can match the model name alone. Bundled provider prices apply only +Set `FabricConfig(pricing=PricingConfig(catalog_path="prices.json"))` for a +Relay-compatible catalog override; planning freezes its bytes. An explicitly established provider must match the catalog entry's provider, including aliases and shorter model paths. A different provider's entry cannot price that source solely because the model name matches. When no provider identity is available, an explicit user catalog can match the model name alone. Bundled provider prices apply only when the configured route establishes the native provider. Unknown model, cache-price or request-tier evidence affects estimates without erasing tokens. Nonzero Anthropic cache writes currently lack expiry-price evidence, so their From 630820f93384bf5ce9729edd1e2c9c18d63f1999 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 16:44:48 -0700 Subject: [PATCH 122/159] fix: return detached applied pricing rates in Python Return the documented JSON-compatible rate snapshot instead of exposing a frozen internal mapping. Preserve optional cache rates and snapshot isolation. Signed-off-by: mnajafian-nv --- sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py | 1 + tests/python/test_model_usage.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index bea167f4a..f5a52cc4c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2577,6 +2577,7 @@ class AppliedPricing(FabricMapping): "rates", } ) + _json_fields = frozenset({"rates"}) @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index fb018ffaa..805263033 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -486,9 +486,13 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): "rates": {"input": 10, "output": 20, **cache_rates}, } ) - assert parsed.rates == { + expected = { "input": 10, "output": 20, "cache_read": cache_rates.get("cache_read"), "cache_write": cache_rates.get("cache_write"), } + rates = parsed.rates + assert json.loads(json.dumps(rates)) == expected + rates["input"] = 999 + assert parsed.rates == expected From 93b00ecdbf82a618e8b6947692b9ea333f59f38b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 17:23:32 -0700 Subject: [PATCH 123/159] fix: expose omitted optional SDK usage attributes Signed-off-by: mnajafian-nv --- .../nemo_fabric.types.md | 16 +++++++- .../src/nemo_fabric/types.py | 12 +++++- tests/python/test_model_usage.py | 39 +++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/docs/reference/api/python-library-reference/nemo_fabric.types.md b/docs/reference/api/python-library-reference/nemo_fabric.types.md index 83cdfab2b..b930369db 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.types.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.types.md @@ -2067,7 +2067,6 @@ The mapping exposes the following typed fields: | `normalization_version` | `str` | | `assumption_policy` | `str` | | `bundled_catalog_sha256` | `str` | -| `user_catalog_sha256` | `str \| None` | ### method `__init__` @@ -2088,6 +2087,13 @@ Return an immutable view of preserved extension fields. +--- + +### property user_catalog_sha256 + +Return the user catalog fingerprint when supplied, otherwise None. + + --- ### classmethod `from_mapping` @@ -2475,7 +2481,6 @@ The mapping exposes the following typed fields: | Field | Type | | --- | --- | | `model` | `str` | -| `provider` | `str \| None` | | `input_tokens` | `int \| None` | | `cached_input_tokens` | `int \| None` | | `cache_write_input_tokens` | `int \| None` | @@ -2504,6 +2509,13 @@ Return an immutable view of preserved extension fields. +--- + +### property provider + +Return the accounting provider when reported, otherwise None. + + --- ### classmethod `from_mapping` diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index f5a52cc4c..2dfcd259b 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2527,7 +2527,6 @@ class PricingBasis(FabricMapping): normalization_version: str assumption_policy: str bundled_catalog_sha256: str - user_catalog_sha256: str | None _fields = frozenset( { "basis", @@ -2539,6 +2538,11 @@ class PricingBasis(FabricMapping): } ) + @property + def user_catalog_sha256(self) -> str | None: + """Return the user catalog fingerprint when supplied, otherwise None.""" + return self._data.get("user_catalog_sha256") + @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if set(data).difference(cls._fields) or any( @@ -2796,7 +2800,6 @@ class PerModelUsage(FabricMapping): """ model: str - provider: str | None input_tokens: int | None cached_input_tokens: int | None cache_write_input_tokens: int | None @@ -2809,6 +2812,11 @@ class PerModelUsage(FabricMapping): {*_MODEL_USAGE_TOKEN_FIELDS, "model", "provider", "observed", "gaps", "cost"} ) + @property + def provider(self) -> str | None: + """Return the accounting provider when reported, otherwise None.""" + return self._data.get("provider") + @classmethod def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: data["model"] = _required_text(data.get("model"), "model") diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 805263033..4c731b546 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -496,3 +496,42 @@ def test_applied_pricing_accepts_schema_optional_cache_rates(cache_rates): assert json.loads(json.dumps(rates)) == expected rates["input"] = 999 assert parsed.rates == expected + + +@pytest.mark.parametrize( + "provider", [{}, {"provider": None}, {"provider": "fixture-provider"}] +) +def test_per_model_optional_provider_attribute_preserves_source_presence( + per_model_usage, provider +): + from nemo_fabric.types import PerModelUsage + + per_model_usage.update(provider) + parsed = PerModelUsage.from_mapping(per_model_usage) + assert parsed.provider == provider.get("provider") + serialized = parsed.to_mapping() + assert ("provider" in serialized) == ("provider" in provider) + if "provider" in provider: + assert serialized["provider"] == provider["provider"] + + +@pytest.mark.parametrize( + "user_catalog", + [{}, {"user_catalog_sha256": None}, {"user_catalog_sha256": "sha256:" + "b" * 64}], +) +def test_pricing_basis_optional_user_catalog_attribute_preserves_source_presence( + user_catalog, +): + from nemo_fabric.types import PricingBasis + + source = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": "sha256:" + "a" * 64, + **user_catalog, + } + parsed = PricingBasis.from_mapping(source) + assert parsed.user_catalog_sha256 == user_catalog.get("user_catalog_sha256") + assert parsed.to_mapping() == source From f5aa5beaf69a793dde7c80fa36447a0b229cc0e0 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 18:15:27 -0700 Subject: [PATCH 124/159] docs: clarify run record crash durability Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/writer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs index a1ece00d5..b8240ba33 100644 --- a/crates/fabric-core/src/record/writer.rs +++ b/crates/fabric-core/src/record/writer.rs @@ -3,10 +3,10 @@ //! The run record file: created once, owner-only, appended one line per write. //! -//! Each line is one `write_all` on an unbuffered file, so a SIGKILL of Fabric -//! or the adapter host keeps every line whose write returned. Only the end -//! line is synced, once, so the record survives a crash of the machine only -//! up to its last synced line. +//! Each line uses `write_all` on an unbuffered file. If the Fabric process +//! terminates, lines whose writes completed remain in the file. The end line +//! is synced once. Durability after a machine crash or power loss is not +//! guaranteed because the containing directory is not synced. use std::fs::{File, OpenOptions}; use std::io::{self, Write}; From 101bce6206a4ce6663e1fc876f99c35404de6fe8 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 20:53:44 -0700 Subject: [PATCH 125/159] fix: reject Relay exit during successful health probe Signed-off-by: mnajafian-nv --- adapters/python/common/README.md | 6 ++++-- .../nemo_fabric_adapters/common/relay_gateway.py | 5 +++++ .../adapters/test_adapters_common_relay_gateway.py | 13 +++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index 919bdd233..df9e8aa93 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -126,8 +126,10 @@ runtime. The Claude and Codex adapters wait up to 30 seconds for the supervised Relay gateway's HTTP health endpoint. This budget includes configuration validation -and required worker initialization. Early process exit fails startup immediately; -a gateway that never becomes healthy fails when the budget expires and is stopped. +and required worker initialization. Process exit is checked before each health +probe and again before accepting a successful response. An exit during an +unsuccessful probe is detected at the next poll. A gateway that never becomes +healthy fails when the budget expires and is stopped. The adapters retain its log for diagnosis. Local Python hosts separately enforce a 90-second adapter startup limit. diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 959f54fa7..1a57c5fe4 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -132,6 +132,11 @@ def wait_for_relay_gateway( health_url, timeout=min(1.0, remaining) ) as response: if 200 <= response.status < 300: + returncode = process.poll() + if returncode is not None: + raise RelayGatewayError( + f"NeMo Relay gateway exited with status {returncode} before becoming ready" + ) return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index e478f85de..fbd9ca893 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -239,6 +239,19 @@ def test_wait_for_relay_gateway_reports_early_exit(): ) +def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch): + process = MagicMock() + process.poll.side_effect = [None, 17] + response = MagicMock() + response.__enter__.return_value.status = 200 + monkeypatch.setattr( + relay_gateway.urllib.request, "urlopen", MagicMock(return_value=response) + ) + + with pytest.raises(relay_gateway.RelayGatewayError, match="status 17"): + relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From 2014cbe68af1259c837ab3da256883ff3dcd4a74 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 21:28:53 -0700 Subject: [PATCH 126/159] fix: exclude unknown keys from request usage Signed-off-by: mnajafian-nv --- adapter-contract/typescript/src/usage.ts | 7 ++++++- adapter-contract/typescript/test/usage-reducer.test.js | 8 ++++++++ adapters/python/deepagents/README.md | 7 +++++++ docs/sdk/python.mdx | 2 +- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index cb8f0dd9f..16c8cc5ae 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -144,7 +144,12 @@ function aligned(records: readonly UsageObservation[]): boolean { function requestUsage(record: UsageObservation): ReportedUsage { // Retain uncertain literal counts in model observations, not complete calls. - const fields = { ...record.reported }; + const fields: ReportedUsage = {}; + const keys = [...dimensions, "input_tokens_include_cache", "output_tokens_include_reasoning"] as const; + for (const key of keys) { + const value = record.reported[key]; + if (value !== undefined) (fields as Record)[key] = value; + } for (const gap of record.gaps ?? []) for (const dimension of gap.dimensions) delete fields[dimension]; return fields; } diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js index e83f1ad8b..de7fbdfb2 100644 --- a/adapter-contract/typescript/test/usage-reducer.test.js +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -46,3 +46,11 @@ test("scoped request gaps retain independent fields without certifying cache", ( assert.equal(model.requests[0].gap, undefined); assert.equal(model.requests[0].tokens_aligned, true); }); + +test("request usage omits provider-specific keys outside the wire schema", () => { + const usage = usageFromObservations([{ model: "m", is_model_call: true, + reported: { input_tokens: 7, input_tokens_include_cache: true, prompt_tokens: 7 }, + }]); + const request = usage.extensions[USAGE_ACCOUNTING_KEY].models[0].requests[0].usage; + assert.deepEqual(request, { input_tokens: 7, input_tokens_include_cache: true }); +}); diff --git a/adapters/python/deepagents/README.md b/adapters/python/deepagents/README.md index fdfb09f2e..af2f9fc34 100644 --- a/adapters/python/deepagents/README.md +++ b/adapters/python/deepagents/README.md @@ -241,6 +241,13 @@ The descriptor declares `nemo_fabric.usage_accounting.v1` for complete dimension observations and gaps, and `nemo_fabric.detailed_token_usage.v1` for raw cache and reasoning conventions. An omitted count stays unknown; a reported zero stays zero. +**Upgrade note:** For providers whose usage is visible only in LangGraph +messages, `RunResult.usage.input_tokens` and `output_tokens` can now be `null`. +The previous graph-message subtotal remains in `RunResult.output.usage`, labeled +`usage_source: graph_state_messages`. It is a diagnostic subtotal, not a +complete invocation count; consumers that previously read the top-level fields +should handle unknown counts rather than substituting this subtotal. + For adapter-created OpenAI-compatible clients, the collector observes sync and async calls and their HTTP attempts without changing requests or retries. Normal reported calls stay known. An actual failed or unfinished attempt without usable diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index f55fe8ba9..d47a7bdb1 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1229,7 +1229,7 @@ accounting remains available in `result.usage` and is not certified complete by `summarize_usage`. `cost.reported` contains only provider-reported amounts. Complete and observed -amounts use integer nano-USD. Catalog estimates remain unavailable in this slice; +amounts use integer nano-USD. Catalog estimates are not available; `cost.estimated` has no amounts and no typed pricing setting is exposed. The stable result shape retains optional pricing-basis fields for persisted evidence. From c038b361008beab1d78f90289de03fc1519a59f2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 21:57:07 -0700 Subject: [PATCH 127/159] fix: validate reported MCP transport facts Signed-off-by: mnajafian-nv --- crates/fabric-core/src/what_ran.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/fabric-core/src/what_ran.rs b/crates/fabric-core/src/what_ran.rs index 916efce53..12f0dcc91 100644 --- a/crates/fabric-core/src/what_ran.rs +++ b/crates/fabric-core/src/what_ran.rs @@ -1162,7 +1162,8 @@ fn apply_mcp_report(what_ran: &mut WhatRan, plan: &RunPlan, report: &Report<'_>) transport: entry .get("transport") .and_then(Value::as_str) - .map(str::to_lowercase), + .map(str::to_lowercase) + .filter(|transport| is_code(transport)), endpoint: Fact::missing( FactState::NotApplicable, REASON_NOT_CONFIGURED, @@ -2457,6 +2458,22 @@ mod tests { ); } + #[test] + fn long_reported_transport_is_discarded_without_dropping_its_server() { + let plan = plan_declaring(&[MCP_REPORT_KEY]); + let report = json!({"servers": { + "plugin-docs": {"transport": "X".repeat(60 * 1024), "runtime": {}} + }}); + + let (what_ran, _) = ingest(&plan, json!({MCP_REPORT_KEY: report})); + + assert!(what_ran["mcp"]["servers"]["plugin-docs"]["transport"].is_null()); + assert_eq!( + what_ran["mcp"]["servers"]["plugin-docs"]["origin"], + "harness" + ); + } + #[test] fn a_malformed_helper_marker_invalidates_tool_names() { let plan = plan_declaring(&[MCP_REPORT_KEY]); From 3fd1566c08cb7307af1005597fc42dc017f3f57e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:10:52 -0700 Subject: [PATCH 128/159] fix: skip execution-host pricing during inspection Signed-off-by: mnajafian-nv --- crates/fabric-core/src/config.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 19fc846d8..3d975e04f 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -2465,6 +2465,8 @@ pub fn inspect_adapter_config( ); } config.discovery = None; + // Inspection returns metadata only; catalog paths belong to the execution host. + config.pricing = None; validate_config(&config)?; let path = descriptor_path .canonicalize() @@ -4368,6 +4370,10 @@ mod tests { local_paths: vec![PathBuf::from("/task-only/nonexistent-descriptor")], ..Default::default() }); + config.pricing = Some(PricingConfig { + catalog_path: Some(PathBuf::from("/task-only/nonexistent-pricing-catalog.json")), + ..Default::default() + }); let inspection = inspect_adapter_config(config.clone(), &path).expect("host inspection"); assert_eq!( inspection.descriptor, From 6efff4d4e0345c9e2c5373d9874ffd48b995ffa2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:11:34 -0700 Subject: [PATCH 129/159] docs: use full product name in pricing references Signed-off-by: mnajafian-nv --- crates/fabric-core/src/price_catalog.rs | 4 ++-- .../nemo-fabric-core/price-catalog/index.mdx | 2 +- .../nemo-fabric-core/price-catalog/struct-modelpricing.mdx | 2 +- schemas/sdk/run-plan.schema.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs index f457829f1..108bb3563 100644 --- a/crates/fabric-core/src/price_catalog.rs +++ b/crates/fabric-core/src/price_catalog.rs @@ -4,7 +4,7 @@ //! Model price catalogs for estimating token cost. //! //! A catalog uses the NeMo Relay model pricing catalog format, version 1, so -//! one file can configure both tools. NeMo Fabric parses the format itself and +//! one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and //! does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. //! An explicit provider restricts aliases and model-path suffixes to that provider. //! Without a provider, model-only lookup uses the first matching entry in listed order. @@ -74,7 +74,7 @@ pub struct ModelPricing { /// Other identifiers of the same model, such as dated snapshots. #[serde(default)] pub aliases: Vec, - /// ISO 4217 currency of the rates. NeMo Fabric estimates cost only from + /// ISO 4217 currency of the rates. NVIDIA NeMo Fabric estimates cost only from /// `USD` entries. #[serde(default = "default_currency")] pub currency: String, diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx index c9aa13b29..07a60a689 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx @@ -12,7 +12,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Model price catalogs for estimating token cost. -A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, model-only lookup uses the first matching entry in listed order. +A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, model-only lookup uses the first matching entry in listed order. NeMo Fabric ships one catalog, embedded at build time, with the list prices of models that its presets, docs, and examples use. Each entry cites the provider pricing page it was taken from and the date it was checked. A caller can add a catalog of their own, whose entries take precedence. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx index 814233c4e..4b9be5cf7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-modelpricing.mdx @@ -30,7 +30,7 @@ Other identifiers of the same model, such as dated snapshots. ### `currency: String` -ISO 4217 currency of the rates. NeMo Fabric estimates cost only from `USD` entries. +ISO 4217 currency of the rates. NVIDIA NeMo Fabric estimates cost only from `USD` entries. ### `unit: PricingUnit` diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index ec8bcc2c7..aa7c4c4c7 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -1965,7 +1965,7 @@ }, "currency": { "default": "USD", - "description": "ISO 4217 currency of the rates. NeMo Fabric estimates cost only from\n`USD` entries.", + "description": "ISO 4217 currency of the rates. NVIDIA NeMo Fabric estimates cost only from\n`USD` entries.", "type": "string" }, "model_id": { From aef62ddcf6f23626f4220ff980bf9ec8e14c96a7 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:16:19 -0700 Subject: [PATCH 130/159] test: clean up recorded invocation fixtures Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 7dc0e379d..81225e0f2 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -7110,19 +7110,20 @@ for line in sys.stdin: } let (record, _) = only_record(&root); assert!(!record.contains("peak_request_input_tokens")); - let record_ref = result - .artifacts - .artifacts - .iter() - .find(|a| a.kind == "run_record") - .expect("record artifact"); - println!("ACCOUNTING_RECORD_RECEIPT={}", record_ref.path.display()); + assert!( + result + .artifacts + .artifacts + .iter() + .any(|a| a.kind == "run_record") + ); + let _ = fs::remove_dir_all(root); } } #[test] fn recorded_native_stream_publishes_the_returned_result() { - let (_, mut plan) = record_plan("success"); + let (root, mut plan) = record_plan("success"); plan.capabilities.streaming = true; plan.adapter_descriptor .as_mut() @@ -7147,5 +7148,6 @@ for line in sys.stdin: .expect("typed saved result"); assert_eq!(decoded, result); assert!(saved.path.with_file_name("run-record.jsonl").is_file()); + let _ = fs::remove_dir_all(root); } } From e7c495b55e295f1c9d34523deda4e430c0ef5e84 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:18:52 -0700 Subject: [PATCH 131/159] test: require exact Python API reference coverage Signed-off-by: mnajafian-nv --- tests/docs/test_python_api_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docs/test_python_api_docs.py b/tests/docs/test_python_api_docs.py index dbedf2b94..a71839aa9 100644 --- a/tests/docs/test_python_api_docs.py +++ b/tests/docs/test_python_api_docs.py @@ -192,7 +192,7 @@ def test_python_reference_exactly_covers_exported_sdk_classes() -> None: documented = { match.group(1) for match in FUNCTION_SIGNATURE.finditer(page.read_text(encoding="utf-8")) } - assert names <= documented + assert names == documented assert f'slug: "{MODULE_SLUGS[module]}"' in page.read_text(encoding="utf-8") From 835598987388966bcd8195f711b4824dba92858e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 22:56:59 -0700 Subject: [PATCH 132/159] fix: avoid repeating a timed-out Hermes version probe Signed-off-by: mnajafian-nv --- .../nemo_fabric_adapters/hermes/adapter.py | 4 --- tests/adapters/test_hermes_adapter.py | 30 ++++++++++++++++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py index 54ae775d7..0f280a386 100755 --- a/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py +++ b/adapters/python/hermes/src/nemo_fabric_adapters/hermes/adapter.py @@ -458,10 +458,6 @@ def clear_active_invoke_task( code="hermes_reported_failure", message="Hermes did not complete the invocation", ) - if self._harness_version.get("reason") == "timeout": - # Pick up a version call that finished after its limit. While it - # still runs, the probe starts no new one. - self._harness_version = await probe_hermes_version(self._version_probe) return AgentRunResult( status=AgentRunStatus.FAILED if failed else AgentRunStatus.SUCCEEDED, output=output, diff --git a/tests/adapters/test_hermes_adapter.py b/tests/adapters/test_hermes_adapter.py index c200cc0ce..3bd123d22 100644 --- a/tests/adapters/test_hermes_adapter.py +++ b/tests/adapters/test_hermes_adapter.py @@ -17,7 +17,7 @@ from io import StringIO from pathlib import Path from types import ModuleType -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import pytest from nemo_fabric_adapter_contract.models import AgentConfig @@ -1849,6 +1849,34 @@ def blocking() -> str: assert calls == ["called"] +async def test_timeout_version_probe_is_not_repeated_after_turn(monkeypatch): + runtime = adapter.HermesRuntime() + runtime._started = True + runtime._runtime_id = "runtime-version-timeout" + runtime._agent_config = _agent_config( + {"models": {"default": {"provider": "test", "model": "test-model"}}} + ) + runtime._model_config = runtime._agent_config.models["default"] + runtime._agent = MagicMock() + runtime._mcp_authentication_checked = True + runtime._harness_version = _version_state("collection_failed", "timeout") + monkeypatch.setattr( + adapter, + "_invoke_hermes_turn", + lambda **_kwargs: ({"completed": True, "final_response": "done"}, ""), + ) + probe = AsyncMock(return_value=_version_state("collection_failed", "timeout")) + monkeypatch.setattr(adapter, "probe_hermes_version", probe) + + context = _runtime_context(runtime_id=runtime._runtime_id) + for message in ("first", "second"): + result = await runtime.invoke(AgentRunRequest(input=message), context) + assert result.extensions["nemo_fabric.versions.v1"]["harness"]["version"] == ( + _version_state("collection_failed", "timeout") + ) + probe.assert_not_awaited() + + def test_hermes_version_keeps_the_commit_and_dirty_marker(monkeypatch): version_info = ModuleType("hermes_cli.version_info") version_info.get_version_info = lambda: MagicMock( # type: ignore[attr-defined] From f2235969e98a95623211f3d7def0b22aa6bd52fc Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 22 Oct 2026 23:23:16 -0700 Subject: [PATCH 133/159] fix: describe execution fact truncation accurately Signed-off-by: mnajafian-nv --- sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py | 2 +- tests/python/test_invocation_reporting.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py index 7b5fb9b43..ceef3ca7a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py @@ -545,7 +545,7 @@ def _render_provenance( if what_ran is None: return [] if what_ran.get("what_ran_truncated") is not None: - return ["Execution facts: record exceeded its byte budget."] + return ["Execution facts were truncated to fit a byte budget."] lines = [] def value(fact: Any) -> str | None: diff --git a/tests/python/test_invocation_reporting.py b/tests/python/test_invocation_reporting.py index 27b363464..a4f29dfd6 100644 --- a/tests/python/test_invocation_reporting.py +++ b/tests/python/test_invocation_reporting.py @@ -627,7 +627,7 @@ def test_minimal_execution_facts_show_byte_budget_omission(result): } summary = summarize_result(result) assert summary["what_ran"] == result["what_ran"] - assert "Execution facts: record exceeded its byte budget." in render_result_summary( + assert "Execution facts were truncated to fit a byte budget." in render_result_summary( summary ) From 11f72a48ccae09da27dd424bd8879c5815b6f65f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 00:03:45 -0700 Subject: [PATCH 134/159] fix: enforce Relay gateway startup deadline Signed-off-by: mnajafian-nv --- .../common/relay_gateway.py | 2 ++ .../test_adapters_common_relay_gateway.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 1a57c5fe4..f0e6d5cd3 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -137,6 +137,8 @@ def wait_for_relay_gateway( raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) + if time.monotonic() >= deadline: + break return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index fbd9ca893..957b2f967 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -252,6 +252,24 @@ def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") +def test_wait_for_relay_gateway_rejects_success_after_deadline(monkeypatch, gateway_clock): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += 0.26 + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From 4f7e551ca0845920074f81b88676b79f1bcf9685 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 00:27:40 -0700 Subject: [PATCH 135/159] fix: preserve lifecycle stage for recorded one-shot failures Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 13 +++++++++++-- crates/fabric-python/src/lib.rs | 29 +++++++++++++++++------------ tests/python/test_recording.py | 16 +++++++++++++++- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 81225e0f2..db9d90fed 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -898,12 +898,21 @@ enum LocalServiceState { /// Invoke a NeMo Fabric run plan. pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { - let runtime = start_runtime(plan)?; + run_plan_with_stage(plan, request).map_err(|(error, _)| error) +} + +/// Preserve the phase of a one-shot failure for language bindings. +#[doc(hidden)] +pub fn run_plan_with_stage( + plan: &RunPlan, + request: RunRequest, +) -> std::result::Result { + let runtime = start_runtime(plan).map_err(|error| (error, ErrorStage::Start))?; let mut result = match invoke_runtime_unpersisted(plan, &runtime, request) { Ok(result) => result, Err(error) => { let _ = stop_runtime(plan, &runtime); - return Err(error); + return Err((error, ErrorStage::Invoke)); } }; match stop_runtime(plan, &runtime) { diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index ede9a2eb8..a6e013d7f 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -10,9 +10,9 @@ use std::thread; use std::time::{Duration, Instant}; use nemo_fabric_core::{ - EnvironmentHandle, EnvironmentReference, FabricConfig, FabricError, OpenAiStreamTransport, - ResolveContext, RunPlan, RunRequest, RuntimeHandle, ServiceHandle, ServiceReference, - doctor_plan, inspect_adapter_config, + EnvironmentHandle, EnvironmentReference, ErrorStage, FabricConfig, FabricError, + OpenAiStreamTransport, ResolveContext, RunPlan, RunRequest, RuntimeHandle, ServiceHandle, + ServiceReference, doctor_plan, inspect_adapter_config, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config_with_adapter_directories, run_plan, }; @@ -157,18 +157,23 @@ fn run_config( fn run_resolved_plan(py: Python<'_>, plan_json: String, request_json: String) -> PyResult { let plan = parse_run_plan(plan_json)?; let request = parse_run_request(request_json)?; - let result = match py.detach(|| run_plan(&plan, request)) { + let result = match py.detach(|| nemo_fabric_core::runtime::run_plan_with_stage(&plan, request)) + { Ok(result) => result, - Err(error) => { - let stage = match &error { - FabricError::AdapterLifecycleOperation { operation, .. } => Some(*operation), - _ => None, + Err((error, stage)) => { + let stage = match stage { + ErrorStage::Config => "config", + ErrorStage::Plan => "plan", + ErrorStage::Prepare => "prepare", + ErrorStage::Start => "start", + ErrorStage::Invoke => "invoke", + ErrorStage::Stop => "stop", + ErrorStage::Release => "release", + ErrorStage::Artifact => "artifact", }; let failure = to_py_error(error); - if let Some(stage) = stage { - // Preserve lifecycle stage without parsing diagnostic text. - failure.value(py).setattr("fabric_stage", stage)?; - } + // Preserve lifecycle stage without parsing diagnostic text. + failure.value(py).setattr("fabric_stage", stage)?; return Err(failure); } }; diff --git a/tests/python/test_recording.py b/tests/python/test_recording.py index dab7170bc..471acb785 100644 --- a/tests/python/test_recording.py +++ b/tests/python/test_recording.py @@ -11,7 +11,7 @@ import pytest from pydantic import ValidationError -from nemo_fabric import RecordConfig +from nemo_fabric import Fabric, RecordConfig from nemo_fabric.errors import FabricRuntimeError from nemo_fabric.runtime import _run_native_lifecycle @@ -91,6 +91,20 @@ async def test_recorded_failure_preserves_native_lifecycle_stage(recorded_plan): assert caught.value.__cause__ is original +async def test_recorded_process_launch_failure_retains_start_stage( + blocking_adapter_config, tmp_path +): + descriptor_path = tmp_path / "blocking.fabric-adapter.json" + descriptor = json.loads(descriptor_path.read_text()) + descriptor["runner"]["command"] = str(tmp_path / "missing-runner") + descriptor_path.write_text(json.dumps(descriptor)) + blocking_adapter_config.runtime.record = RecordConfig(enabled=True) + + with pytest.raises(FabricRuntimeError) as caught: + await Fabric().run(blocking_adapter_config, base_dir=tmp_path, input="hello") + assert caught.value.stage == "start" + + async def test_recorded_cancellation_keeps_cancellation_and_evidence_location( recorded_plan, tmp_path ): From e3a2355b585e1eb231a7395cb1f4e7945c76d12b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 01:41:33 -0700 Subject: [PATCH 136/159] fix: classify empty usage coverage as unverified Signed-off-by: mnajafian-nv --- .../python/src/nemo_fabric_adapter_contract/usage.py | 2 +- adapter-contract/typescript/src/usage.ts | 2 +- adapter-contract/typescript/test/usage-reducer.test.js | 4 ++++ tests/fixtures/usage_accounting/reducer-cases.json | 10 ++++++++++ tests/python/test_usage_reducer_parity.py | 3 +++ 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py index d28998d36..f296d5d21 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/usage.py @@ -79,7 +79,7 @@ def _reduce( if total is not None and not causes: complete[key] = total elif not causes: - causes.add(GapReason.MISSING_USAGE) + causes.add(GapReason.SCOPE_UNVERIFIED if not records else GapReason.MISSING_USAGE) for cause in sorted(causes): gaps[cause].append(dimension) for flag, counter in ( diff --git a/adapter-contract/typescript/src/usage.ts b/adapter-contract/typescript/src/usage.ts index 16c8cc5ae..bf75221c4 100644 --- a/adapter-contract/typescript/src/usage.ts +++ b/adapter-contract/typescript/src/usage.ts @@ -111,7 +111,7 @@ function reduce(records: readonly UsageObservation[]): { reported: ReportedUsage if (values.length !== records.length) causes.add("missing_usage"); if (values.length && value === undefined) causes.add("scope_unverified"); if (value !== undefined && !causes.size) reported[key] = value; - else if (!causes.size) causes.add("missing_usage"); + else if (!causes.size) causes.add(records.length ? "missing_usage" : "scope_unverified"); for (const reason of causes) gap(reason, key); } for (const [flag, counter] of [["input_tokens_include_cache", "input_tokens"], ["output_tokens_include_reasoning", "output_tokens"]] as const) { diff --git a/adapter-contract/typescript/test/usage-reducer.test.js b/adapter-contract/typescript/test/usage-reducer.test.js index de7fbdfb2..4c6e7040c 100644 --- a/adapter-contract/typescript/test/usage-reducer.test.js +++ b/adapter-contract/typescript/test/usage-reducer.test.js @@ -12,6 +12,10 @@ for (const vector of cases) { const usage = usageFromObservations(vector.records); const accounting = usage.extensions[USAGE_ACCOUNTING_KEY]; for (const [field, value] of Object.entries(vector.expected)) { + if (field === "gap_reason") { + assert.deepEqual(accounting.gaps.map(gap => gap.reason), [value]); + continue; + } const actual = field === "exhaustive" ? accounting.models_exhaustive : field.startsWith("observed_") ? accounting.observed[field.slice(9) + "_tokens"] : field === "peak_request_input_tokens" ? usage.extensions["nemo_fabric.detailed_token_usage.v1"][field] diff --git a/tests/fixtures/usage_accounting/reducer-cases.json b/tests/fixtures/usage_accounting/reducer-cases.json index cba63db2e..39b4c4959 100644 --- a/tests/fixtures/usage_accounting/reducer-cases.json +++ b/tests/fixtures/usage_accounting/reducer-cases.json @@ -1,4 +1,14 @@ [ + { + "name": "empty_observations", + "records": [], + "expected": { + "input_tokens": null, + "output_tokens": null, + "exhaustive": false, + "gap_reason": "scope_unverified" + } + }, { "name": "complete_calls", "records": [ diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py index c305b4b33..ccdf1628c 100644 --- a/tests/python/test_usage_reducer_parity.py +++ b/tests/python/test_usage_reducer_parity.py @@ -51,6 +51,9 @@ def test_python_reducer_matches_shared_independent_expectations(reduced): for vector, usage in zip(VECTORS, reduced, strict=True): accounting = usage["extensions"][KEY] for field, value in vector["expected"].items(): + if field == "gap_reason": + assert [gap["reason"] for gap in accounting["gaps"]] == [value] + continue if field == "exhaustive": actual = accounting["models_exhaustive"] elif field.startswith("observed_"): From a8f6fd807b9a2dc5cd3ae3f1b01538aec4820a7b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 02:05:52 -0700 Subject: [PATCH 137/159] test: account for unverified empty usage coverage Signed-off-by: mnajafian-nv --- tests/adapters/test_deepagents_accounting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/adapters/test_deepagents_accounting.py b/tests/adapters/test_deepagents_accounting.py index dccaf29a3..4b674815a 100644 --- a/tests/adapters/test_deepagents_accounting.py +++ b/tests/adapters/test_deepagents_accounting.py @@ -210,7 +210,8 @@ async def test_remote_tool_callback_dispatch_marks_new_work_only_unit( dispatched = operation in {"start_async_task", "update_async_task"} assert result.input_tokens == (100 if foreground_call and not dispatched else None) reasons = {gap["reason"] for gap in result.extensions[KEY]["gaps"]} - assert ("scope_unverified" in reasons) is dispatched + # With no foreground call, an empty callback list cannot prove zero usage. + assert ("scope_unverified" in reasons) is (dispatched or not foreground_call) if foreground_call: assert result.extensions[KEY]["observed"]["input_tokens"] == 100 elif dispatched: From 46e7ccfa8cdc774e41142b00dd289e53ae6103e3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 02:53:46 -0700 Subject: [PATCH 138/159] docs: clarify providerless pricing lookup order Signed-off-by: mnajafian-nv --- crates/fabric-core/src/price_catalog.rs | 6 ++++-- .../nemo-fabric-core/price-catalog/index.mdx | 2 +- .../price-catalog/struct-pricingcatalog.mdx | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs index 108bb3563..a7473c119 100644 --- a/crates/fabric-core/src/price_catalog.rs +++ b/crates/fabric-core/src/price_catalog.rs @@ -7,7 +7,9 @@ //! one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and //! does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. //! An explicit provider restricts aliases and model-path suffixes to that provider. -//! Without a provider, model-only lookup uses the first matching entry in listed order. +//! Without a provider, lookup tries provider-qualified keys from the model path +//! before model-only suffixes. For each candidate key, the first matching +//! catalog entry wins. //! //! NeMo Fabric ships one catalog, embedded at build time, with the list prices //! of models that its presets, docs, and examples use. Each entry cites the @@ -59,7 +61,7 @@ struct BundledCatalog { pub struct PricingCatalog { /// Catalog format version. Must be `1`. pub version: u32, - /// Price entries. Lookup uses the first entry that matches. + /// Price entries. For each lookup key, the first matching entry wins. pub entries: Vec, } diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx index 07a60a689..44a339036 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/index.mdx @@ -12,7 +12,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Model price catalogs for estimating token cost. -A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, model-only lookup uses the first matching entry in listed order. +A catalog uses the NeMo Relay model pricing catalog format, version 1, so one file can configure both tools. NVIDIA NeMo Fabric parses the format itself and does not depend on NeMo Relay. Names are trimmed and compared case-insensitively. An explicit provider restricts aliases and model-path suffixes to that provider. Without a provider, lookup tries provider-qualified keys from the model path before model-only suffixes. For each candidate key, the first matching catalog entry wins. NeMo Fabric ships one catalog, embedded at build time, with the list prices of models that its presets, docs, and examples use. Each entry cites the provider pricing page it was taken from and the date it was checked. A caller can add a catalog of their own, whose entries take precedence. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx index 01d5d4765..027636e72 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx @@ -24,7 +24,7 @@ Catalog format version. Must be `1`. ### `entries: Vec` -Price entries. Lookup uses the first entry that matches. +Price entries. For each lookup key, the first matching entry wins. ## Implementations From 05132d83996be4487d8e1aeabdaa0a36b8611dca Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 03:17:41 -0700 Subject: [PATCH 139/159] docs: regenerate pricing run-plan schema Signed-off-by: mnajafian-nv --- schemas/sdk/run-plan.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index aa7c4c4c7..786559ed2 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -2050,7 +2050,7 @@ "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", "properties": { "entries": { - "description": "Price entries. Lookup uses the first entry that matches.", + "description": "Price entries. For each lookup key, the first matching entry wins.", "items": { "$ref": "#/$defs/ModelPricing" }, From 85100cfec65cf021090ee896a7ce0a54ed13a88c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 07:49:53 -0700 Subject: [PATCH 140/159] docs: clarify pricing references and usage summary guidance Signed-off-by: mnajafian-nv --- crates/fabric-core/src/price_catalog.rs | 2 +- .../nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx | 2 +- schemas/sdk/run-plan.schema.json | 2 +- skills/nemo-fabric-integrate/references/sdk-api-inventory.md | 4 +++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/fabric-core/src/price_catalog.rs b/crates/fabric-core/src/price_catalog.rs index a7473c119..97a646139 100644 --- a/crates/fabric-core/src/price_catalog.rs +++ b/crates/fabric-core/src/price_catalog.rs @@ -54,7 +54,7 @@ struct BundledCatalog { /// A model price catalog in the NeMo Relay model pricing catalog format, /// version 1. /// -/// NeMo Fabric reads and writes the format unchanged, with one extra check: +/// NVIDIA NeMo Fabric reads and writes the format unchanged, with one extra check: /// every rate must be below one million US dollars per million tokens. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx index 027636e72..4b180e69b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/price-catalog/struct-pricingcatalog.mdx @@ -14,7 +14,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. A model price catalog in the NeMo Relay model pricing catalog format, version 1. -NeMo Fabric reads and writes the format unchanged, with one extra check: every rate must be below one million US dollars per million tokens. +NVIDIA NeMo Fabric reads and writes the format unchanged, with one extra check: every rate must be below one million US dollars per million tokens. ## Fields diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index 786559ed2..7016616fd 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -2047,7 +2047,7 @@ }, "PricingCatalog": { "additionalProperties": false, - "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", + "description": "A model price catalog in the NeMo Relay model pricing catalog format,\nversion 1.\n\nNVIDIA NeMo Fabric reads and writes the format unchanged, with one extra check:\nevery rate must be below one million US dollars per million tokens.", "properties": { "entries": { "description": "Price entries. For each lookup key, the first matching entry wins.", diff --git a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md index 6bfe68637..6b83beb23 100644 --- a/skills/nemo-fabric-integrate/references/sdk-api-inventory.md +++ b/skills/nemo-fabric-integrate/references/sdk-api-inventory.md @@ -90,10 +90,12 @@ another turn. For native OpenAI streaming, `result()` also finalizes and discards unread chunks. Relay `InvokeStream.result()` does not consume unread ATOF records. -## Execution Model +## Usage Summaries `summarize_usage(results, *, expected=...)` reduces normalized `RunResult.model_usage` with an explicit inventory of expected `(runtime_id, invocation_id)` pairs. It returns complete totals, observed sums and coverage without certifying legacy counters or silently dropping missing results. See [results and errors](results-and-errors.md) for field semantics and a single-invocation example. +## Execution Model + NVIDIA NeMo Fabric separates configuration, planning, runtime lifecycle, and individual invocations: From 4663e407c49e7507c9e5faba4a6f3e56b9093443 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 08:31:23 -0700 Subject: [PATCH 141/159] fix: preserve one-shot startup error stages Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 46 +++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index db9d90fed..b38e3c30d 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -907,7 +907,7 @@ pub fn run_plan_with_stage( plan: &RunPlan, request: RunRequest, ) -> std::result::Result { - let runtime = start_runtime(plan).map_err(|error| (error, ErrorStage::Start))?; + let runtime = start_runtime_staged(plan)?; let mut result = match invoke_runtime_unpersisted(plan, &runtime, request) { Ok(result) => result, Err(error) => { @@ -1191,15 +1191,24 @@ fn validate_start_plan(plan: &RunPlan) -> Result<()> { /// For a non-local provider, prepare or attach to the environment explicitly /// and call [`start_runtime_in`] with the returned handle. pub fn start_runtime(plan: &RunPlan) -> Result { - validate_start_plan(plan)?; + start_runtime_staged(plan).map_err(|(error, _)| error) +} + +fn start_runtime_staged( + plan: &RunPlan, +) -> std::result::Result { + validate_start_plan(plan).map_err(|error| (error, ErrorStage::Plan))?; let provider = planned_environment_provider(plan); if provider != "local" { - return Err(FabricError::EnvironmentHandleRequired { - provider: provider.to_string(), - }); + return Err(( + FabricError::EnvironmentHandleRequired { + provider: provider.to_string(), + }, + ErrorStage::Prepare, + )); } - let environment = prepare_environment(plan)?; - start_runtime_in_validated(plan, &environment) + let environment = prepare_environment(plan).map_err(|error| (error, ErrorStage::Prepare))?; + start_runtime_in_validated(plan, &environment).map_err(|error| (error, ErrorStage::Start)) } /// Invoke a started harness runtime. @@ -5758,7 +5767,9 @@ for line in sys.stdin: fn local_host_start_failure_preserves_stage_and_diagnostics() { let (root, plan) = local_host_plan("start_failure"); - let error = start_runtime(&plan).expect_err("start must fail"); + let (error, stage) = + run_plan_with_stage(&plan, RunRequest::text("input")).expect_err("start must fail"); + assert_eq!(stage, ErrorStage::Start); let message = error.to_string(); assert!(message.contains("lifecycle start"), "{message}"); assert!(message.contains("fake_start"), "{message}"); @@ -5767,6 +5778,25 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + fn one_shot_reports_prepare_stage_when_environment_is_required() { + let (root, mut plan) = local_host_plan("success"); + plan.environment_plan + .as_mut() + .expect("resolved environment plan") + .provider = "docker".to_string(); + + let (error, stage) = run_plan_with_stage(&plan, RunRequest::text("input")) + .expect_err("non-local provider needs an environment handle"); + assert!(matches!( + error, + FabricError::EnvironmentHandleRequired { .. } + )); + assert_eq!(stage, ErrorStage::Prepare); + + let _ = fs::remove_dir_all(root); + } + #[test] fn local_host_supplies_fresh_adapter_deadlines() { let (root, mut plan) = local_host_plan("success"); From 83d8adace3ca3dd886c8769ba1cdc1a01d3a7c3b Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 08:55:34 -0700 Subject: [PATCH 142/159] docs: keep adapter usage guidance within current contract Signed-off-by: mnajafian-nv --- skills/nemo-fabric-build-adapter/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/nemo-fabric-build-adapter/SKILL.md b/skills/nemo-fabric-build-adapter/SKILL.md index 064429506..7b014b3a7 100644 --- a/skills/nemo-fabric-build-adapter/SKILL.md +++ b/skills/nemo-fabric-build-adapter/SKILL.md @@ -231,7 +231,7 @@ never share mutable continuation state. Normalize usage per invocation in `AgentUsage`. If the target reports cumulative session totals, retain a session-local baseline and difference successive observed counters rather than summing cumulative snapshots or using only the last model response. Missing or reset counters remain unknown. Keep the closed `fabric.adapter/v1alpha2` usage fields unchanged. Put cache-write, reasoning, peak request, and per-model detail in `usage.extensions["nemo_fabric.detailed_token_usage.v1"]`, and declare that object with `extension_schemas.usage` in the adapter descriptor. The detailed object repeats the normalized token counters so its inclusion flags are self-contained; `input_tokens_include_cache` covers both cached reads and cache writes there. NeMo Fabric preserves the object as `RunResult.usage.metadata["nemo_fabric.detailed_token_usage.v1"]`. Consumers treat any detailed field for an input or output dimension as selecting the detailed counters for that dimension without falling back to its top-level counters. Report the primary counter, inclusion flag, and every excluded component together so consumers can calculate an inclusive total. Preserve available usage on unsuccessful terminal results. Do not infer missing cost or promote estimates to `cost_usd`. -Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts and price them. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. +Preserve field presence: an omitted count stays unknown and a provider-reported zero stays zero. If the target fills missing fields with zero, recover original report presence before certifying a complete count. Capture each call's invocation owner when the call starts, including descendant work. Freeze its usage snapshot when `invoke` returns; late reports cannot amend that result or transfer into a later invocation. A failed execution alone is not an accounting gap: keep settled usage, and distinguish valid observations from complete totals when owned work is missing or unfinished. Use the shared Python `usage_from_observations` or TypeScript `usageFromObservations` reducer with the declared `nemo_fabric.usage_accounting.v1` extension for this evidence. Keep source-specific parsing and lifecycle ownership in the adapter; let core normalize counts. See the [results contract](https://github.com/NVIDIA/NeMo-Fabric/blob/main/docs/adapter-contract/tutorials/results.md) for the extension and compatibility boundary. Test observable continuation rather than merely calling `invoke` twice: make the second result depend on the first turn without caller-side replay, then From ebcfe97c48bcc90e93f2ae74078f1b82e9049132 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 10:26:13 -0700 Subject: [PATCH 143/159] test: run OpenShell launcher checks with system Bash Use the system Bash path so macOS tests exercise Bash 3.2 even when PATH contains a newer Bash. This keeps the empty-array regression meaningful in CI. Signed-off-by: mnajafian-nv --- tests/scripts/test_openshell_launcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/test_openshell_launcher.py b/tests/scripts/test_openshell_launcher.py index 15ab68caa..f54157496 100644 --- a/tests/scripts/test_openshell_launcher.py +++ b/tests/scripts/test_openshell_launcher.py @@ -165,7 +165,7 @@ def _operations(root: Path) -> list[dict]: def _run(launcher, workload, env): script, _, base_env = launcher return subprocess.run( - ["bash", str(script), str(workload), "--input", "words with spaces"], + ["/bin/bash", str(script), str(workload), "--input", "words with spaces"], env={**base_env, **env}, text=True, capture_output=True, @@ -385,7 +385,7 @@ def test_invalid_platform_or_mode_stops_before_building(launcher, environment, m "stage_workload_image() { :; }\nrun_workload_consumer() { :; }\n" ) completed = subprocess.run( - ["bash", str(script), str(workload)], + ["/bin/bash", str(script), str(workload)], env={**base_env, **environment}, text=True, capture_output=True, From 8db3c576f7d56d8a86d69c6289d8c0169d6120c2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 10:31:36 -0700 Subject: [PATCH 144/159] fix: preserve recorded startup stages in Python errors Signed-off-by: mnajafian-nv --- .../nemo-fabric-runtime/src/nemo_fabric/runtime.py | 2 +- tests/python/test_recording.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index fa20cf8a2..b8f15423c 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -1018,7 +1018,7 @@ async def _run_native_lifecycle( raise except Exception as error: stage = getattr(error, "fabric_stage", "run") - if stage not in {"start", "invoke", "stop"}: + if stage not in {"plan", "prepare", "start", "invoke", "stop"}: stage = "run" failure = _runtime_error(error, stage=stage) root = _record_root(plan, runtime_config) diff --git a/tests/python/test_recording.py b/tests/python/test_recording.py index 471acb785..92af558b6 100644 --- a/tests/python/test_recording.py +++ b/tests/python/test_recording.py @@ -80,14 +80,15 @@ def test_recording_rejects_unrecognized_settings(): RecordConfig(enabled=True, retries=2) -async def test_recorded_failure_preserves_native_lifecycle_stage(recorded_plan): - original = RuntimeError("native start error") - original.fabric_stage = "start" +@pytest.mark.parametrize("stage", ["plan", "prepare", "start", "invoke", "stop"]) +async def test_recorded_failure_preserves_native_lifecycle_stage(recorded_plan, stage): + original = RuntimeError("native lifecycle error") + original.fabric_stage = stage native = MagicMock(spec=["run_plan"]) native.run_plan.side_effect = original with pytest.raises(FabricRuntimeError) as caught: await _run_native_lifecycle(native, recorded_plan, {"request_id": "req"}) - assert caught.value.stage == "start" + assert caught.value.stage == stage assert caught.value.__cause__ is original From 83b5e76b8b5f3629e7ef9215bd5be505e3a13298 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 15:07:21 -0700 Subject: [PATCH 145/159] fix: reject Relay readiness after the startup deadline Signed-off-by: mnajafian-nv --- .../common/relay_gateway.py | 2 ++ .../test_adapters_common_relay_gateway.py | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py index 1a57c5fe4..f0e6d5cd3 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/relay_gateway.py @@ -137,6 +137,8 @@ def wait_for_relay_gateway( raise RelayGatewayError( f"NeMo Relay gateway exited with status {returncode} before becoming ready" ) + if time.monotonic() >= deadline: + break return except (OSError, urllib.error.URLError): pass diff --git a/tests/adapters/test_adapters_common_relay_gateway.py b/tests/adapters/test_adapters_common_relay_gateway.py index fbd9ca893..86c23ede2 100644 --- a/tests/adapters/test_adapters_common_relay_gateway.py +++ b/tests/adapters/test_adapters_common_relay_gateway.py @@ -252,6 +252,26 @@ def test_wait_for_relay_gateway_rejects_exit_during_successful_probe(monkeypatch relay_gateway.wait_for_relay_gateway(process, "http://127.0.0.1:43210/healthz") +def test_wait_for_relay_gateway_rejects_success_after_deadline( + monkeypatch, gateway_clock +): + process = MagicMock() + process.poll.return_value = None + response = MagicMock() + response.__enter__.return_value.status = 200 + + def probe_health(url, *, timeout): + gateway_clock.monotonic.return_value += 0.26 + return response + + monkeypatch.setattr(relay_gateway.urllib.request, "urlopen", probe_health) + + with pytest.raises(relay_gateway.RelayGatewayError, match="did not become ready"): + relay_gateway.wait_for_relay_gateway( + process, "http://127.0.0.1:43210/healthz", timeout=0.25 + ) + + def test_wait_for_relay_gateway_times_out(): process = MagicMock() process.poll.return_value = None From 2149bd1a0e98d996f64065724c3658e6ea43f06d Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 16:29:34 -0700 Subject: [PATCH 146/159] fix: bound Pi Relay readiness checks Signed-off-by: mnajafian-nv --- adapters/typescript/pi/src/relay-gateway.ts | 31 +++++++-- adapters/typescript/pi/test/relay.test.mjs | 71 +++++++++++++++++++++ 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/adapters/typescript/pi/src/relay-gateway.ts b/adapters/typescript/pi/src/relay-gateway.ts index 163bcac09..99446bf14 100644 --- a/adapters/typescript/pi/src/relay-gateway.ts +++ b/adapters/typescript/pi/src/relay-gateway.ts @@ -147,26 +147,43 @@ export async function waitForRelayGateway( const fetchRequest = options.fetch ?? fetch; const pollIntervalMs = options.pollIntervalMs ?? 100; const deadline = performance.now() + timeoutMs; - while (performance.now() < deadline) { + while (true) { + const remaining = deadline - performance.now(); + if (remaining <= 0) { + break; + } if (processExited(child)) { throw new RelayGatewayError( `NeMo Relay gateway exited with status ${child.exitCode ?? child.signalCode} before becoming ready`, ); } + let ready = false; try { const response = await fetchRequest(healthUrl, { method: "HEAD", - signal: AbortSignal.timeout(1_000), + signal: AbortSignal.timeout(Math.min(1_000, Math.ceil(remaining))), }); - const ready = response.ok; await response.body?.cancel(); - if (ready) { - return; - } + ready = response.ok; } catch { // Retry until the gateway exits or the readiness deadline expires. } - await delay(Math.min(pollIntervalMs, Math.max(0, deadline - performance.now()))); + if (ready) { + if (processExited(child)) { + throw new RelayGatewayError( + `NeMo Relay gateway exited with status ${child.exitCode ?? child.signalCode} before becoming ready`, + ); + } + if (performance.now() >= deadline) { + break; + } + return; + } + const remainingAfterProbe = deadline - performance.now(); + if (remainingAfterProbe <= 0) { + break; + } + await delay(Math.min(pollIntervalMs, remainingAfterProbe)); } throw new RelayGatewayError(`NeMo Relay gateway did not become ready at ${healthUrl}`); } diff --git a/adapters/typescript/pi/test/relay.test.mjs b/adapters/typescript/pi/test/relay.test.mjs index ee0d541d5..b77e3c459 100644 --- a/adapters/typescript/pi/test/relay.test.mjs +++ b/adapters/typescript/pi/test/relay.test.mjs @@ -844,6 +844,77 @@ test("drains unsuccessful and successful Relay health responses", async () => { assert.deepEqual(cancelled, [0, 1]); }); +test("rejects a gateway exit during a successful health probe", async (t) => { + for (const boundary of ["fetch", "body cancellation"]) { + await t.test(boundary, async () => { + const child = new MockChild(); + await assert.rejects( + waitForRelayGateway(child, "http://127.0.0.1:41000/healthz", { + async fetch() { + if (boundary === "fetch") child.exitCode = 17; + return { + ok: true, + body: { + async cancel() { + if (boundary === "body cancellation") child.exitCode = 17; + }, + }, + }; + }, + }), + /exited with status 17 before becoming ready/u, + ); + }); + } +}); + +test("rejects a successful health probe at the readiness deadline", async (t) => { + for (const boundary of ["fetch", "body cancellation"]) { + await t.test(boundary, async (subtest) => { + let now = 0; + subtest.mock.method(performance, "now", () => now); + await assert.rejects( + waitForRelayGateway(new MockChild(), "http://127.0.0.1:41000/healthz", { + timeoutMs: 25, + async fetch() { + if (boundary === "fetch") now = 25; + return { + ok: true, + body: { + async cancel() { + if (boundary === "body cancellation") now = 25; + }, + }, + }; + }, + }), + /did not become ready/u, + ); + }); + } +}); + +test("limits a health probe to the remaining readiness budget", async (t) => { + let now = 0; + t.mock.method(performance, "now", () => now); + const timeouts = []; + t.mock.method(AbortSignal, "timeout", (milliseconds) => { + timeouts.push(milliseconds); + return new AbortController().signal; + }); + await assert.rejects( + waitForRelayGateway(new MockChild(), "http://127.0.0.1:41000/healthz", { + timeoutMs: 25, + async fetch() { + now = 25; + throw new Error("still initializing"); + }, + }), + /did not become ready/u, + ); + assert.deepEqual(timeouts, [25]); +}); + test("escalates gateway shutdown from terminate to kill and is safe after exit", async () => { const mockChild = new MockChild(); mockChild.kill = function (signal) { From 56e59c7c7e75e904eb2f65821e9d86489b6db7b3 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Fri, 23 Oct 2026 17:00:26 -0700 Subject: [PATCH 147/159] test: allow zero coverage in resource sampling checks Sparse valid readings can produce zero covered time. Accept the full coverage range while preserving spike, artifact and result checks. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 3 ++- tests/python/test_system_stats.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 042777da9..cbd1d35b9 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -5286,7 +5286,8 @@ for line in sys.stdin: "peak was lost after worker exit: {summary:?}" ); assert!(summary.sample_count >= 3); - assert!(gauge.covered_fraction > 0.0); + // A spike sample need not have a neighbor within twice the interval. + assert!((0.0..=1.0).contains(&gauge.covered_fraction)); let path = &summary.samples.as_ref().unwrap().path; let rows: Vec = fs::read_to_string(path) .unwrap() diff --git a/tests/python/test_system_stats.py b/tests/python/test_system_stats.py index 3b89672e5..950df16a3 100644 --- a/tests/python/test_system_stats.py +++ b/tests/python/test_system_stats.py @@ -85,7 +85,8 @@ async def test_sdk_returns_typed_samples_and_artifacts_without_relay_or_recordin sampled.resident_memory_bytes.sampled_peak >= sampled.resident_memory_bytes.final_value ) - assert 0 < sampled.resident_memory_bytes.covered_fraction <= 1 + # Gaps above twice the sampling interval do not contribute coverage. + assert 0 <= sampled.resident_memory_bytes.covered_fraction <= 1 assert ( RunResult.from_mapping(result.to_mapping()).system_stats == result.system_stats ) From 008ac1d3f3d2dccce7879c92ca1606c06d605610 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sun, 25 Oct 2026 22:26:02 -0700 Subject: [PATCH 148/159] docs: clarify when Relay readiness timing starts Signed-off-by: mnajafian-nv --- adapters/python/codex/README.md | 7 ++++--- adapters/python/common/README.md | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/adapters/python/codex/README.md b/adapters/python/codex/README.md index ba7eab1b7..639e50b7a 100644 --- a/adapters/python/codex/README.md +++ b/adapters/python/codex/README.md @@ -154,9 +154,10 @@ Codex adapter does not provide a separate `relay` extra; its `harness` and Enable Relay with `FabricConfig.enable_relay(...)`. The adapter starts the installed `nemo-relay` CLI as a supervised sidecar; do not start the gateway -separately. Gateway readiness has a 30-second budget covering configuration -validation and required worker initialization. Startup fails on early process exit -or an expired budget, and the gateway log is retained for diagnosis. +separately. The 30-second readiness wait starts after gateway launch and covers +required worker initialization. The Relay CLI check and plugin configuration +writing happen before launch and are outside this wait. Startup fails on early +process exit or an expired budget, and the gateway log is retained for diagnosis. NeMo Fabric routes the selected Responses-compatible provider through the gateway and passes its explicit `base_url` to Relay as the upstream endpoint. diff --git a/adapters/python/common/README.md b/adapters/python/common/README.md index df9e8aa93..b080fd22c 100644 --- a/adapters/python/common/README.md +++ b/adapters/python/common/README.md @@ -125,8 +125,9 @@ runtime. ## Relay Gateway Readiness The Claude and Codex adapters wait up to 30 seconds for the supervised Relay -gateway's HTTP health endpoint. This budget includes configuration validation -and required worker initialization. Process exit is checked before each health +gateway's HTTP health endpoint after launch, including required worker initialization. +The Relay CLI check and plugin configuration writing happen before launch and +are outside this readiness wait. Process exit is checked before each health probe and again before accepting a successful response. An exit during an unsuccessful probe is detected at the next poll. A gateway that never becomes healthy fails when the budget expires and is stopped. From a1d552799517a68a3bea1c136305a4ab7ab6fa06 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sun, 25 Oct 2026 22:28:15 -0700 Subject: [PATCH 149/159] fix: reject positive usage with exhaustive empty models Signed-off-by: mnajafian-nv --- crates/fabric-core/src/model_usage.rs | 54 +++++++++++++++++++ crates/fabric-core/src/usage_accounting.rs | 30 +++++++++-- .../src/nemo_fabric/types.py | 11 ++++ tests/python/test_model_usage.py | 48 +++++++++++++++++ 4 files changed, 138 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/model_usage.rs b/crates/fabric-core/src/model_usage.rs index 9d9aaff80..ca2e708e0 100644 --- a/crates/fabric-core/src/model_usage.rs +++ b/crates/fabric-core/src/model_usage.rs @@ -313,6 +313,60 @@ mod accounting_tests { } } + #[test] + fn exhaustive_empty_model_list_rejects_positive_tokens() { + for observed in [false, true] { + for field in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ] { + let tokens = json!({field: 1}); + let mut usage: AgentUsage = + serde_json::from_value(if observed || field == "cache_write_input_tokens" { + json!({}) + } else { + tokens.clone() + }) + .unwrap(); + let mut accounting = json!({"models_exhaustive": true, "models": [], "gaps": []}); + if observed { + accounting["observed"] = tokens; + } else if field == "cache_write_input_tokens" { + usage + .extensions + .insert("nemo_fabric.detailed_token_usage.v1".into(), tokens); + } + usage + .extensions + .insert("nemo_fabric.usage_accounting.v1".into(), accounting); + let error = normalize_model_usage(&usage).unwrap_err(); + assert!( + error.to_string().contains("exhaustive empty model list"), + "{field}, observed={observed}: {error}" + ); + } + } + } + + #[test] + fn exhaustive_empty_model_list_preserves_unknown_and_zero_work() { + for tokens in [ + json!({}), + json!({"input_tokens": 0, "cached_input_tokens": 0, "input_tokens_include_cache": true, "output_tokens": 0, "total_tokens": 0}), + ] { + let result = normalize( + tokens.clone(), + json!({"models_exhaustive": true, "models": [], "gaps": [], "observed": tokens}), + ); + assert_eq!(result["models_exhaustive"], true); + assert!(result.get("models").is_none()); + assert_eq!(result["input_tokens"], tokens["input_tokens"]); + } + } + #[test] fn legacy_usage_does_not_gain_a_completeness_guarantee() { let usage: AgentUsage = serde_json::from_value(json!({"input_tokens": 100})).unwrap(); diff --git a/crates/fabric-core/src/usage_accounting.rs b/crates/fabric-core/src/usage_accounting.rs index c9cfa2fa6..aaf275911 100644 --- a/crates/fabric-core/src/usage_accounting.rs +++ b/crates/fabric-core/src/usage_accounting.rs @@ -216,11 +216,31 @@ impl UsageAccounting { }; let evidence: Self = serde_json::from_value(value.clone()) .map_err(|error| AccountingError(error.to_string()))?; - validate_record( - &reported_usage(usage)?, - evidence.observed.as_ref(), - &evidence.gaps, - )?; + let reported = reported_usage(usage)?; + validate_record(&reported, evidence.observed.as_ref(), &evidence.gaps)?; + if evidence.models_exhaustive + && evidence.models.is_empty() + && std::iter::once(&reported) + .chain(evidence.observed.as_ref()) + .any(|counts| { + [ + counts.input_tokens, + counts.cached_input_tokens, + counts.cache_write_input_tokens, + counts.output_tokens, + counts.reasoning_tokens, + counts.total_tokens, + counts.peak_request_input_tokens, + ] + .into_iter() + .flatten() + .any(|count| count > 0) + }) + { + return Err(AccountingError( + "exhaustive empty model list conflicts with positive tokens".into(), + )); + } let mut identities = BTreeSet::new(); for model in &evidence.models { if model.model.trim().is_empty() diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 2dfcd259b..e8cb52224 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -2863,6 +2863,17 @@ def _normalize(cls, data: dict[str, Any]) -> dict[str, Any]: if not isinstance(models, list): raise FabricConfigError("models must be an array") data["models"] = tuple(PerModelUsage.from_mapping(model) for model in models) + if ( + data["models_exhaustive"] + and not data["models"] + and any( + (data[field] or 0) > 0 or (data["observed"].get(field) or 0) > 0 + for field in _MODEL_USAGE_TOKEN_FIELDS + ) + ): + raise FabricConfigError( + "exhaustive empty model list conflicts with positive tokens" + ) identities = [(model.get("provider"), model.model) for model in data["models"]] if len(set(identities)) != len(identities): raise FabricConfigError("duplicate provider/model identity") diff --git a/tests/python/test_model_usage.py b/tests/python/test_model_usage.py index 4c731b546..8bd256da1 100644 --- a/tests/python/test_model_usage.py +++ b/tests/python/test_model_usage.py @@ -535,3 +535,51 @@ def test_pricing_basis_optional_user_catalog_attribute_preserves_source_presence parsed = PricingBasis.from_mapping(source) assert parsed.user_catalog_sha256 == user_catalog.get("user_catalog_sha256") assert parsed.to_mapping() == source + + +@pytest.mark.parametrize( + "field", + [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ], +) +@pytest.mark.parametrize("observed", [False, True]) +def test_exhaustive_empty_model_list_rejects_positive_tokens(field, observed): + usage = {"models_exhaustive": True, "models": [], "gaps": [], "cost": {}} + values = {field: 1} + if field == "total_tokens": + values.update(input_tokens=1, output_tokens=0) + if observed: + usage["observed"] = values + else: + usage.update(values) + with pytest.raises(FabricConfigError, match="exhaustive empty model list"): + ModelUsage.from_mapping(usage) + + +@pytest.mark.parametrize("value", [None, 0]) +def test_exhaustive_empty_model_list_preserves_unknown_and_zero_work(value): + fields = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "total_tokens", + ] + usage = { + "models_exhaustive": True, + "models": [], + "gaps": [], + "cost": {}, + **dict.fromkeys(fields, value), + } + usage["observed"] = dict.fromkeys(fields, value) + parsed = ModelUsage.from_mapping(usage) + assert parsed.models_exhaustive is True + assert parsed.models == () + assert parsed.input_tokens == value + assert parsed.observed.input_tokens == value From 68e5c6ff3bf9cb378683dbee974dee961f24636e Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Sun, 25 Oct 2026 23:11:04 -0700 Subject: [PATCH 150/159] feat: save run events and results in one journal Signed-off-by: mnajafian-nv --- README.md | 6 +- crates/fabric-cli/src/app.rs | 4 +- crates/fabric-core/src/config.rs | 11 +- crates/fabric-core/src/record/invocation.rs | 1120 +++++----- crates/fabric-core/src/record/lines.rs | 319 +-- crates/fabric-core/src/record/mod.rs | 23 +- crates/fabric-core/src/record/results.rs | 243 +-- crates/fabric-core/src/record/writer.rs | 209 +- crates/fabric-core/src/runtime.rs | 517 +++-- crates/fabric-core/src/schema.rs | 2 +- .../api/python-library-reference/index.md | 14 +- .../nemo_fabric.models.md | 4 +- .../nemo_fabric.reporting.md | 27 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../config/struct-recordconfig.mdx | 8 +- .../config/struct-runtimeconfig.mdx | 2 +- .../nemo-fabric-core/fn-run-record-folder.mdx | 4 +- .../nemo-fabric-core/index.mdx | 2 +- .../schema/enum-schemaname.mdx | 2 +- docs/sdk/python.mdx | 95 +- schemas/SCHEMA.md | 2 +- schemas/sdk/agent.schema.json | 6 +- schemas/sdk/run-plan.schema.json | 6 +- schemas/sdk/run-record-line.schema.json | 1852 ++++++++++++++--- .../src/nemo_fabric/_report_format.py | 13 +- .../src/nemo_fabric/models.py | 14 +- .../src/nemo_fabric/reporting.py | 1207 ++++++++--- tests/python/test_invocation_reporting.py | 97 +- tests/python/test_report_format.py | 12 +- tests/python/test_run_log_reporting.py | 622 ++++++ 30 files changed, 4282 insertions(+), 2163 deletions(-) create mode 100644 tests/python/test_run_log_reporting.py diff --git a/README.md b/README.md index e1bb08aa7..3437b2ee9 100644 --- a/README.md +++ b/README.md @@ -51,9 +51,9 @@ NeMo Fabric includes adapters for the following harnesses and execution targets. Opt in to local run evidence with `runtime.record.enabled` and an artifact root, or use `nemo-fabric run --preset scripted --record --artifacts ./runs`. -Recording saves a bounded invocation record and the finalized full result. -Inspect a saved result with `python -m nemo_fabric.reporting `; -add `--details` for resources and pricing evidence or `--json` for exact values. +Recording saves request events and the final result in one `run.jsonl` file. +Inspect it with `python -m nemo_fabric.reporting `; +add `--details` for execution history and measurement sources or `--json` for a structured summary. See [run recording](docs/sdk/python.mdx#opt-in-run-recording) for boundaries and limitations. ## NVIDIA Ecosystem diff --git a/crates/fabric-cli/src/app.rs b/crates/fabric-cli/src/app.rs index 54a41d38b..1a7eff300 100644 --- a/crates/fabric-cli/src/app.rs +++ b/crates/fabric-cli/src/app.rs @@ -87,10 +87,10 @@ enum ExampleCommand { }, } -/// Config switches that turn on the run record. +/// Config switches that enable the request journal. #[derive(Debug, Default, Args)] struct RecordSwitches { - /// Write a run record for each invocation (sets `runtime.record.enabled`). + /// Save request events and the final result or error in `run.jsonl`. /// Needs `--artifacts` unless the example sets an artifact root. #[arg(long)] record: bool, diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 360d6e23b..9d6e46e55 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -1076,7 +1076,7 @@ pub struct RuntimeConfig { #[serde(default, skip_serializing_if = "Option::is_none")] #[schemars(range(min = 1, max = u32::MAX))] pub max_turns: Option, - /// Opt-in per-invocation run record. Core-only; never sent to adapters. + /// Opt-in local request journal. Core-only; never sent to adapters. #[serde(default, skip_serializing_if = "Option::is_none")] pub record: Option, /// Additive normalized runtime fields. @@ -1091,14 +1091,15 @@ impl RuntimeConfig { } } -/// Per-invocation run record settings. +/// Opt-in local request journal settings. /// -/// When enabled, core appends a start line and an end line for each local -/// invocation to `/.fabric///run-record.jsonl`. +/// When enabled, core appends lifecycle events and a terminal summary for each +/// request to `/.fabric/requests//run.jsonl`. +/// The journal can contain sensitive result data and error messages. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RecordConfig { - /// Whether to write a run record for each invocation. Requires an + /// Whether to write a journal for each request. Requires an /// artifact root, the `local` environment provider, and a `process` or /// `python` adapter. pub enabled: bool, diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 11df9c900..ed10d43ea 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -1,698 +1,594 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! One invocation's record: the start line before the request, the end line -//! after the invocation. -//! -//! Record I/O errors do not replace the invocation result. A failed create, -//! write or sync stops the record for that invocation and adds a -//! `record_write_failed` event to the result; nothing here returns a record -//! error to the caller. - -use std::collections::BTreeMap; -use std::io; -use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant, SystemTime}; - -use serde_json::Value; +//! Explicit request-scoped ownership of a shared run journal. use super::lines::{ - ContextValue, EndArtifact, EndError, EndLine, EndUsage, LineBody, ModelName, NotRecorded, - RecordLine, RecordSchema, RecordVersion, StartLine, cut_text, recorded_code, + ContextValue, InvocationFinished, LineBody, ModelName, OutcomeScope, RecordLine, RecordSchema, + RecordSource, RecordVersion, RequestStarted, Summary, }; -use super::writer::{RECORD_FILE_NAME, RecordWriter}; +use super::writer::{AppendOutcome, MAX_DIAGNOSTIC_BYTES, RECORD_FILE_NAME, RecordWriter}; use crate::config::RunPlan; use crate::error::{FabricError, Result}; use crate::runtime::{ - ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, INVALID_AGENT_RUN_RESULT, - InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, RuntimeHandle, - event_with_metadata, unique_artifact_name, + ArtifactRef, ErrorInfo, ErrorStage, InvocationHandle, RunRequest, RunResult, RunStatus, + RuntimeHandle, event_with_metadata, new_id, unique_artifact_name, }; use crate::time::rfc3339_millis; +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime}; -/// Name of the result artifact that points at the record. -const RECORD_ARTIFACT_NAME: &str = "record"; -/// Kind of the result artifact that points at the record. -const RECORD_ARTIFACT_KIND: &str = "run_record"; -/// Event added to the result when the record could not be written. -const RECORD_WRITE_FAILED: &str = "record_write_failed"; -/// `RunRequest.context` keys whose string or integer values the start line -/// keeps: task identity and the Harbor trial link. -const RECORDED_CONTEXT_KEYS: [&str; 7] = [ - "task_id", - "task_revision", - "attempt", - "step", - "harbor_session_id", - "harbor_context_id", - "harbor_trial_name", -]; -/// End-line error code for a failed Fabric file write: `adapter-invocation.json` -/// before the request, or a result artifact after the adapter returned. -const ARTIFACT_WRITE_FAILED: &str = "artifact_write_failed"; -/// End-line error code for any other Fabric error after the record started. -const FABRIC_ERROR: &str = "fabric_error"; - -/// What the start line needs from core's invocation path. -pub(crate) struct RecordStart<'a> { - pub plan: &'a RunPlan, - pub runtime: &'a RuntimeHandle, - pub invocation: &'a InvocationHandle, - pub request: &'a RunRequest, - /// The invocation folder, `/.fabric//`. - pub fabric_home: &'a Path, - /// The host timeout that applies to this invocation. - pub timeout: Duration, -} - -/// The record of one running invocation. #[derive(Debug)] -pub(crate) struct InvocationRecorder { - runtime_id: String, - invocation_id: String, +struct RecorderState { + writer: Option, + recording_id: String, request_id: String, + runtime_id: Option, + invocation_id: Option, path: PathBuf, started: Instant, - writer: Option, - wrote_start: bool, - failure: Option, + started_at: String, + sequence: u64, + invocation_started: Option, + adapter_exchange_elapsed_seconds: Option, + scope: OutcomeScope, + failure: Option, + closed: bool, + diagnostic_omitted_bytes: u64, } -impl InvocationRecorder { - /// Create the record and write its start line. - pub(crate) fn start(start: RecordStart<'_>) -> Self { - let path = start.fabric_home.join(RECORD_FILE_NAME); - let mut recorder = Self { - runtime_id: start.invocation.runtime_id.clone(), - invocation_id: start.invocation.invocation_id.clone(), - request_id: start.invocation.request_id.clone(), - started: Instant::now(), - writer: None, - wrote_start: false, - failure: None, - path, - }; - match RecordWriter::create(&recorder.path) { - Ok(writer) => { - recorder.writer = Some(writer); - recorder.append(LineBody::Start(start_line(&start))); - recorder.wrote_start = recorder.failure.is_none(); - } - Err(error) => recorder.failure = Some(error), - } - recorder - } +/// Cloneable sink. One writer serializes producers without global ownership. +#[derive(Debug, Clone)] +pub(crate) struct RunRecorder { + state: Arc>, +} - /// Write and sync the end line for `outcome`, then return it. A result - /// gains the `record` artifact when the start line was written, and a - /// `record_write_failed` event when any write or the sync failed. - pub(crate) fn finish(mut self, outcome: Result) -> Result { - let elapsed_ms = u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX); - let end = match &outcome { - Ok(result) => result_end_line(result, elapsed_ms), - Err(error) => error_end_line(error, elapsed_ms), +impl RunRecorder { + pub(crate) fn start(plan: &RunPlan, request: &RunRequest, scope: OutcomeScope) -> Option { + let root = super::run_record_folder(plan)?; + let recording_id = new_id("recording"); + let directory = root.join("requests").join(&recording_id); + let path = directory.join(RECORD_FILE_NAME); + // Registration time includes directory preparation, validation and host + // lock wait. It never shortens the adapter's own timeout budget. + let started = Instant::now(); + let started_at = rfc3339_millis(SystemTime::now()); + let opened = std::fs::create_dir_all(&directory).and_then(|()| RecordWriter::create(&path)); + let (writer, failure) = match opened { + Ok(writer) => (Some(writer), None), + Err(error) => (None, Some(error.to_string())), }; - self.append(LineBody::End(Box::new(end))); - if let Some(writer) = &self.writer - && let Err(error) = writer.sync() - { - self.failure = Some(error); - } - if outcome.is_err() - && let Some(error) = &self.failure - { - // There is no returned result to carry the recording event. - eprintln!("nemo-fabric: invocation recording failed: {error}"); - } - let mut result = outcome?; - if self.wrote_start { - let name = unique_artifact_name(&result.artifacts, RECORD_ARTIFACT_NAME); - result.artifacts.artifacts.push(ArtifactRef { - name, - kind: RECORD_ARTIFACT_KIND.to_string(), - path: self.path.clone(), - media_type: Some("application/jsonl".to_string()), - metadata: BTreeMap::new(), - }); - } - if let Some(error) = &self.failure { - result.events.push(event_with_metadata( - RECORD_WRITE_FAILED, - format!("run record write failed; the record stops here: {error}"), - BTreeMap::from([ + let recorder = Self { + state: Arc::new(Mutex::new(RecorderState { + writer, + recording_id, + request_id: request.request_id.clone(), + runtime_id: None, + invocation_id: None, + path, + started, + started_at, + sequence: 0, + invocation_started: None, + adapter_exchange_elapsed_seconds: None, + scope, + failure, + closed: false, + diagnostic_omitted_bytes: 0, + })), + }; + recorder.append(LineBody::RequestStarted(RequestStarted { + outcome_scope: scope, + agent_name: plan.agent_name.clone(), + harness: plan + .config + .harness + .as_ref() + .map(|harness| harness.adapter_id.clone()) + .unwrap_or_else(|| "unknown".to_string()), + context: request + .context + .iter() + .map(|(key, value)| { + let kept = match key.as_str() { + "task_id" | "task_revision" | "attempt" | "step" | "harbor_session_id" + | "harbor_context_id" | "harbor_trial_name" => value + .as_str() + .map(|text| ContextValue::Text(text.chars().take(256).collect())) + .or_else(|| value.as_i64().map(ContextValue::Integer)), + _ => None, + }; ( - "runtime_id".to_string(), - Value::from(self.runtime_id.as_str()), - ), + key.clone(), + kept.unwrap_or(ContextValue::NotRecorded { recorded: false }), + ) + }) + .collect(), + override_keys: request + .overrides + .as_ref() + .and_then(|value| value.as_object()) + .map(|value| value.keys().cloned().collect()) + .unwrap_or_default(), + models: plan + .config + .models + .iter() + .map(|(role, model)| { ( - "invocation_id".to_string(), - Value::from(self.invocation_id.as_str()), - ), - ]), - )); - } - Ok(result) + role.clone(), + ModelName { + provider: model.provider.clone(), + model: model.model.clone(), + }, + ) + }) + .collect(), + })); + Some(recorder) + } + + pub(crate) fn runtime_ready(&self, runtime: &RuntimeHandle) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + state.runtime_id = Some(runtime.runtime_id.clone()); + Self::append_locked( + &mut state, + LineBody::RuntimeReady { + runtime_id: runtime.runtime_id.clone(), + }, + ); + } + + pub(crate) fn set_runtime(&self, runtime: &RuntimeHandle) { + self.state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .runtime_id = Some(runtime.runtime_id.clone()); + } + + pub(crate) fn invocation_started(&self, invocation: &InvocationHandle, timeout: Duration) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + state.runtime_id = Some(invocation.runtime_id.clone()); + state.invocation_id = Some(invocation.invocation_id.clone()); + state.invocation_started = Some(Instant::now()); + Self::append_locked( + &mut state, + LineBody::InvocationStarted { + timeout_seconds: timeout.as_secs_f64(), + }, + ); } - /// Append one line. After the first failure the record stops. - fn append(&mut self, body: LineBody) { - let Some(writer) = self.writer.as_mut() else { + pub(crate) fn invocation_finished(&self, outcome: &Result) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + let Some(started) = state.invocation_started else { return; }; - let line = RecordLine { - schema: RecordSchema::RunRecord, - version: RecordVersion::V1Alpha1, - runtime_id: self.runtime_id.clone(), - invocation_id: self.invocation_id.clone(), - request_id: self.request_id.clone(), - time: rfc3339_millis(SystemTime::now()), - body, + let elapsed = started.elapsed().as_secs_f64(); + let (status, error, model_usage, system_stats) = match outcome { + Ok(result) => ( + result.status, + result.error.clone(), + result.model_usage.clone(), + result.system_stats.clone(), + ), + Err(error) => ( + RunStatus::Failed, + Some(recorded_error(error, ErrorStage::Invoke)), + None, + None, + ), }; - if let Err(error) = writer.append(&line) { - self.failure = Some(error); - self.writer = None; - } + let exchange_elapsed = state.adapter_exchange_elapsed_seconds; + Self::append_locked( + &mut state, + LineBody::InvocationFinished(Box::new(InvocationFinished { + status, + error, + elapsed_seconds: elapsed, + adapter_exchange_elapsed_seconds: exchange_elapsed, + model_usage, + system_stats, + })), + ); } -} -fn start_line(start: &RecordStart<'_>) -> StartLine { - StartLine { - agent_name: cut_text(&start.runtime.agent_name), - harness: cut_text(&start.runtime.harness), - adapter_id: start.runtime.adapter_id.as_deref().map(cut_text), - models: start - .plan - .config - .models - .iter() - .map(|(role, model)| { - ( - role.clone(), - ModelName { - provider: cut_text(&model.provider), - model: cut_text(&model.model), - }, - ) - }) - .collect(), - fabric_version: crate::version().to_string(), - timeout_ms: u64::try_from(start.timeout.as_millis()).unwrap_or(u64::MAX), - context: start - .request - .context - .iter() - .map(|(key, value)| (key.clone(), context_value(key, value))) - .collect(), - override_keys: override_keys(start.request.overrides.as_ref()), + /// Freeze the already-measured exchange interval before endpoint/artifact I/O. + pub(crate) fn exchange_ended(&self, elapsed_seconds: f64) { + self.state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .adapter_exchange_elapsed_seconds = Some(elapsed_seconds); } -} - -/// The sorted top-level key names of `overrides`; empty unless it is an object. -fn override_keys(overrides: Option<&Value>) -> Vec { - let Some(Value::Object(overrides)) = overrides else { - return Vec::new(); - }; - let mut keys: Vec = overrides.keys().map(|key| cut_text(key)).collect(); - keys.sort(); - keys -} -/// An allowlisted key keeps a string (cut) or integer value; any other key, -/// or an allowlisted key with another value type, is recorded by name only. -fn context_value(key: &str, value: &Value) -> ContextValue { - if !RECORDED_CONTEXT_KEYS.contains(&key) { - return ContextValue::NotRecorded(NotRecorded::VALUE); + pub(crate) fn runtime_stopped(&self, outcome: &Result>) { + self.append(LineBody::RuntimeStopped { + succeeded: outcome.is_ok(), + error: outcome + .as_ref() + .err() + .map(|error| recorded_error(error, ErrorStage::Stop)), + }); } - match value { - Value::String(text) => ContextValue::Text(cut_text(text)), - Value::Number(number) => number - .as_i64() - .map_or(ContextValue::NotRecorded(NotRecorded::VALUE), |integer| { - ContextValue::Integer(integer) - }), - _ => ContextValue::NotRecorded(NotRecorded::VALUE), + + /// The timestamp observes capture, not the original production of stderr. + /// A warm host may produce background diagnostics outside any invocation. + pub(crate) fn diagnostic(&self, stream: &str, message: &str) { + // Bound chunks by UTF-8 bytes. JSON escaping can expand by six, so this + // leaves room for the envelope without splitting a Unicode character. + let chunk_limit = (MAX_DIAGNOSTIC_BYTES - 4096) / 6; + let mut remaining = message; + while !remaining.is_empty() { + let mut end = remaining.len().min(chunk_limit); + while !remaining.is_char_boundary(end) { + end -= 1; + } + self.append(LineBody::Diagnostic { + stream: stream.to_string(), + scope: "runtime".to_string(), + message: remaining[..end].to_string(), + }); + remaining = &remaining[end..]; + } } -} -fn result_end_line(result: &RunResult, elapsed_ms: u64) -> EndLine { - EndLine { - outcome_scope: "invocation", - result: Some("run-result.json".to_string()), - status: result.status, - elapsed_ms, - error: result.error.as_ref().map(result_error), - usage: result.usage.as_ref().map(end_usage), - model_usage: result.model_usage.clone(), - artifacts: end_artifacts(&result.artifacts), + fn append(&self, body: LineBody) { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + Self::append_locked(&mut state, body); } -} -fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { - let (code, stage) = match error { - FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), - // After the record starts, the check of the adapter's result - // extensions raises these: an extension the descriptor does not - // declare, or one that breaks its declared schema. Both are the same - // fault as a rejected result. - FabricError::AdapterCompatibility { .. } | FabricError::InvalidAdapterExtension { .. } => { - (INVALID_AGENT_RUN_RESULT, ErrorStage::Invoke) + fn append_locked(state: &mut RecorderState, body: LineBody) { + if state.closed || state.writer.is_none() { + return; + } + let line = RecordLine { + schema: RecordSchema::Run, + version: RecordVersion::V1Alpha1, + recording_id: state.recording_id.clone(), + request_id: state.request_id.clone(), + sequence: state.sequence, + time: rfc3339_millis(SystemTime::now()), + elapsed_seconds: state.started.elapsed().as_secs_f64(), + source: RecordSource::Fabric, + runtime_id: state.runtime_id.clone(), + invocation_id: state.invocation_id.clone(), + body, + }; + let appended = state.writer.as_mut().expect("writer checked").append(&line); + match appended { + Ok(AppendOutcome::Written) => state.sequence += 1, + Ok(AppendOutcome::Omitted) => { + if let LineBody::Diagnostic { message, .. } = &line.body { + state.diagnostic_omitted_bytes = state + .diagnostic_omitted_bytes + .saturating_add(message.len() as u64); + } + // Optional samples report rejected retention through their + // existing series diagnostics. Neither refusal wrote bytes. + } + Err(error) => { + state.failure = Some(error.to_string()); + // A partial line may be present. Never append valid data behind it. + state.writer = None; + } } - FabricError::Write { .. } => (ARTIFACT_WRITE_FAILED, ErrorStage::Artifact), - _ => (FABRIC_ERROR, ErrorStage::Invoke), - }; - EndLine { - outcome_scope: "invocation", - result: None, - status: RunStatus::Failed, - elapsed_ms, - error: Some(EndError { - code: recorded_code(code), - stage, - retryable: false, - }), - usage: None, - model_usage: None, - artifacts: Vec::new(), } -} -fn result_error(error: &ErrorInfo) -> EndError { - EndError { - code: recorded_code(&error.code), - stage: error.stage, - retryable: error.retryable, + /// Freeze admission, append one terminal record and preserve the API outcome. + pub(crate) fn finish( + &self, + mut outcome: Result, + stage: ErrorStage, + ) -> Result { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + if state.closed { + return outcome; + } + if let Ok(result) = &mut outcome { + state.runtime_id = Some(result.runtime_id.clone()); + state.invocation_id = Some(result.invocation_id.clone()); + if state.sequence > 0 { + result.artifacts.artifacts.push(ArtifactRef { + name: unique_artifact_name(&result.artifacts, "run_log"), + kind: "run_log".to_string(), + path: state.path.clone(), + media_type: Some("application/x-ndjson".to_string()), + metadata: BTreeMap::from([( + "boundary".to_string(), + serde_json::to_value(state.scope).expect("scope"), + )]), + }); + } + } + let summary = Summary { + outcome_scope: state.scope, + request_started_at: state.started_at.clone(), + request_finished_at: rfc3339_millis(SystemTime::now()), + elapsed_seconds: state.started.elapsed().as_secs_f64(), + adapter_exchange_elapsed_seconds: state.adapter_exchange_elapsed_seconds, + result: outcome.as_ref().ok().cloned(), + error: outcome + .as_ref() + .err() + .map(|error| recorded_error(error, stage)), + }; + let omitted_bytes = state.diagnostic_omitted_bytes; + if omitted_bytes != 0 { + Self::append_locked( + &mut state, + LineBody::DiagnosticRetention { + omitted_bytes, + read_error: None, + }, + ); + } + Self::append_locked(&mut state, LineBody::Summary(Box::new(summary))); + state.closed = true; + if let Some(writer) = &state.writer + && let Err(error) = writer.sync() + { + state.failure = Some(error.to_string()); + } + if let Some(failure) = &state.failure { + if let Ok(result) = &mut outcome { + result.events.push(event_with_metadata( + "record_write_failed", + format!("could not finish run journal: {failure}"), + BTreeMap::from([( + "path".to_string(), + serde_json::Value::from(state.path.to_string_lossy().as_ref()), + )]), + )); + } else { + eprintln!("nemo-fabric: run journal failed: {failure}"); + } + } + outcome } } -fn end_usage(usage: &RunUsage) -> EndUsage { - EndUsage { - input_tokens: usage.input_tokens, - cached_input_tokens: usage.cached_input_tokens, - input_tokens_include_cache: usage.input_tokens_include_cache, - output_tokens: usage.output_tokens, - total_tokens: usage.total_tokens, - cost_usd: usage.cost_usd, +fn recorded_error(error: &FabricError, stage: ErrorStage) -> ErrorInfo { + let code = match error { + FabricError::AdapterLifecycleOperation { code, .. } => code.clone(), + FabricError::InvalidAdapterExtension { .. } => "invalid_agent_run_result".to_string(), + FabricError::Write { .. } => "artifact_write_failed".to_string(), + _ => "fabric_error".to_string(), + }; + ErrorInfo { + stage, + code, + message: error.to_string(), + retryable: false, + metadata: BTreeMap::new(), } } -fn end_artifacts(manifest: &ArtifactManifest) -> Vec { - manifest - .artifacts - .iter() - .map(|artifact| EndArtifact { - name: cut_text(&artifact.name), - kind: cut_text(&artifact.kind), - media_type: artifact.media_type.as_deref().map(cut_text), - path: manifest - .root - .as_deref() - .and_then(|root| artifact.path.strip_prefix(root).ok()) - .map(|relative| relative.to_string_lossy().into_owned()), - }) - .collect() -} - #[cfg(test)] mod tests { use super::*; use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; use crate::record::test_hooks::{self, Fault}; - - #[test] - fn rejected_result_extensions_read_invalid_agent_run_result() { - let undeclared = FabricError::AdapterCompatibility { - adapter_id: "demo".to_string(), - field: "result.extensions".to_string(), - reason: "not declared".to_string(), - }; - let schema_invalid = FabricError::InvalidAdapterExtension { - adapter_id: "demo".to_string(), - descriptor_source: crate::config::DescriptorSource::Bundled, - descriptor_path: std::path::PathBuf::from("adapter.json"), - extension_path: "result.extensions.demo".to_string(), - reason: "breaks the declared schema".to_string(), - }; - for error in [undeclared, schema_invalid] { - let line = error_end_line(&error, 1); - assert_eq!( - line.error.expect("end line error").code, - INVALID_AGENT_RUN_RESULT, - "{error}" - ); - } - } - - fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { - let config: FabricConfig = serde_json::from_value(serde_json::json!({ - "schema_version": "fabric.agent/v1alpha1", - "metadata": {"name": "demo"}, - "harness": {"adapter_id": "nvidia.fabric.hermes", "resolution": "preinstalled"}, - "runtime": {}, - "models": {"default": {"provider": "nvidia", "model": "nemotron"}}, + use serde_json::{Value, json}; + + fn fixture() -> (RunPlan, RunRequest, RunResult) { + let root = std::env::temp_dir().join(new_id("run-log-contract")); + std::fs::create_dir_all(&root).unwrap(); + let config: FabricConfig = serde_json::from_value(json!({ + "schema_version":"fabric.agent/v1alpha1", "metadata":{"name":"demo"}, + "harness":{"adapter_id":"nvidia.fabric.hermes","resolution":"preinstalled"}, + "runtime":{"artifacts":root,"record":{"enabled":true}} })) - .expect("config"); - let plan = resolve_run_plan_from_config(config, ResolveContext::new("/tmp/fabric-base")) - .expect("plan"); - let runtime: RuntimeHandle = serde_json::from_value(serde_json::json!({ - "runtime_id": "rt", "runtime_binding": "binding", "agent_name": "demo", - "harness": "nvidia.fabric.hermes", "adapter_kind": "python", - "adapter_id": "nvidia.fabric.hermes", - "environment": { - "environment_id": "env", "provider": "local", - "control_location": "external_control", "ownership": "caller_owned", - }, + .unwrap(); + let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)).unwrap(); + let mut request = RunRequest::text("test"); + request.request_id = "req".to_string(); + let result = serde_json::from_value(json!({ + "agent_name":"demo", "harness":"nvidia.fabric.hermes", "adapter_kind":"python", + "runtime_id":"rt", "invocation_id":"inv", "request_id":"req", + "status":"failed", "output":{"answer":"kept"}, + "usage":{"input_tokens":42,"output_tokens":0} })) - .expect("runtime handle"); - let invocation = InvocationHandle { - invocation_id: "inv".to_string(), - request_id: "req".to_string(), - runtime_id: "rt".to_string(), - }; - (plan, runtime, invocation) + .unwrap(); + (plan, request, result) } - #[test] - fn start_line_keeps_override_and_unknown_context_key_names_only() { - let (plan, runtime, invocation) = record_test_fixture(); - let mut request = RunRequest::text("input"); - request.context = BTreeMap::from([ - ("task_id".to_string(), Value::from("t".repeat(300))), - ("task_revision".to_string(), Value::from(1.5)), - ( - "headers".to_string(), - serde_json::json!({"authorization": "Bearer abc"}), - ), - ]); - request.overrides = Some(serde_json::json!({"zeta": 1, "acme": {"api_key": "sk-123"}})); - let start = RecordStart { - plan: &plan, - runtime: &runtime, - invocation: &invocation, - request: &request, - fabric_home: Path::new("/unused"), - timeout: Duration::from_secs(5), - }; - - let line = serde_json::to_value(RecordLine { - schema: RecordSchema::RunRecord, - version: RecordVersion::V1Alpha1, - runtime_id: "rt".to_string(), - invocation_id: "inv".to_string(), - request_id: "req".to_string(), - time: "2026-10-14T18:01:10.205Z".to_string(), - body: LineBody::Start(start_line(&start)), - }) - .expect("start line"); + fn rows(recorder: &RunRecorder) -> Vec { + let path = recorder.state.lock().unwrap().path.clone(); + std::fs::read_to_string(path) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() + } - let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunRecordLine) - .expect("record line schema"); - let validator = jsonschema::validator_for(&schema).expect("valid schema"); - assert!(validator.is_valid(&line), "{line}"); - assert_eq!(line["override_keys"], serde_json::json!(["acme", "zeta"])); - assert_eq!( - line["context"]["headers"], - serde_json::json!({"recorded": false}) - ); - assert_eq!( - line["context"]["task_revision"], - serde_json::json!({"recorded": false}) - ); - assert_eq!( - line["context"]["task_id"] - .as_str() - .expect("task id") - .chars() - .count(), - 256 - ); - assert_eq!(line["timeout_ms"], 5_000); + #[test] + fn oversized_diagnostic_envelope_keeps_terminal_result_and_counts_omitted_bytes() { + let (plan, mut request, mut result) = fixture(); + request.request_id = "r".repeat(70 * 1024); + result.request_id = request.request_id.clone(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + recorder.diagnostic("stderr", "first"); + recorder.diagnostic("stderr", "second"); + let returned = recorder + .finish(Ok(result.clone()), ErrorStage::Invoke) + .unwrap(); + let rows = rows(&recorder); + assert_eq!(rows.last().unwrap()["kind"], "summary"); assert_eq!( - line["models"], - serde_json::json!({"default": {"provider": "nvidia", "model": "nemotron"}}) + rows.last().unwrap()["data"]["result"], + serde_json::to_value(&returned).unwrap() ); - let text = line.to_string(); + assert_eq!(returned.status, result.status); + assert_eq!(returned.usage, result.usage); assert!( - !text.contains("sk-123") && !text.contains("Bearer"), - "{text}" + !returned + .events + .iter() + .any(|event| event.kind == "record_write_failed") ); - } - - #[test] - fn start_line_keeps_long_role_and_context_key_names_apart() { - let (mut plan, runtime, invocation) = record_test_fixture(); - let shared = "k".repeat(300); - let model = plan.config.models["default"].clone(); - for suffix in ["a", "b"] { - plan.config - .models - .insert(format!("{shared}{suffix}"), model.clone()); - } - let mut request = RunRequest::text("input"); - request.context = BTreeMap::from([ - (format!("{shared}a"), Value::from(1)), - (format!("{shared}b"), Value::from(2)), - ]); - let start = RecordStart { - plan: &plan, - runtime: &runtime, - invocation: &invocation, - request: &request, - fabric_home: Path::new("/unused"), - timeout: Duration::from_secs(5), - }; - - let line = start_line(&start); - - assert_eq!(line.models.len(), 3); - assert_eq!(line.context.len(), 2); - for suffix in ["a", "b"] { - assert!(line.models.contains_key(&format!("{shared}{suffix}"))); - assert!(line.context.contains_key(&format!("{shared}{suffix}"))); - } - } - - fn temp_folder(name: &str) -> PathBuf { - let folder = std::env::temp_dir().join(format!( - "fabric-recorder-{name}-{}-{}", - std::process::id(), - SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - std::fs::create_dir_all(&folder).expect("folder"); - folder - } - - fn succeeded_result(invocation: &InvocationHandle) -> RunResult { - serde_json::from_value(serde_json::json!({ - "agent_name": "a", "harness": "h", "adapter_kind": "python", - "runtime_id": invocation.runtime_id, "invocation_id": invocation.invocation_id, - "request_id": invocation.request_id, "status": "succeeded", "output": "done", - })) - .expect("result") - } - - #[test] - fn finish_names_the_record_artifact_apart_from_an_adapter_artifact_named_record() { - let (plan, runtime, invocation) = record_test_fixture(); - let folder = temp_folder("unique-name"); - let request = RunRequest::text("input"); - let recorder = InvocationRecorder::start(RecordStart { - plan: &plan, - runtime: &runtime, - invocation: &invocation, - request: &request, - fabric_home: &folder, - timeout: Duration::from_secs(5), - }); - let mut result = succeeded_result(&invocation); - result.artifacts.artifacts.push(ArtifactRef { - name: RECORD_ARTIFACT_NAME.to_string(), - kind: "report".to_string(), - path: folder.join("report.json"), - media_type: None, - metadata: BTreeMap::new(), - }); - - let finished = recorder.finish(Ok(result)).expect("result"); - - let names: Vec<&str> = finished - .artifacts - .artifacts + let retention: Vec<_> = rows .iter() - .map(|artifact| artifact.name.as_str()) + .filter(|row| row["kind"] == "diagnostic_retention") .collect(); - assert_eq!(names, ["record", "record_2"]); - assert_eq!(finished.artifacts.artifacts[1].kind, RECORD_ARTIFACT_KIND); + assert_eq!(retention.len(), 1); + assert_eq!(retention[0]["data"]["omitted_bytes"], 11); + assert!(retention[0]["data"]["read_error"].is_null()); + assert!(rows.iter().all(|row| row["kind"] != "diagnostic")); + for (sequence, row) in rows.iter().enumerate() { + assert_eq!(row["sequence"], sequence as u64); + } } #[test] - fn finish_after_a_failed_create_adds_the_event_and_no_record_artifact() { - let (plan, runtime, invocation) = record_test_fixture(); - let folder = temp_folder("create-failure"); - std::fs::write(folder.join(RECORD_FILE_NAME), "planted").expect("planted file"); - let request = RunRequest::text("input"); - let recorder = InvocationRecorder::start(RecordStart { - plan: &plan, - runtime: &runtime, - invocation: &invocation, - request: &request, - fabric_home: &folder, - timeout: Duration::from_secs(5), - }); - let result = succeeded_result(&invocation); - - let finished = recorder.finish(Ok(result.clone())).expect("result"); - - assert_eq!(finished.artifacts, result.artifacts); - assert_eq!(finished.events.len(), 1); - assert_eq!(finished.events[0].kind, RECORD_WRITE_FAILED); - assert_eq!(finished.status, RunStatus::Succeeded); + fn terminal_result_retains_large_output_and_reported_zero() { + let (plan, request, mut result) = fixture(); + result.output = Value::String("x".repeat(128 * 1024)); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + let returned = recorder.finish(Ok(result), ErrorStage::Invoke).unwrap(); + let rows = rows(&recorder); + assert_eq!(rows.len(), 2); assert_eq!( - std::fs::read_to_string(folder.join(RECORD_FILE_NAME)).expect("read"), - "planted" + rows[1]["data"]["result"], + serde_json::to_value(returned).unwrap() ); + assert_eq!(rows[1]["data"]["result"]["usage"]["output_tokens"], 0); } - fn failed_result_with_usage(invocation: &InvocationHandle) -> RunResult { - serde_json::from_value(serde_json::json!({ - "agent_name": "a", "harness": "h", "adapter_kind": "python", - "runtime_id": invocation.runtime_id, "invocation_id": invocation.invocation_id, - "request_id": invocation.request_id, "status": "failed", "output": "partial", - "error": { - "stage": "invoke", "code": "adapter_failure", - "message": "adapter gave up", "retryable": true, - }, - "usage": {"input_tokens": 3, "output_tokens": 5, "cost_usd": 0.25}, - })) - .expect("result") - } - - /// Start and finish one record with `fault` injected, and return the - /// finished outcome with the record file's lines. - fn finish_with_fault( - name: &str, - fault: Fault, - outcome: Result, - ) -> (Result, Vec) { - let (plan, runtime, invocation) = record_test_fixture(); - let folder = temp_folder(name); - let request = RunRequest::text("input"); - test_hooks::FAULT.with(|slot| slot.set(Some(fault))); - let recorder = InvocationRecorder::start(RecordStart { - plan: &plan, - runtime: &runtime, - invocation: &invocation, - request: &request, - fabric_home: &folder, - timeout: Duration::from_secs(5), - }); - let finished = recorder.finish(outcome); - test_hooks::FAULT.with(|slot| slot.set(None)); - let lines = std::fs::read_to_string(folder.join(RECORD_FILE_NAME)) - .expect("read record") - .lines() - .map(|line| serde_json::from_str(line).expect("record line")) + #[test] + fn concurrent_producers_share_sequence_and_stop_after_summary() { + let (plan, request, result) = fixture(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + let workers: Vec<_> = (0..4) + .map(|worker| { + let sink = recorder.clone(); + std::thread::spawn(move || { + for item in 0..20 { + sink.diagnostic("stderr", &format!("worker {worker}: {item}")); + } + }) + }) .collect(); - (finished, lines) + for worker in workers { + worker.join().unwrap(); + } + recorder.finish(Ok(result), ErrorStage::Invoke).unwrap(); + recorder.diagnostic("stderr", "must not appear after terminal summary"); + let rows = rows(&recorder); + assert_eq!(rows.len(), 82); + assert_eq!(rows.last().unwrap()["kind"], "summary"); + for (sequence, row) in rows.iter().enumerate() { + assert_eq!(row["sequence"], sequence as u64); + } } - /// `finished` equals `original` apart from the artifacts and events that - /// recording adds, and carries one `record_write_failed` event. - fn assert_result_kept_with_failure_event(finished: &RunResult, original: &RunResult) { - let mut without_record = finished.clone(); - without_record.artifacts = original.artifacts.clone(); - without_record.events = original.events.clone(); - assert_eq!(&without_record, original); - let failures: Vec<_> = finished - .events - .iter() - .filter(|event| event.kind == RECORD_WRITE_FAILED) - .collect(); - assert_eq!(failures.len(), 1, "{:?}", finished.events); - assert_eq!( - failures[0].metadata["runtime_id"], - original.runtime_id.as_str() - ); - assert_eq!( - failures[0].metadata["invocation_id"], - original.invocation_id.as_str() + #[test] + fn failed_start_append_keeps_execution_without_claiming_a_saved_result() { + let (plan, request, result) = fixture(); + test_hooks::FAULT.with(|slot| slot.set(Some(Fault::StartAppend))); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + test_hooks::FAULT.with(|slot| slot.set(None)); + let returned = recorder + .finish(Ok(result.clone()), ErrorStage::Invoke) + .unwrap(); + assert_eq!(returned.output, result.output); + assert_eq!(returned.usage, result.usage); + assert!(returned.artifacts.artifacts.is_empty()); + assert!( + returned + .events + .iter() + .any(|event| event.kind == "record_write_failed") ); - } - - fn has_record_artifact(result: &RunResult) -> bool { - result - .artifacts - .artifacts - .iter() - .any(|artifact| artifact.kind == RECORD_ARTIFACT_KIND) + assert!(rows(&recorder).is_empty()); } #[test] - fn finish_after_a_failed_start_line_keeps_the_result_and_writes_nothing() { - let (_, _, invocation) = record_test_fixture(); - let original = failed_result_with_usage(&invocation); - - let (finished, lines) = - finish_with_fault("start-append", Fault::StartAppend, Ok(original.clone())); - - let finished = finished.expect("result"); - assert_result_kept_with_failure_event(&finished, &original); - assert!(!has_record_artifact(&finished)); - assert!(lines.is_empty(), "{lines:?}"); + fn failed_terminal_append_does_not_resume_or_change_execution() { + let (plan, request, result) = fixture(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + test_hooks::FAULT.with(|slot| slot.set(Some(Fault::EndAppend))); + let returned = recorder + .finish(Ok(result.clone()), ErrorStage::Invoke) + .unwrap(); + test_hooks::FAULT.with(|slot| slot.set(None)); + recorder.diagnostic("stderr", "late"); + assert_eq!(returned.status, result.status); + assert_eq!(returned.usage, result.usage); + assert_eq!(returned.output, result.output); + assert!( + returned + .events + .iter() + .any(|event| event.kind == "record_write_failed") + ); + assert_eq!(rows(&recorder).len(), 1); } #[test] - fn finish_after_a_failed_end_line_keeps_the_result_and_the_start_line() { - let (_, _, invocation) = record_test_fixture(); - let original = failed_result_with_usage(&invocation); - - let (finished, lines) = - finish_with_fault("end-append", Fault::EndAppend, Ok(original.clone())); - - let finished = finished.expect("result"); - assert_result_kept_with_failure_event(&finished, &original); - assert!(has_record_artifact(&finished)); - let kinds: Vec<&Value> = lines.iter().map(|line| &line["kind"]).collect(); - assert_eq!(kinds, ["start"]); + fn failed_sync_preserves_final_evidence_and_reports_durability_failure() { + let (plan, request, result) = fixture(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + test_hooks::FAULT.with(|slot| slot.set(Some(Fault::Sync))); + let returned = recorder + .finish(Ok(result.clone()), ErrorStage::Invoke) + .unwrap(); + test_hooks::FAULT.with(|slot| slot.set(None)); + assert_eq!(returned.output, result.output); + assert_eq!(rows(&recorder).last().unwrap()["kind"], "summary"); + assert!( + returned + .events + .iter() + .any(|event| event.kind == "record_write_failed") + ); } #[test] - fn finish_after_a_failed_sync_keeps_the_result_and_both_lines() { - let (_, _, invocation) = record_test_fixture(); - let original = failed_result_with_usage(&invocation); - - let (finished, lines) = finish_with_fault("sync", Fault::Sync, Ok(original.clone())); - - let finished = finished.expect("result"); - assert_result_kept_with_failure_event(&finished, &original); - assert!(has_record_artifact(&finished)); - let kinds: Vec<&Value> = lines.iter().map(|line| &line["kind"]).collect(); - assert_eq!(kinds, ["start", "end"]); - assert_eq!(lines[1]["usage"]["input_tokens"], 3); + fn no_result_summary_keeps_outer_failure_without_inner_identity() { + let (plan, request, _) = fixture(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::OneShotLifecycle).unwrap(); + let error = FabricError::InvalidConfig { + field: "test".to_string(), + reason: "invalid".to_string(), + }; + let original = error.to_string(); + assert_eq!( + recorder + .finish(Err(error), ErrorStage::Prepare) + .unwrap_err() + .to_string(), + original + ); + let rows = rows(&recorder); + assert!(rows[1].get("runtime_id").is_none()); + assert!(rows[1]["data"]["result"].is_null()); + assert_eq!(rows[1]["data"]["error"]["stage"], "prepare"); } #[test] - fn finish_returns_the_invocation_error_when_recording_also_fails() { - for fault in [Fault::StartAppend, Fault::EndAppend, Fault::Sync] { - let error = FabricError::AdapterLifecycleOperation { - operation: "invoke", - runtime_id: "rt".to_string(), - code: "host_timeout".to_string(), - message: "adapter host did not answer".to_string(), - diagnostics: String::new(), - }; - let expected = error.to_string(); - - let (finished, _) = finish_with_fault("error", fault, Err(error)); - - let error = finished.expect_err("invocation error"); - assert_eq!(error.to_string(), expected, "{fault:?}"); - assert!( - matches!(&error, FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout"), - "{fault:?}: {error}" - ); - } + fn exchange_time_freezes_before_invocation_finalization() { + let (plan, request, result) = fixture(); + let recorder = RunRecorder::start(&plan, &request, OutcomeScope::Invocation).unwrap(); + let invocation = InvocationHandle { + runtime_id: "rt".to_string(), + invocation_id: "inv".to_string(), + request_id: "req".to_string(), + }; + recorder.invocation_started(&invocation, Duration::from_secs(60)); + recorder.exchange_ended(0.125); + recorder.invocation_finished(&Ok(result.clone())); + recorder.finish(Ok(result), ErrorStage::Invoke).unwrap(); + let rows = rows(&recorder); + let finished = rows + .iter() + .find(|row| row["kind"] == "invocation_finished") + .unwrap(); + assert_eq!(finished["data"]["adapter_exchange_elapsed_seconds"], 0.125); + assert_eq!( + rows.last().unwrap()["data"]["adapter_exchange_elapsed_seconds"], + 0.125 + ); } } diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs index 9ba4aba10..c7e50732b 100644 --- a/crates/fabric-core/src/record/lines.rs +++ b/crates/fabric-core/src/record/lines.rs @@ -1,275 +1,134 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Line types of `run-record.jsonl`, schema `nemo.fabric.run_record` version -//! `v1alpha1`. -//! -//! Every field is typed. A line holds identifiers, configuration names, normalized -//! accounting, resource observations, closed codes and allowlisted request context -//! values. It never captures prompts, output, exception messages or arbitrary -//! adapter metadata. - -use std::collections::BTreeMap; +//! Typed lines in the shared, request-scoped run journal. +use crate::model_usage::ModelUsage; +use crate::runtime::{ErrorInfo, RunResult, RunStatus}; +use crate::system_stats::InvocationSystemStats; use schemars::JsonSchema; use serde::Serialize; +use std::collections::BTreeMap; -use crate::runtime::{ErrorStage, RunStatus}; - -/// Longest value a line keeps from config or the request, in characters. -/// Model role and context key names are kept whole, so map keys stay distinct. -const MAX_VALUE_CHARS: usize = 256; -/// Longest adapter error code a line keeps. -const MAX_CODE_CHARS: usize = 128; -/// Error code written in place of a code outside `^[a-z0-9_.:-]{1,128}$`. -const UNRECOGNIZED_CODE: &str = "unrecognized"; - -/// One line of a run record. -#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +/// One independently parseable line of a recorded request. +#[derive(Debug, Clone, Serialize, JsonSchema)] #[schemars(title = "RunRecordLine")] pub(crate) struct RecordLine { - /// Always `nemo.fabric.run_record`. pub schema: RecordSchema, - /// Line format version. `v1alpha1` may change until a record reader ships. pub version: RecordVersion, - /// Runtime id. - pub runtime_id: String, - /// Invocation id. - pub invocation_id: String, - /// Request id. + pub recording_id: String, pub request_id: String, - /// UTC wall-clock time the line was written, RFC 3339 with milliseconds. + pub sequence: u64, #[schemars(extend("format" = "date-time"))] pub time: String, - /// Line kind and its fields. + #[schemars(range(min = 0.0))] + pub elapsed_seconds: f64, + pub source: RecordSource, + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub invocation_id: Option, #[serde(flatten)] pub body: LineBody, } -/// The record's schema name. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, Copy, Serialize, JsonSchema)] pub(crate) enum RecordSchema { - /// `nemo.fabric.run_record`. - #[serde(rename = "nemo.fabric.run_record")] - RunRecord, + #[serde(rename = "nemo.fabric.run")] + Run, } -/// The record's line format version. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, Copy, Serialize, JsonSchema)] pub(crate) enum RecordVersion { - /// `v1alpha1`. #[serde(rename = "v1alpha1")] V1Alpha1, } -/// A line's kind, with the fields that kind carries. -#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub(crate) enum LineBody { - /// Written once, before core sends the request to the adapter host. - Start(StartLine), - /// Written once, after the invocation returned a result or failed. - End(Box), +#[derive(Debug, Clone, Copy, Serialize, JsonSchema)] +pub(crate) enum RecordSource { + #[serde(rename = "fabric")] + Fabric, +} + +/// The operation whose return boundary closes this journal. +#[derive(Debug, Clone, Copy, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum OutcomeScope { + Invocation, + OneShotLifecycle, } -/// Fields of the start line. -#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] -pub(crate) struct StartLine { - /// Config name (`metadata.name`). +#[derive(Debug, Clone, Serialize, JsonSchema)] +#[serde(tag = "kind", content = "data", rename_all = "snake_case")] +pub(crate) enum LineBody { + RequestStarted(RequestStarted), + RuntimeReady { + runtime_id: String, + }, + InvocationStarted { + timeout_seconds: f64, + }, + InvocationFinished(Box), + RuntimeStopped { + succeeded: bool, + error: Option, + }, + Diagnostic { + stream: String, + scope: String, + message: String, + }, + DiagnosticRetention { + omitted_bytes: u64, + read_error: Option, + }, + Summary(Box), +} + +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub(crate) struct RequestStarted { + pub outcome_scope: OutcomeScope, pub agent_name: String, - /// Harness identifier. pub harness: String, - /// Adapter implementation id. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub adapter_id: Option, - /// Configured models by role: provider and model name only. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub models: BTreeMap, - /// NeMo Fabric version that wrote the record. - pub fabric_version: String, - /// Host invocation timeout that applied, in milliseconds. - pub timeout_ms: u64, - /// `RunRequest.context`. Allowlisted identity and Harbor link keys keep a - /// string or integer value; every other key is listed by name only. - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub context: BTreeMap, - /// Top-level key names of `RunRequest.overrides`, sorted. Values are never - /// recorded. - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub override_keys: Vec, } -/// One configured model. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[derive(Debug, Clone, Serialize, JsonSchema)] pub(crate) struct ModelName { - /// Model provider name. pub provider: String, - /// Provider model identifier. pub model: String, } -/// A request context value as the start line keeps it. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +/// Evidence at invocation return is independent from later owned shutdown. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub(crate) struct InvocationFinished { + pub status: RunStatus, + pub error: Option, + pub elapsed_seconds: f64, + pub adapter_exchange_elapsed_seconds: Option, + pub model_usage: Option, + pub system_stats: Option, +} + +/// Authoritative final evidence for this recorded operation. +#[derive(Debug, Clone, Serialize, JsonSchema)] +pub(crate) struct Summary { + pub outcome_scope: OutcomeScope, + pub request_started_at: String, + pub request_finished_at: String, + pub elapsed_seconds: f64, + pub adapter_exchange_elapsed_seconds: Option, + pub result: Option, + pub error: Option, +} + +/// Unknown context values are intentionally not copied into request evidence. +#[derive(Debug, Clone, Serialize, JsonSchema)] #[serde(untagged)] pub(crate) enum ContextValue { - /// An allowlisted key's string value, cut to 256 characters. Text(String), - /// An allowlisted key's integer value. Integer(i64), - /// Any other key or value: the key name only. - NotRecorded(NotRecorded), -} - -/// Marks a key whose value the record leaves out. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] -pub(crate) struct NotRecorded { - /// Always `false`. - #[schemars(extend("const" = false))] - pub recorded: bool, -} - -impl NotRecorded { - pub(crate) const VALUE: Self = Self { recorded: false }; -} - -/// Fields of the end line. -#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] -pub(crate) struct EndLine { - /// This line observes the invocation, before one-shot runtime shutdown. - #[schemars(extend("const" = "invocation"))] - pub outcome_scope: &'static str, - /// Expected full-result file next to this record; absence means not saved. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub result: Option, - /// Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`. - pub status: RunStatus, - /// Monotonic time from the start line to the end line, in milliseconds. - pub elapsed_ms: u64, - /// Error code, stage and retryable flag, without the message. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub error: Option, - /// Numeric usage counters, without adapter metadata. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub usage: Option, - /// Validated normalized accounting frozen at the invocation boundary. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub model_usage: Option, - /// Result artifacts, with paths relative to the artifact root. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub artifacts: Vec, -} - -/// The end line's error. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] -pub(crate) struct EndError { - /// Error code matching `^[a-z0-9_.:-]{1,128}$`, else `unrecognized`. - #[schemars(regex(pattern = r"^[a-z0-9_.:-]{1,128}$"))] - pub code: String, - /// Lifecycle stage where the failure surfaced. - pub stage: ErrorStage, - /// Whether a consumer-level retry is considered safe. - pub retryable: bool, -} - -/// The end line's usage: the numeric `RunUsage` fields. -#[derive(Debug, Clone, Default, PartialEq, Serialize, JsonSchema)] -pub(crate) struct EndUsage { - /// Input tokens. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub input_tokens: Option, - /// Cached input tokens. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cached_input_tokens: Option, - /// Whether `input_tokens` includes `cached_input_tokens`; absent means unknown. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub input_tokens_include_cache: Option, - /// Output tokens. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub output_tokens: Option, - /// Total tokens reported by the provider. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub total_tokens: Option, - /// Reported cost in US dollars. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[schemars(range(min = 0.0))] - pub cost_usd: Option, -} - -/// One result artifact on the end line. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] -pub(crate) struct EndArtifact { - /// Logical artifact name. - pub name: String, - /// Artifact kind. - pub kind: String, - /// Media type. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub media_type: Option, - /// Path relative to the artifact root; absent for a path outside it. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub path: Option, -} - -/// Cut `text` to [`MAX_VALUE_CHARS`] characters: 255 plus "…". -pub(crate) fn cut_text(text: &str) -> String { - if text.chars().count() <= MAX_VALUE_CHARS { - return text.to_string(); - } - let mut cut: String = text.chars().take(MAX_VALUE_CHARS - 1).collect(); - cut.push('…'); - cut -} - -/// An error code as the end line keeps it: unchanged when it matches -/// `^[a-z0-9_.:-]{1,128}$`, else [`UNRECOGNIZED_CODE`], because a code outside -/// that pattern may be a message. -pub(crate) fn recorded_code(code: &str) -> String { - let valid = (1..=MAX_CODE_CHARS).contains(&code.len()) - && code.bytes().all(|byte| { - byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"_.:-".contains(&byte) - }); - if valid { - code.to_string() - } else { - UNRECOGNIZED_CODE.to_string() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cut_text_over_the_limit_keeps_255_chars_and_an_ellipsis() { - let cut = cut_text(&"é".repeat(1000)); - - assert_eq!(cut.chars().count(), MAX_VALUE_CHARS); - assert!(cut.ends_with('…')); - assert_eq!(cut_text("short"), "short"); - assert_eq!(cut_text(&"a".repeat(256)), "a".repeat(256)); - } - - #[test] - fn recorded_code_keeps_only_codes_in_the_pattern() { - for code in [ - "timeout", - "host_timeout", - "provider.rate_limit", - "http:429", - "a-b", - ] { - assert_eq!(recorded_code(code), code); - } - for code in [ - "", - "Timeout", - "rate limit", - "error: key sk-123", - "über", - &"a".repeat(129), - ] { - assert_eq!(recorded_code(code), UNRECOGNIZED_CODE, "{code}"); - } - assert_eq!(recorded_code(&"a".repeat(128)), "a".repeat(128)); - } + NotRecorded { recorded: bool }, } diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs index f6a677b0f..ba7bebb4a 100644 --- a/crates/fabric-core/src/record/mod.rs +++ b/crates/fabric-core/src/record/mod.rs @@ -1,27 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! The opt-in per-invocation run record, `run-record.jsonl`. +//! One best-effort, append-only `run.jsonl` per recorded SDK operation. //! -//! When `runtime.record.enabled` is set, core appends one start line before it -//! sends the request to the adapter host and one end line after the invocation -//! returns or fails. Recording is synchronous and best-effort: record I/O -//! errors do not replace the invocation result. A failed write stops the -//! record and becomes a `record_write_failed` event on the result, and slow -//! storage delays the invocation. -//! -//! Everything here is crate-private. The line format is public only through -//! the `run-record-line` JSON Schema, version `v1alpha1`. +//! Recording starts before validation and startup. I/O failures never replace +//! execution outcomes. The terminal summary preserves the returned result. mod invocation; mod lines; +mod results; mod writer; -pub(crate) use invocation::{InvocationRecorder, RecordStart}; -pub(crate) use lines::RecordLine; +pub(crate) use invocation::RunRecorder; +pub(crate) use lines::{OutcomeScope, RecordLine}; +pub use results::run_record_folder; #[cfg(test)] pub(crate) use writer::test_hooks; - -mod results; -pub(crate) use results::persist_result; -pub use results::run_record_folder; diff --git a/crates/fabric-core/src/record/results.rs b/crates/fabric-core/src/record/results.rs index fce000db4..bf9e57eb5 100644 --- a/crates/fabric-core/src/record/results.rs +++ b/crates/fabric-core/src/record/results.rs @@ -1,23 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Full finalized invocation or one-shot results. -//! These files may contain result output; the bounded invocation record does not. - -use std::collections::BTreeMap; -use std::fs::{File, OpenOptions}; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; - -use serde_json::Value; +//! Recording-root resolution, available before a runtime exists. use crate::config::RunPlan; -use crate::runtime::{ArtifactRef, RunResult, event_with_metadata, new_id, unique_artifact_name}; +use std::path::PathBuf; -pub(crate) const RESULT_FILE_NAME: &str = "run-result.json"; -const WRITE_FAILED: &str = "record_write_failed"; - -/// The recording root. This is known before startup and requires no inner IDs. +/// The parent of request-scoped run journals. Requires recording to be enabled. pub fn run_record_folder(plan: &RunPlan) -> Option { if !plan.config.runtime.record_enabled() { return None; @@ -40,229 +29,3 @@ pub fn run_record_folder(plan: &RunPlan) -> Option { }) .map(|root| root.join(".fabric")) } - -fn private_new_file(path: &Path) -> io::Result { - let mut options = OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - } - options.open(path) -} - -/// Publish a complete JSON file without replacing a file or a symlink. -/// A failed write may leave an explicitly named staging file, never a final file. -fn publish(path: &Path, value: &impl serde::Serialize) -> io::Result<()> { - let bytes = serde_json::to_vec_pretty(value).map_err(io::Error::other)?; - let pending = path.with_extension(format!("{}.pending", new_id("write"))); - let mut file = private_new_file(&pending)?; - let published = (|| { - file.write_all(&bytes)?; - file.sync_all()?; - // Unlike rename, this cannot replace an existing file or symlink. - std::fs::hard_link(&pending, path) - })(); - drop(file); - // Only this operation's newly created staging path can be removed. - if let Err(error) = cleanup_staging(&pending) { - // Publication has its own outcome. A private staging-link cleanup - // failure must not invalidate or mutate an already frozen result. - eprintln!( - "nemo-fabric: could not clean recording staging file {}: {error}", - pending.display() - ); - } - published -} - -fn cleanup_staging(path: &Path) -> io::Result<()> { - #[cfg(test)] - if CLEANUP_FAILURE.with(std::cell::Cell::get) { - return Err(io::Error::other("injected staging cleanup failure")); - } - std::fs::remove_file(path) -} - -#[cfg(test)] -thread_local! { - static CLEANUP_FAILURE: std::cell::Cell = const { std::cell::Cell::new(false) }; -} - -fn add_failure(result: &mut RunResult, error: &io::Error, artifact: &str) { - result.events.push(event_with_metadata( - WRITE_FAILED, - format!("could not save {artifact}: {error}"), - BTreeMap::from([("artifact".to_string(), Value::from(artifact))]), - )); -} - -/// Save exactly the returned result after its invocation or one-shot lifecycle. -pub(crate) fn persist_result(plan: &RunPlan, mut result: RunResult, boundary: &str) -> RunResult { - let Some(root) = run_record_folder(plan) else { - return result; - }; - let directory = root.join(&result.runtime_id).join(&result.invocation_id); - let path = directory.join(RESULT_FILE_NAME); - let name = unique_artifact_name(&result.artifacts, "run_result"); - let artifact = ArtifactRef { - name: name.clone(), - kind: "run_result".to_string(), - path: path.clone(), - media_type: Some("application/json".to_string()), - metadata: BTreeMap::from([("boundary".to_string(), Value::from(boundary))]), - }; - result.artifacts.artifacts.push(artifact); - let saved = std::fs::create_dir_all(&directory).and_then(|()| publish(&path, &result)); - if let Err(error) = saved { - result - .artifacts - .artifacts - .retain(|artifact| artifact.name != name); - add_failure(&mut result, &error, RESULT_FILE_NAME); - } - result -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; - use serde_json::json; - - fn fixture() -> (PathBuf, RunPlan, RunResult) { - let root = std::env::temp_dir().join(new_id("recording-result-test")); - std::fs::create_dir_all(&root).expect("test root"); - let config: FabricConfig = serde_json::from_value(json!({ - "schema_version":"fabric.agent/v1alpha1", "metadata":{"name":"demo"}, - "harness":{"adapter_id":"nvidia.fabric.hermes","resolution":"preinstalled"}, - "runtime":{"artifacts":root,"record":{"enabled":true}} - })) - .expect("config"); - let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)).expect("plan"); - let result = serde_json::from_value(json!({ - "agent_name":"demo","harness":"nvidia.fabric.hermes","adapter_kind":"python", - "runtime_id":"rt","invocation_id":"inv","request_id":"req", - "status":"failed","output":{"answer":"kept"}, - "metadata":{"existing":"kept"},"usage":{"input_tokens":42,"output_tokens":0} - })) - .expect("result"); - (root, plan, result) - } - - #[test] - fn recorded_full_result_is_exact_and_keeps_failed_usage() { - let (_, plan, result) = fixture(); - let returned = persist_result(&plan, result, "one_shot_lifecycle"); - let artifact = returned - .artifacts - .artifacts - .iter() - .find(|item| item.kind == "run_result") - .expect("saved result artifact"); - let saved: Value = - serde_json::from_slice(&std::fs::read(&artifact.path).expect("result file")) - .expect("JSON"); - assert_eq!( - saved, - serde_json::to_value(&returned).expect("returned result") - ); - assert_eq!(saved["usage"]["input_tokens"], 42); - assert_eq!(saved["usage"]["output_tokens"], 0); - assert_eq!(artifact.metadata["boundary"], "one_shot_lifecycle"); - assert!( - std::fs::read_dir(artifact.path.parent().unwrap()) - .expect("directory") - .all(|entry| !entry - .unwrap() - .file_name() - .to_string_lossy() - .ends_with(".pending")) - ); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - assert_eq!( - std::fs::metadata(&artifact.path) - .unwrap() - .permissions() - .mode() - & 0o777, - 0o600 - ); - } - } - - #[test] - fn published_result_remains_exact_when_owned_staging_cleanup_fails() { - let (_, plan, result) = fixture(); - CLEANUP_FAILURE.with(|slot| slot.set(true)); - let returned = persist_result(&plan, result, "invocation"); - CLEANUP_FAILURE.with(|slot| slot.set(false)); - let artifact = returned - .artifacts - .artifacts - .iter() - .find(|item| item.kind == "run_result") - .expect("publication remains successful"); - let saved: Value = serde_json::from_slice(&std::fs::read(&artifact.path).unwrap()).unwrap(); - assert_eq!(saved, serde_json::to_value(&returned).unwrap()); - assert!( - returned - .events - .iter() - .all(|event| event.kind != WRITE_FAILED) - ); - assert_eq!( - std::fs::read_dir(artifact.path.parent().unwrap()) - .unwrap() - .count(), - 2 - ); - } - - #[test] - fn existing_full_result_is_not_overwritten_and_execution_is_kept() { - let (root, plan, result) = fixture(); - let folder = root.join(".fabric/rt/inv"); - std::fs::create_dir_all(&folder).expect("folder"); - let path = folder.join(RESULT_FILE_NAME); - std::fs::write(&path, b"existing evidence").expect("existing file"); - let original = result.clone(); - let returned = persist_result(&plan, result, "invocation"); - assert_eq!(returned.status, original.status); - assert_eq!(returned.usage, original.usage); - assert_eq!(returned.output, original.output); - assert_eq!(std::fs::read(&path).unwrap(), b"existing evidence"); - assert!( - returned - .artifacts - .artifacts - .iter() - .all(|item| item.kind != "run_result") - ); - assert!( - returned - .events - .iter() - .any(|event| event.kind == WRITE_FAILED) - ); - assert_eq!(std::fs::read_dir(folder).unwrap().count(), 1); - } - - #[test] - #[cfg(unix)] - fn atomic_publication_refuses_symlink_and_cleans_only_owned_stage() { - let root = std::env::temp_dir().join(new_id("recording-link-test")); - std::fs::create_dir_all(&root).unwrap(); - let original = root.join("original.json"); - std::fs::write(&original, "unchanged").unwrap(); - let target = root.join("result.json"); - std::os::unix::fs::symlink(&original, &target).unwrap(); - assert!(publish(&target, &json!({"new":"content"})).is_err()); - assert_eq!(std::fs::read_to_string(original).unwrap(), "unchanged"); - assert!(target.is_symlink()); - assert_eq!(std::fs::read_dir(root).unwrap().count(), 2); - } -} diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs index b8240ba33..bef726f0c 100644 --- a/crates/fabric-core/src/record/writer.rs +++ b/crates/fabric-core/src/record/writer.rs @@ -1,34 +1,30 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! The run record file: created once, owner-only, appended one line per write. -//! -//! Each line uses `write_all` on an unbuffered file. If the Fabric process -//! terminates, lines whose writes completed remain in the file. The end line -//! is synced once. Durability after a machine crash or power loss is not -//! guaranteed because the containing directory is not synced. +//! Exclusive append writer. A failed write disables its owning recorder. +use super::lines::{LineBody, RecordLine}; use std::fs::{File, OpenOptions}; use std::io::{self, Write}; use std::path::Path; -use super::lines::RecordLine; +pub(crate) const RECORD_FILE_NAME: &str = "run.jsonl"; +pub(super) const MAX_DIAGNOSTIC_BYTES: usize = 64 * 1024; -/// File name of the record inside the invocation folder. -pub(crate) const RECORD_FILE_NAME: &str = "run-record.jsonl"; -/// Maximum serialized JSON size of one line, excluding its newline. -const MAX_LINE_BYTES: usize = 64 * 1024; +/// A size refusal writes no bytes; I/O failures remain ordinary errors. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum AppendOutcome { + Written, + Omitted, +} -/// Appends lines to one invocation's record. #[derive(Debug)] -pub(crate) struct RecordWriter { +pub(super) struct RecordWriter { file: File, } impl RecordWriter { - /// Create the record at `path`. `create_new` refuses an existing file or - /// symbolic link at `path`, and on Unix the file is mode 0600. - pub(crate) fn create(path: &Path) -> io::Result { + pub(super) fn create(path: &Path) -> io::Result { #[cfg(test)] std::thread::sleep(test_hooks::CREATE_DELAY.with(std::cell::Cell::get)); let mut options = OpenOptions::new(); @@ -43,60 +39,57 @@ impl RecordWriter { }) } - /// Append `line` and its newline with one write. - pub(crate) fn append(&mut self, line: &RecordLine) -> io::Result<()> { + pub(super) fn append(&mut self, line: &RecordLine) -> io::Result { #[cfg(test)] test_hooks::fail_append(&line.body)?; + // Lifecycle/result evidence is retained without the old journal's 64 KiB + // cap. Diagnostic producers bound each message before it reaches here. let mut bytes = serde_json::to_vec(line).map_err(io::Error::other)?; - if bytes.len() > MAX_LINE_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("run record line exceeds {MAX_LINE_BYTES} bytes"), - )); + if matches!(line.body, LineBody::Diagnostic { .. }) + && bytes.len().saturating_add(1) > MAX_DIAGNOSTIC_BYTES + { + return Ok(AppendOutcome::Omitted); } bytes.push(b'\n'); - self.file.write_all(&bytes) + self.file.write_all(&bytes)?; + Ok(AppendOutcome::Written) } - /// Flush the file's data and metadata to stable storage. - pub(crate) fn sync(&self) -> io::Result<()> { + pub(super) fn sync(&self) -> io::Result<()> { #[cfg(test)] test_hooks::fail_if(test_hooks::Fault::Sync)?; self.file.sync_all() } } -/// Per-thread hooks that make record I/O slow or fail in tests. #[cfg(test)] pub(crate) mod test_hooks { + use super::super::lines::LineBody; use std::cell::Cell; use std::io; use std::time::Duration; - use super::super::lines::LineBody; - - /// A record operation that a test makes fail. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Fault { StartAppend, EndAppend, Sync, } - thread_local! { - /// Delay added before each record file is created on this thread. pub(crate) static CREATE_DELAY: Cell = const { Cell::new(Duration::ZERO) }; - /// Operation that fails on this thread. + pub(crate) static INVOCATION_START_DELAY: Cell = const { Cell::new(Duration::ZERO) }; pub(crate) static FAULT: Cell> = const { Cell::new(None) }; } - pub(super) fn fail_append(body: &LineBody) -> io::Result<()> { + if matches!(body, LineBody::InvocationStarted { .. }) { + std::thread::sleep(INVOCATION_START_DELAY.with(Cell::get)); + } match body { - LineBody::Start(_) => fail_if(Fault::StartAppend), - LineBody::End(_) => fail_if(Fault::EndAppend), + LineBody::RequestStarted(_) => fail_if(Fault::StartAppend), + LineBody::Summary(_) => fail_if(Fault::EndAppend), + _ => Ok(()), } } - pub(super) fn fail_if(fault: Fault) -> io::Result<()> { if FAULT.with(Cell::get) == Some(fault) { return Err(io::Error::other(format!("injected {fault:?} failure"))); @@ -108,92 +101,86 @@ pub(crate) mod test_hooks { #[cfg(test)] mod tests { use super::*; + use crate::runtime::new_id; - fn temp_folder(name: &str) -> std::path::PathBuf { - let folder = std::env::temp_dir().join(format!( - "fabric-record-writer-{name}-{}-{}", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - std::fs::create_dir_all(&folder).expect("create temp folder"); - folder + #[test] + fn optional_record_limit_includes_the_terminating_newline() { + use crate::record::lines::{RecordSchema, RecordSource, RecordVersion}; + let folder = std::env::temp_dir().join(new_id("record-byte-limit")); + std::fs::create_dir_all(&folder).unwrap(); + let bodies = [LineBody::Diagnostic { + stream: "stderr".into(), + scope: "runtime".into(), + message: String::new(), + }]; + for (index, body) in bodies.into_iter().enumerate() { + let path = folder.join(format!("optional-{index}.jsonl")); + let mut writer = RecordWriter::create(&path).unwrap(); + let mut line = RecordLine { + schema: RecordSchema::Run, + version: RecordVersion::V1Alpha1, + recording_id: "recording".into(), + request_id: "request".into(), + sequence: 0, + time: "2026-10-25T00:00:00.000Z".into(), + elapsed_seconds: 0.0, + source: RecordSource::Fabric, + runtime_id: None, + invocation_id: None, + body, + }; + let overhead = serde_json::to_vec(&line).unwrap().len() + 1; + let capacity = MAX_DIAGNOSTIC_BYTES - overhead; + let fill = |line: &mut RecordLine, size| match &mut line.body { + LineBody::Diagnostic { message, .. } => *message = "x".repeat(size), + + _ => panic!("expected optional record"), + }; + fill(&mut line, capacity); + assert_eq!(writer.append(&line).unwrap(), AppendOutcome::Written); + assert_eq!( + std::fs::metadata(&path).unwrap().len(), + MAX_DIAGNOSTIC_BYTES as u64 + ); + fill(&mut line, capacity + 1); + assert_eq!(writer.append(&line).unwrap(), AppendOutcome::Omitted); + assert_eq!( + std::fs::metadata(&path).unwrap().len(), + MAX_DIAGNOSTIC_BYTES as u64 + ); + } } #[test] - fn record_writer_refuses_an_existing_file_or_a_symlink() { - let folder = temp_folder("excl"); + fn writer_refuses_existing_files_and_symlinks() { + let folder = std::env::temp_dir().join(new_id("record-writer")); + std::fs::create_dir_all(&folder).unwrap(); let path = folder.join(RECORD_FILE_NAME); - std::fs::write(&path, "old").expect("seed file"); - - let error = RecordWriter::create(&path).expect_err("existing file"); - - assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); - assert_eq!(std::fs::read_to_string(&path).expect("read"), "old"); + std::fs::write(&path, b"existing evidence").unwrap(); + assert_eq!( + RecordWriter::create(&path).unwrap_err().kind(), + io::ErrorKind::AlreadyExists + ); + assert_eq!(std::fs::read(&path).unwrap(), b"existing evidence"); #[cfg(unix)] { - let linked = folder.join("linked"); - std::fs::create_dir_all(&linked).expect("linked folder"); - let target = folder.join("target"); - std::os::unix::fs::symlink(&target, linked.join(RECORD_FILE_NAME)).expect("symlink"); - assert!(RecordWriter::create(&linked.join(RECORD_FILE_NAME)).is_err()); - assert!(!target.exists()); + let link = folder.join("linked.jsonl"); + std::os::unix::fs::symlink(&path, &link).unwrap(); + assert!(RecordWriter::create(&link).is_err()); } } - #[cfg(unix)] #[test] - fn record_writer_new_file_is_owner_only() { + #[cfg(unix)] + fn writer_creates_owner_only_file() { use std::os::unix::fs::PermissionsExt; - let folder = temp_folder("mode"); + let folder = std::env::temp_dir().join(new_id("record-private")); + std::fs::create_dir_all(&folder).unwrap(); let path = folder.join(RECORD_FILE_NAME); - - RecordWriter::create(&path).expect("create"); - let mode = std::fs::metadata(&path).expect("stat").permissions().mode(); - - assert_eq!(mode & 0o777, 0o600); - } - - #[test] - fn line_byte_limit_accepts_exact_boundary_and_rejects_oversize_without_appending() { - use crate::record::lines::{EndLine, LineBody, RecordSchema, RecordVersion}; - use crate::runtime::RunStatus; - - let folder = temp_folder("line-size"); - let path = folder.join(RECORD_FILE_NAME); - let mut writer = RecordWriter::create(&path).expect("writer"); - let mut line = RecordLine { - schema: RecordSchema::RunRecord, - version: RecordVersion::V1Alpha1, - runtime_id: "rt".to_string(), - invocation_id: "inv".to_string(), - request_id: "req".to_string(), - time: "2026-10-21T00:00:00.000Z".to_string(), - body: LineBody::End(Box::new(EndLine { - outcome_scope: "invocation", - result: None, - status: RunStatus::Succeeded, - elapsed_ms: 0, - error: None, - usage: None, - model_usage: None, - artifacts: Vec::new(), - })), - }; - let overhead = serde_json::to_vec(&line).expect("line JSON").len(); - line.request_id - .push_str(&"a".repeat(MAX_LINE_BYTES - overhead)); - assert_eq!(serde_json::to_vec(&line).unwrap().len(), MAX_LINE_BYTES); - writer.append(&line).expect("exact limit is valid"); - let before = std::fs::read(&path).expect("written line"); - assert_eq!(before.len(), MAX_LINE_BYTES + 1); - assert_eq!(before.last(), Some(&b'\n')); - - line.request_id.push('a'); - let error = writer.append(&line).expect_err("one byte too large"); - assert_eq!(error.kind(), io::ErrorKind::InvalidData); - assert_eq!(std::fs::read(&path).unwrap(), before); + let _writer = RecordWriter::create(&path).unwrap(); + assert_eq!( + std::fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o600 + ); } } diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 5aaacaac2..405dd0b1e 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -32,7 +32,7 @@ use crate::config::{ }; use crate::environment::resolve_environment_provider; use crate::error::{FabricError, Result}; -use crate::record::{InvocationRecorder, RecordStart}; +use crate::record::{OutcomeScope, RunRecorder}; use crate::resource_snapshot::{LocalHostProcess, RuntimeResourceSnapshot, snapshot_local_host}; #[cfg(test)] use crate::system_stats::ResourceObservation; @@ -42,8 +42,6 @@ mod openshell; use openshell::OpenShellRuntimeAdapter; static NEXT_ID: AtomicU64 = AtomicU64::new(1); -/// Lifecycle code for a result rejected by core. -pub(crate) const INVALID_AGENT_RUN_RESULT: &str = "invalid_agent_run_result"; const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; const LOCAL_HOST_START_TIMEOUT: Duration = Duration::from_secs(90); @@ -905,17 +903,40 @@ pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { #[doc(hidden)] pub fn run_plan_with_stage( plan: &RunPlan, - request: RunRequest, + mut request: RunRequest, ) -> std::result::Result { - let runtime = start_runtime_staged(plan)?; - let mut result = match invoke_runtime_unpersisted(plan, &runtime, request) { + ensure_request_id(&mut request); + let recorder = RunRecorder::start(plan, &request, OutcomeScope::OneShotLifecycle); + let runtime = match start_runtime_staged(plan) { + Ok(runtime) => runtime, + Err((error, stage)) => { + return Err(( + finish_recording(recorder.as_ref(), Err(error), stage).unwrap_err(), + stage, + )); + } + }; + if let Some(recorder) = &recorder { + recorder.runtime_ready(&runtime); + } + let mut result = match invoke_runtime_unpersisted(plan, &runtime, request, recorder.as_ref()) { Ok(result) => result, Err(error) => { - let _ = stop_runtime(plan, &runtime); - return Err((error, ErrorStage::Invoke)); + let stopped = stop_runtime(plan, &runtime); + if let Some(recorder) = &recorder { + recorder.runtime_stopped(&stopped); + } + return Err(( + finish_recording(recorder.as_ref(), Err(error), ErrorStage::Invoke).unwrap_err(), + ErrorStage::Invoke, + )); } }; - match stop_runtime(plan, &runtime) { + let stopped = stop_runtime(plan, &runtime); + if let Some(recorder) = &recorder { + recorder.runtime_stopped(&stopped); + } + match stopped { Ok(events) => result.events.extend(events), Err(error) if result.status == RunStatus::Succeeded => { result.status = RunStatus::Failed; @@ -929,11 +950,27 @@ pub fn run_plan_with_stage( } Err(_) => {} } - Ok(crate::record::persist_result( - plan, - result, - "one_shot_lifecycle", - )) + Ok( + finish_recording(recorder.as_ref(), Ok(result), ErrorStage::Stop) + .expect("recording preserves a returned result"), + ) +} + +fn ensure_request_id(request: &mut RunRequest) { + if request.request_id.is_empty() { + request.request_id = new_id("request"); + } +} + +fn finish_recording( + recorder: Option<&RunRecorder>, + outcome: Result, + stage: ErrorStage, +) -> Result { + match recorder { + Some(recorder) => recorder.finish(outcome, stage), + None => outcome, + } } /// Prepare an execution environment for a run plan. @@ -1215,16 +1252,22 @@ fn start_runtime_staged( pub fn invoke_runtime( plan: &RunPlan, runtime: &RuntimeHandle, - request: RunRequest, + mut request: RunRequest, ) -> Result { - invoke_runtime_unpersisted(plan, runtime, request) - .map(|result| crate::record::persist_result(plan, result, "invocation")) + ensure_request_id(&mut request); + let recorder = RunRecorder::start(plan, &request, OutcomeScope::Invocation); + if let Some(recorder) = &recorder { + recorder.set_runtime(runtime); + } + let outcome = invoke_runtime_unpersisted(plan, runtime, request, recorder.as_ref()); + finish_recording(recorder.as_ref(), outcome, ErrorStage::Invoke) } fn invoke_runtime_unpersisted( plan: &RunPlan, runtime: &RuntimeHandle, request: RunRequest, + recorder: Option<&RunRecorder>, ) -> Result { validate_adapter_compatibility(plan)?; validate_runtime_handle(plan, runtime)?; @@ -1232,7 +1275,14 @@ fn invoke_runtime_unpersisted( if runtime.environment.provider == "openshell" { return OpenShellRuntimeAdapter.invoke(plan, runtime, request); } - return LocalHostAdapter.invoke(plan, runtime, request); + return run_local_host_invocation_with_recorder( + plan, + runtime, + request, + LocalHostInvocation::Invoke, + local_host_invoke_timeout(plan)?, + recorder, + ); } Err(FabricError::UnsupportedRuntimeAdapter { harness: harness(plan), @@ -1285,34 +1335,48 @@ fn local_runtime_resource_snapshot(runtime: &RuntimeHandle) -> Result Result { - validate_adapter_compatibility(plan)?; - validate_runtime_handle(plan, runtime)?; - let descriptor_supports_streaming = plan - .adapter_descriptor - .as_ref() - .is_some_and(|adapter| adapter.descriptor.capabilities.streaming); - if !plan.capabilities.streaming || !descriptor_supports_streaming { - return Err(FabricError::UnsupportedRuntimeCapability { - adapter_id: adapter_id(plan).unwrap_or_else(|| harness(plan)), - capability: "streaming", - }); - } - validate_openai_stream_transport(&transport)?; - if uses_local_host(plan) { - if runtime.environment.provider == "openshell" { - return OpenShellRuntimeAdapter.invoke_openai_stream(plan, runtime, request, transport); + ensure_request_id(&mut request); + let recorder = RunRecorder::start(plan, &request, OutcomeScope::Invocation); + if let Some(recorder) = &recorder { + recorder.set_runtime(runtime); + } + let outcome = (|| { + validate_adapter_compatibility(plan)?; + validate_runtime_handle(plan, runtime)?; + let descriptor_supports_streaming = plan + .adapter_descriptor + .as_ref() + .is_some_and(|adapter| adapter.descriptor.capabilities.streaming); + if !plan.capabilities.streaming || !descriptor_supports_streaming { + return Err(FabricError::UnsupportedRuntimeCapability { + adapter_id: adapter_id(plan).unwrap_or_else(|| harness(plan)), + capability: "streaming", + }); } - return LocalHostAdapter - .invoke_openai_stream(plan, runtime, request, transport) - .map(|result| crate::record::persist_result(plan, result, "invocation")); - } - Err(FabricError::UnsupportedRuntimeAdapter { - harness: harness(plan), - adapter_kind: adapter_kind(plan), - }) + validate_openai_stream_transport(&transport)?; + if uses_local_host(plan) { + if runtime.environment.provider == "openshell" { + return OpenShellRuntimeAdapter + .invoke_openai_stream(plan, runtime, request, transport); + } + return run_local_host_invocation_with_recorder( + plan, + runtime, + request, + LocalHostInvocation::OpenAiStream(transport), + local_host_invoke_timeout(plan)?, + recorder.as_ref(), + ); + } + Err(FabricError::UnsupportedRuntimeAdapter { + harness: harness(plan), + adapter_kind: adapter_kind(plan), + }) + })(); + finish_recording(recorder.as_ref(), outcome, ErrorStage::Invoke) } fn validate_openai_stream_transport(transport: &OpenAiStreamTransport) -> Result<()> { @@ -2001,25 +2065,48 @@ impl LocalHostInvocation { } fn run_local_host_invocation_with_timeout( + plan: &RunPlan, + runtime: &RuntimeHandle, + mut request: RunRequest, + invocation_kind: LocalHostInvocation, + invoke_timeout: Duration, +) -> Result { + ensure_request_id(&mut request); + let recorder = RunRecorder::start(plan, &request, OutcomeScope::Invocation); + if let Some(recorder) = &recorder { + recorder.set_runtime(runtime); + } + let outcome = run_local_host_invocation_with_recorder( + plan, + runtime, + request, + invocation_kind, + invoke_timeout, + recorder.as_ref(), + ); + finish_recording(recorder.as_ref(), outcome, ErrorStage::Invoke) +} + +fn run_local_host_invocation_with_recorder( plan: &RunPlan, runtime: &RuntimeHandle, request: RunRequest, invocation_kind: LocalHostInvocation, invoke_timeout: Duration, + recorder: Option<&RunRecorder>, ) -> Result { - let mut recorder = None; let outcome = local_host_invocation_result( plan, runtime, request, invocation_kind, invoke_timeout, - &mut recorder, + recorder, ); - match recorder { - Some(recorder) => recorder.finish(outcome), - None => outcome, + if let Some(recorder) = recorder { + recorder.invocation_finished(&outcome); } + outcome } fn local_host_invocation_result( @@ -2028,7 +2115,7 @@ fn local_host_invocation_result( mut request: RunRequest, invocation_kind: LocalHostInvocation, invoke_timeout: Duration, - recorder: &mut Option, + recorder: Option<&RunRecorder>, ) -> Result { let operation = invocation_kind.operation(); if request.request_id.is_empty() { @@ -2054,6 +2141,9 @@ fn local_host_invocation_result( let exchange_result = { let mut host_guard = host.lock().unwrap_or_else(|error| error.into_inner()); + if let Some(recorder) = recorder { + recorder.invocation_started(&invocation, invoke_timeout); + } let artifacts = host_guard.artifacts.clone(); let relay_config = host_guard.relay_config.clone(); let fabric_home = prepare_fabric_home(&artifacts, runtime, &invocation)?; @@ -2070,17 +2160,6 @@ fn local_host_invocation_result( &host_guard.environment_env, &adapter_invocation.runtime_context.environment.env, )?; - // Record preparation precedes the adapter deadline, preserving its budget. - if plan.config.runtime.record_enabled() && artifacts.root.is_some() { - *recorder = Some(InvocationRecorder::start(RecordStart { - plan, - runtime, - invocation: &invocation, - request: &request, - fabric_home: &fabric_home, - timeout: invoke_timeout, - })); - } adapter_invocation.runtime_context.deadline_millis = adapter_deadline_millis(invoke_timeout); let (lifecycle_request, adapter_payload) = match invocation_kind { @@ -2147,6 +2226,9 @@ fn local_host_invocation_result( sampling.close(); } let elapsed_seconds = invocation_started.elapsed().as_secs_f64(); + if let Some(recorder) = recorder { + recorder.exchange_ended(elapsed_seconds); + } // Endpoint collection and final artifact writes are outside elapsed time, // and still under the host lock to exclude the next invocation. let after = stats_config.map(|_| reader.read()); @@ -2319,27 +2401,33 @@ fn local_host_invocation_result( ), ]), )); - let mut stdout = serde_json::to_string(&output).map_err(FabricError::SerializeJson)?; - stdout.push('\n'); - write_artifact( - &mut artifacts, - &fabric_home, - "stdout", - "log", - "stdout.txt", - &stdout, - "text/plain", - )?; - if !stderr.is_empty() { + if let Some(recorder) = recorder { + if !stderr.is_empty() { + recorder.diagnostic("stderr", &stderr); + } + } else { + let mut stdout = serde_json::to_string(&output).map_err(FabricError::SerializeJson)?; + stdout.push('\n'); write_artifact( &mut artifacts, &fabric_home, - "stderr", + "stdout", "log", - "stderr.txt", - &stderr, + "stdout.txt", + &stdout, "text/plain", )?; + if !stderr.is_empty() { + write_artifact( + &mut artifacts, + &fabric_home, + "stderr", + "log", + "stderr.txt", + &stderr, + "text/plain", + )?; + } } if let Some(sampled) = system_stats .as_ref() @@ -6846,7 +6934,7 @@ for line in sys.stdin: for path in entries.flatten().map(|entry| entry.path()) { if path.is_dir() { folders.push(path); - } else if path.ends_with("run-record.jsonl") { + } else if path.ends_with("run.jsonl") { records.push(path); } } @@ -6899,7 +6987,7 @@ for line in sys.stdin: .artifacts .artifacts .iter() - .all(|artifact| artifact.kind != "run_record") + .all(|artifact| artifact.kind != "run_log") ); assert!( result @@ -6911,146 +6999,149 @@ for line in sys.stdin: } #[test] - fn record_on_writes_a_start_line_and_an_end_line() { + fn recorded_request_keeps_one_final_result_and_real_lifecycle_events() { let (root, plan) = record_plan("typed_result"); let mut request = RunRequest::text("record"); request.context = BTreeMap::from([ ("task_id".to_string(), serde_json::json!("repo-io-fix")), ("attempt".to_string(), serde_json::json!(2)), ( - "harbor_trial_name".to_string(), - serde_json::json!("trial-1"), + "step".to_string(), + serde_json::json!({"nested":"not copied"}), ), - ("step".to_string(), serde_json::json!({"nested": "value"})), ("api_key".to_string(), serde_json::json!("sk-live-SECRET")), ]); - let result = run_plan(&plan, request).expect("run"); let (text, lines) = only_record(&root); - - let (start, end) = (line_of_kind(&lines, "start"), line_of_kind(&lines, "end")); - assert_eq!(lines.first(), Some(start), "{text}"); - assert_eq!(lines.last(), Some(end), "{text}"); - for line in &lines { - assert_eq!(line["schema"], "nemo.fabric.run_record"); - assert_eq!(line["version"], "v1alpha1"); - assert_eq!(line["runtime_id"], result.runtime_id.as_str()); - assert_eq!(line["invocation_id"], result.invocation_id.as_str()); - assert_eq!(line["request_id"], result.request_id.as_str()); - } - assert_eq!(start["agent_name"], "local-host-test-agent"); - assert_eq!(start["adapter_id"], "acme.fabric.local-host"); - assert_eq!(start["timeout_ms"], 3_600_000); + let start = line_of_kind(&lines, "request_started"); + let end = line_of_kind(&lines, "summary"); + assert_eq!(lines.first(), Some(start)); + assert_eq!(lines.last(), Some(end)); + assert_eq!(start["data"]["agent_name"], "local-host-test-agent"); assert_eq!( - start["context"], + start["data"]["context"], serde_json::json!({ - "api_key": {"recorded": false}, - "attempt": 2, - "harbor_trial_name": "trial-1", - "step": {"recorded": false}, - "task_id": "repo-io-fix", + "api_key":{"recorded":false},"attempt":2,"step":{"recorded":false},"task_id":"repo-io-fix" }) ); - assert!(!text.contains("SECRET"), "{text}"); - assert_eq!(end["status"], "succeeded"); - assert!(end["elapsed_ms"].is_u64()); + assert!(!text.contains("SECRET")); + assert!(start.get("runtime_id").is_none()); + assert!(start.get("invocation_id").is_none()); + let mut previous_elapsed = 0.0; + for (index, line) in lines.iter().enumerate() { + assert_eq!(line["schema"], "nemo.fabric.run"); + assert_eq!(line["sequence"], index as u64); + assert_eq!(line["version"], "v1alpha1"); + assert_eq!(line["recording_id"], start["recording_id"]); + assert_eq!(line["request_id"], result.request_id); + assert!(line["time"].as_str().unwrap().ends_with('Z')); + let elapsed = line["elapsed_seconds"].as_f64().unwrap(); + assert!(elapsed >= previous_elapsed); + previous_elapsed = elapsed; + } + assert_eq!(end["data"]["outcome_scope"], "one_shot_lifecycle"); + assert_eq!( + end["data"]["result"], + serde_json::to_value(&result).unwrap() + ); assert_eq!( - end["usage"], - serde_json::json!({ - "input_tokens": 3, - "cached_input_tokens": 1, - "input_tokens_include_cache": true, - "output_tokens": 5, - "total_tokens": 8, - "cost_usd": 0.25, - }) + line_of_kind(&lines, "invocation_finished")["data"]["status"], + "succeeded" ); - let report = end["artifacts"] - .as_array() - .expect("artifacts") - .iter() - .find(|artifact| artifact["name"] == "report") - .expect("report artifact"); assert_eq!( - report, - &serde_json::json!({ - "name": "report", - "kind": "report", - "media_type": "application/json", - "path": "reports/result.json", - }) + line_of_kind(&lines, "runtime_stopped")["data"]["succeeded"], + true ); - for marker in ["fake-host", "\"fake\"", "trace-1"] { - assert!(!text.contains(marker), "{marker} in {text}"); - } let record = result .artifacts .artifacts .iter() - .find(|artifact| artifact.name == "record") - .expect("record artifact"); - assert_eq!(record.kind, "run_record"); - assert_eq!(record.media_type.as_deref(), Some("application/jsonl")); + .find(|artifact| artifact.kind == "run_log") + .unwrap(); assert_eq!(record.path, record_files(&root)[0]); - let _ = fs::remove_dir_all(root); + assert_eq!(record.media_type.as_deref(), Some("application/x-ndjson")); + assert!( + result + .artifacts + .artifacts + .iter() + .all(|artifact| !matches!(artifact.kind.as_str(), "run_record" | "run_result")) + ); } #[test] - fn record_end_line_keeps_the_error_code_and_drops_the_message() { + fn recorded_failed_result_preserves_the_returned_error() { let (root, plan) = record_plan("adapter_reported_failure"); - - let result = run_plan(&plan, RunRequest::text("fail")).expect("failed result"); - let (text, lines) = only_record(&root); - + let result = run_plan(&plan, RunRequest::text("fail")).unwrap(); + let (_, lines) = only_record(&root); assert_eq!(result.status, RunStatus::Failed); - assert_eq!( - line_of_kind(&lines, "end")["error"], - serde_json::json!({"code": "fake_adapter_failure", "stage": "invoke", "retryable": true}) - ); - assert!(!text.contains("adapter rejected"), "{text}"); - let mut wrong_kind = line_of_kind(&lines, "end").clone(); - wrong_kind["kind"] = serde_json::json!("start"); - assert!(!record_line_validator().is_valid(&wrong_kind)); - let _ = fs::remove_dir_all(root); + let end = &line_of_kind(&lines, "summary")["data"]; + assert_eq!(end["result"], serde_json::to_value(&result).unwrap()); + assert_eq!(end["result"]["error"]["code"], "fake_adapter_failure"); + assert_eq!(end["result"]["error"]["retryable"], true); + let mut invalid = line_of_kind(&lines, "summary").clone(); + invalid["kind"] = serde_json::json!("unknown_kind"); + assert!(!record_line_validator().is_valid(&invalid)); } #[test] - fn record_end_line_replaces_a_free_text_error_code() { - let (root, plan) = record_plan("free_text_error_code"); - - let result = run_plan(&plan, RunRequest::text("fail")).expect("failed result"); - let (text, lines) = only_record(&root); - - assert!(result.error.expect("error").code.contains("sk-live-123")); - assert_eq!(line_of_kind(&lines, "end")["error"]["code"], "unrecognized"); - assert!(!text.contains("sk-live-123"), "{text}"); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn record_lifecycle_error_writes_a_failed_end_line() { + fn recorded_timeout_retains_error_without_inventing_a_result() { let (root, plan) = record_plan("invoke_timeout"); - let runtime = start_runtime(&plan).expect("start local host"); - + let runtime = start_runtime(&plan).unwrap(); let error = run_local_host_adapter_with_timeout( &plan, &runtime, RunRequest::text("hang"), Duration::from_millis(100), ) - .expect_err("timeout raises"); + .unwrap_err(); let (_, lines) = only_record(&root); - - assert!(error.to_string().contains("host_timeout"), "{error}"); - let end = line_of_kind(&lines, "end"); - assert_eq!(line_of_kind(&lines, "start")["timeout_ms"], 100); - assert_eq!(end["status"], "failed"); + assert!(error.to_string().contains("host_timeout")); assert_eq!( - end["error"], - serde_json::json!({"code": "host_timeout", "stage": "invoke", "retryable": false}) + line_of_kind(&lines, "invocation_started")["data"]["timeout_seconds"], + 0.1 ); + let end = &line_of_kind(&lines, "summary")["data"]; + assert!(end["result"].is_null()); + assert_eq!(end["error"]["stage"], "invoke"); + assert_eq!(end["error"]["code"], "host_timeout"); let _ = stop_runtime(&plan, &runtime); - let _ = fs::remove_dir_all(root); + } + + #[test] + fn recorded_start_failure_keeps_request_identity_without_inner_ids() { + let (root, plan) = record_plan("start_failure"); + let mut request = RunRequest::text("never invoked"); + request.request_id = "startup-failure-request".to_string(); + assert!(run_plan(&plan, request).is_err()); + let (_, lines) = only_record(&root); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0]["request_id"], "startup-failure-request"); + assert!(lines[0].get("runtime_id").is_none()); + assert!(lines[1].get("invocation_id").is_none()); + assert!(lines[1]["data"]["result"].is_null()); + assert_eq!(lines[1]["data"]["error"]["stage"], "start"); + } + + #[test] + fn recorded_shutdown_failure_preserves_successful_invocation_evidence() { + let (root, plan) = record_plan("stop_failure"); + let result = run_plan(&plan, RunRequest::text("stop fails")).unwrap(); + let (_, lines) = only_record(&root); + assert_eq!( + line_of_kind(&lines, "invocation_finished")["data"]["status"], + "succeeded" + ); + assert_eq!( + line_of_kind(&lines, "runtime_stopped")["data"]["succeeded"], + false + ); + assert_eq!(result.status, RunStatus::Failed); + assert_eq!(result.error.as_ref().unwrap().stage, ErrorStage::Stop); + assert_eq!( + line_of_kind(&lines, "summary")["data"]["result"], + serde_json::to_value(result).unwrap() + ); } #[test] @@ -7060,18 +7151,20 @@ for line in sys.stdin: let runtime = start_runtime(&plan).expect("start local host"); let delay = Duration::from_millis(500); crate::record::test_hooks::CREATE_DELAY.with(|slot| slot.set(delay)); + crate::record::test_hooks::INVOCATION_START_DELAY.with(|slot| slot.set(delay)); let before = now_millis(); let result = invoke_runtime(&plan, &runtime, RunRequest::text("slow record storage")); let after = now_millis(); crate::record::test_hooks::CREATE_DELAY.with(|slot| slot.set(Duration::ZERO)); + crate::record::test_hooks::INVOCATION_START_DELAY.with(|slot| slot.set(Duration::ZERO)); let result = result.expect("invoke"); let deadline = u128::from(result.output["deadline_millis"].as_u64().expect("deadline")); - // The adapter's 50-second budget starts after the record is created, - // so slow record storage does not eat into it. + // Both file creation and the invocation-start write precede the + // adapter's 50-second budget; slow recording must not consume it. assert!( - (before + delay.as_millis() + 50_000..=after + 50_000).contains(&deadline), + (before + 2 * delay.as_millis() + 50_000..=after + 50_000).contains(&deadline), "deadline {deadline}, before {before}, after {after}" ); assert_eq!(record_files(&root).len(), 1); @@ -7080,23 +7173,41 @@ for line in sys.stdin: } #[test] - fn record_warm_runtime_writes_one_record_per_invocation() { + fn warm_runtime_requests_have_independent_journals_without_fake_startup() { let (root, plan) = record_plan("success"); - let runtime = start_runtime(&plan).expect("start local host"); - - let first = invoke_runtime(&plan, &runtime, RunRequest::text("one")).expect("first"); - let second = invoke_runtime(&plan, &runtime, RunRequest::text("two")).expect("second"); - stop_runtime(&plan, &runtime).expect("stop"); - - let records = record_files(&root); - assert_eq!(records.len(), 2); + let runtime = start_runtime(&plan).unwrap(); + let first = invoke_runtime(&plan, &runtime, RunRequest::text("one")).unwrap(); + let second = invoke_runtime(&plan, &runtime, RunRequest::text("two")).unwrap(); + stop_runtime(&plan, &runtime).unwrap(); + assert_eq!(record_files(&root).len(), 2); + let mut recordings = Vec::new(); for result in [&first, &second] { - assert!(records.iter().any(|path| path.ends_with(format!( - ".fabric/{}/{}/run-record.jsonl", - result.runtime_id, result.invocation_id - )))); + let reference = result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.kind == "run_log") + .unwrap(); + let rows: Vec = fs::read_to_string(&reference.path) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + recordings.push(rows[0]["recording_id"].clone()); + assert!( + rows.iter() + .all(|row| row["kind"] != "runtime_ready" && row["kind"] != "runtime_stopped") + ); + assert_eq!( + line_of_kind(&rows, "summary")["data"]["outcome_scope"], + "invocation" + ); + assert_eq!( + line_of_kind(&rows, "summary")["invocation_id"], + result.invocation_id + ); } - let _ = fs::remove_dir_all(root); + assert_ne!(recordings[0], recordings[1]); } #[test] fn semantic_accounting_conflict_preserves_execution_and_raw_evidence() { @@ -7138,7 +7249,7 @@ for line in sys.stdin: let (root, plan) = record_plan(mode); let result = run_plan(&plan, RunRequest::text("accounted")).expect("result"); let (_, rows) = only_record(&root); - let end = line_of_kind(&rows, "end"); + let end = &line_of_kind(&rows, "summary")["data"]["result"]; let usage = result.model_usage.expect("normalized accounting"); assert_eq!(end["model_usage"], serde_json::to_value(&usage).unwrap()); assert_eq!(usage.tokens.output_tokens, Some(5)); @@ -7148,14 +7259,13 @@ for line in sys.stdin: } else { assert_eq!(usage.tokens.input_tokens, Some(100)); } - let (record, _) = only_record(&root); - assert!(!record.contains("peak_request_input_tokens")); + assert_eq!(end["usage"], serde_json::to_value(&result.usage).unwrap()); assert!( result .artifacts .artifacts .iter() - .any(|a| a.kind == "run_record") + .any(|a| a.kind == "run_log") ); let _ = fs::remove_dir_all(root); } @@ -7181,13 +7291,18 @@ for line in sys.stdin: .artifacts .artifacts .iter() - .find(|artifact| artifact.kind == "run_result") + .find(|artifact| artifact.kind == "run_log") .expect("streaming saves the full result"); - let decoded: RunResult = - serde_json::from_slice(&fs::read(&saved.path).expect("saved result")) - .expect("typed saved result"); - assert_eq!(decoded, result); - assert!(saved.path.with_file_name("run-record.jsonl").is_file()); + let rows: Vec = fs::read_to_string(&saved.path) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!( + line_of_kind(&rows, "summary")["data"]["result"], + serde_json::to_value(&result).unwrap() + ); + assert_eq!(saved.path.file_name().unwrap(), "run.jsonl"); let _ = fs::remove_dir_all(root); } } diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index 9d8069ea5..31396f98b 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -73,7 +73,7 @@ pub enum SchemaName { FabricEvent, /// Runtime resource snapshot schema. RuntimeResourceSnapshot, - /// One bounded start/end invocation record line. + /// One typed event or terminal summary in a recorded request journal. RunRecordLine, } diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index 1a443c1ca..a5f7f9680 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -11,7 +11,7 @@ SPDX-License-Identifier: Apache-2.0 --> ## Modules - [`nemo_fabric.accounting`](./nemo_fabric.accounting.md#module-nemo_fabricaccounting): Typed model accounting and invocation summaries with explicit coverage. -- [`nemo_fabric.reporting`](./nemo_fabric.reporting.md#module-nemo_fabricreporting): Inspect saved invocation evidence without a benchmark manifest or trace import. +- [`nemo_fabric.reporting`](./nemo_fabric.reporting.md#module-nemo_fabricreporting): Inspect saved run journals and legacy results without importing traces. - [`nemo_fabric.client`](./nemo_fabric.client.md#module-nemo_fabricclient): Native Python client for resolving and running NVIDIA NeMo Fabric agents. - [`nemo_fabric.capabilities`](./nemo_fabric.capabilities.md#module-nemo_fabriccapabilities): Metadata-only adapter inspection for host-side admission. - [`nemo_fabric.runtime`](./nemo_fabric.runtime.md#module-nemo_fabricruntime): Runtime lifecycle support for the NVIDIA NeMo Fabric Python SDK. @@ -47,7 +47,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`models.MetadataConfig`](./nemo_fabric.models.md#class-metadataconfig): Human-readable agent identity. - [`models.ModelConfig`](./nemo_fabric.models.md#class-modelconfig): Configuration for one model role. - [`models.PricingConfig`](./nemo_fabric.models.md#class-pricingconfig): Frozen Relay-compatible price-catalog override, read during planning. -- [`models.RecordConfig`](./nemo_fabric.models.md#class-recordconfig): Opt-in per-invocation run record. +- [`models.RecordConfig`](./nemo_fabric.models.md#class-recordconfig): Opt-in local request journal. - [`models.RelayAtifConfig`](./nemo_fabric.models.md#class-relayatifconfig): NeMo Relay ATIF export configuration. - [`models.RelayAtofConfig`](./nemo_fabric.models.md#class-relayatofconfig): NeMo Relay ATOF export configuration. - [`models.RelayAtofFileSinkConfig`](./nemo_fabric.models.md#class-relayatoffilesinkconfig): NeMo Relay ATOF file sink configuration. @@ -117,12 +117,12 @@ SPDX-License-Identifier: Apache-2.0 --> ## Functions - [`accounting.summarize_usage`](./nemo_fabric.accounting.md#function-summarize_usage): Summarize invocations against an explicit expected identity manifest. -- [`reporting.main`](./nemo_fabric.reporting.md#function-main): Inspect a saved invocation result as a concise summary or JSON. -- [`reporting.read_evidence`](./nemo_fabric.reporting.md#function-read_evidence): Read a full result or invocation record; never follow references. -- [`reporting.read_invocation_record`](./nemo_fabric.reporting.md#function-read_invocation_record): Read bounded start/end evidence; a truncated tail never creates a result. +- [`reporting.main`](./nemo_fabric.reporting.md#function-main): Inspect a saved run journal or legacy result as readable text or JSON. +- [`reporting.read_evidence`](./nemo_fabric.reporting.md#function-read_evidence): Read a run journal or legacy result without opening referenced artifacts. +- [`reporting.read_invocation_record`](./nemo_fabric.reporting.md#function-read_invocation_record): Read legacy start/end evidence; a truncated tail never creates a result. - [`reporting.read_result`](./nemo_fabric.reporting.md#function-read_result): Read one saved full result without requiring task labels or a manifest. -- [`reporting.render_invocation_record_summary`](./nemo_fabric.reporting.md#function-render_invocation_record_summary): Describe the invocation interval without assuming whole-runtime success. -- [`reporting.render_result_summary`](./nemo_fabric.reporting.md#function-render_result_summary): Render a concise overview; details add resource evidence. +- [`reporting.render_invocation_record_summary`](./nemo_fabric.reporting.md#function-render_invocation_record_summary): Render legacy invocation evidence without claiming one-shot completion. +- [`reporting.render_result_summary`](./nemo_fabric.reporting.md#function-render_result_summary): Render concise measurement values; details add retained history and sources. - [`reporting.summarize_result`](./nemo_fabric.reporting.md#function-summarize_result): Describe one returned invocation, preserving independent usable evidence. - [`capabilities.inspect_adapter`](./nemo_fabric.capabilities.md#function-inspect_adapter): Inspect a standalone runtime using descriptor metadata without loading its SDK. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index cfc3fb4a8..a02b54e75 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -483,9 +483,9 @@ Return a detached JSON-compatible mapping for Rust/core calls. ## class `RecordConfig` -Opt-in per-invocation run record. +Opt-in local request journal. -When ``enabled`` is true, NeMo Fabric writes ``/.fabric///run-record.jsonl`` for each invocation. It requires an artifact root, the ``local`` environment provider, and a ``process`` or ``python`` adapter. +When ``enabled`` is true, NeMo Fabric appends lifecycle events and a terminal summary to ``/.fabric/requests//run.jsonl`` for each request. The journal can contain sensitive result data and error messages. It requires an artifact root, the ``local`` environment provider, and a ``process`` or ``python`` adapter. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.reporting.md b/docs/reference/api/python-library-reference/nemo_fabric.reporting.md index c88c9a077..ae4b6abe2 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.reporting.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.reporting.md @@ -8,15 +8,22 @@ SPDX-License-Identifier: Apache-2.0 --> # module `nemo_fabric.reporting` -Inspect saved invocation evidence without a benchmark manifest or trace import. +Inspect saved run journals and legacy results without importing traces. **Global Variables** --------------- - **MAX_RESULT_BYTES** +- **MAX_HISTORY_EVENTS** +- **MAX_HISTORY_BYTES** +- **MAX_HISTORY_MILESTONE_RESERVE_BYTES** +- **MAX_HISTORY_FIELD_CHARS** +- **MAX_EVENT_BYTES** +- **MAX_HISTORY_MESSAGE_CHARS** - **RESOURCE_SCOPE** --- + ## function `summarize_result` ```python @@ -30,6 +37,7 @@ This is not a task verdict or proof of a complete scheduled run. Normalized acco --- + ## function `read_result` ```python @@ -41,30 +49,33 @@ Read one saved full result without requiring task labels or a manifest. --- + ## function `read_evidence` ```python def read_evidence(path: Path) -> dict[str, Any] ``` -Read a full result or invocation record; never follow references. +Read a run journal or legacy result without opening referenced artifacts. --- + ## function `read_invocation_record` ```python def read_invocation_record(path: Path) -> dict[str, Any] ``` -Read bounded start/end evidence; a truncated tail never creates a result. +Read legacy start/end evidence; a truncated tail never creates a result. This reports the core invocation window before one-shot shutdown. Only the typed model_usage projection supplies normalized accounting; legacy counters never gain a completeness guarantee. --- + ## function `render_invocation_record_summary` ```python @@ -75,31 +86,31 @@ def render_invocation_record_summary( ) -> str ``` -Describe the invocation interval without assuming whole-runtime success. +Render legacy invocation evidence without claiming one-shot completion. --- + ## function `render_result_summary` ```python def render_result_summary(summary: Mapping[str, Any], *, details: bool = False) -> str ``` -Render a concise overview; details add resource evidence. - -Token and cost completeness remain separate. JSON quantities and accounting semantics are unchanged by this presentation. +Render concise measurement values; details add retained history and sources. --- + ## function `main` ```python def main() -> None ``` -Inspect a saved invocation result as a concise summary or JSON. +Inspect a saved run journal or legacy result as readable text or JSON. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index d9d2fc049..2c0f454a2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -54,7 +54,7 @@ NeMo Fabric config models and loading helpers. - [MetadataConfig](struct-metadataconfig.mdx): Human-readable metadata. - [ModelConfig](struct-modelconfig.mdx): Model configuration. - [PricingConfig](struct-pricingconfig.mdx): Central token-price configuration; never changes provider requests. -- [RecordConfig](struct-recordconfig.mdx): Per-invocation run record settings. +- [RecordConfig](struct-recordconfig.mdx): Opt-in local request journal settings. - [RelayAtifConfig](struct-relayatifconfig.mdx): Relay ATIF export configuration. - [RelayAtofConfig](struct-relayatofconfig.mdx): Relay ATOF export configuration. - [RelayComponentConfig](struct-relaycomponentconfig.mdx): Generic NeMo Relay plugin component configuration. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx index c49f1c72e..ea7a2f24b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx @@ -1,7 +1,7 @@ --- title: "Struct Record Config" sidebar-title: "RecordConfig" -description: "Per-invocation run record settings." +description: "Opt-in local request journal settings." position: 55 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -11,15 +11,15 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. 
bool,\n}"}} />
-Per-invocation run record settings. +Opt-in local request journal settings. -When enabled, core appends a start line and an end line for each local invocation to `/.fabric///run-record.jsonl`. +When enabled, core appends lifecycle events and a terminal summary for each request to `/.fabric/requests//run.jsonl`. The journal can contain sensitive result data and error messages. ## Fields ### `enabled: bool` -Whether to write a run record for each invocation. Requires an artifact root, the `local` environment provider, and a `process` or `python` adapter. +Whether to write a journal for each request. Requires an artifact root, the `local` environment provider, and a `process` or `python` adapter. ## Trait Implementations diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx index 89f4efaf6..3ddb9e8a3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx @@ -37,7 +37,7 @@ Maximum number of harness turns within one invocation. ### `record: Option` -Opt-in per-invocation run record. Core-only; never sent to adapters. +Opt-in local request journal. Core-only; never sent to adapters. ### `extensions: BTreeMap` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-run-record-folder.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-run-record-folder.mdx index 82a53ea71..f2f0b95d7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-run-record-folder.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-run-record-folder.mdx @@ -1,7 +1,7 @@ --- title: "Function run_record_folder" sidebar-title: "run_record_folder" -description: "The recording root. This is known before startup and requires no inner IDs." +description: "The parent of request-scoped run journals. Requires recording to be enabled." position: 158 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -11,4 +11,4 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. 
RunPlan) -> Option<PathBuf>"}} />
-The recording root. This is known before startup and requires no inner IDs. +The parent of request-scoped run journals. Requires recording to be enabled. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 9d4de77bc..4528edbd7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -176,5 +176,5 @@ Core config and runtime contract for NeMo Fabric. ## Functions -- [run_record_folder](fn-run-record-folder.mdx): The recording root. This is known before startup and requires no inner IDs. +- [run_record_folder](fn-run-record-folder.mdx): The parent of request-scoped run journals. Requires recording to be enabled. - [version](fn-version.mdx): Returns the crate version compiled into this build. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index c787aacae..7306e8eab 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -191,7 +191,7 @@ Runtime resource snapshot schema.
-One bounded start/end invocation record line. +One typed event or terminal summary in a recorded request journal. ## Implementations diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 8798fb91e..3c7a5cf76 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1490,9 +1490,9 @@ backpressure. The consumer decides whether to wait, retry, scale out, or fail. - Review the examples in the GitHub repo [`examples/`](https://github.com/NVIDIA/NeMo-Fabric/tree/main/examples). -## Opt-in run recording +## Opt-in Run Recording -Set `runtime.record.enabled` and a host-side artifact root to save local runs: +Set `runtime.record.enabled` and a host-side artifact root to retain evidence from local runs: ```python from nemo_fabric import RecordConfig @@ -1507,61 +1507,44 @@ The CLI accepts the same settings: nemo-fabric run --preset scripted --input "Inspect the example" --record --artifacts ./runs ``` -Recording supports the local environment provider with Python or process adapters. -It is disabled when the record setting is omitted. Enabling it without an artifact -root or with an unsupported execution location fails validation. - -Each invocation writes bounded start/end lines to -`.fabric///run-record.jsonl`. The start line retains -identities and configured model names. It omits prompts, output, credentials, -exception messages and arbitrary adapter metadata. Allowlisted context identity -keys retain string or integer values; other context values are omitted. - -The sibling `run-result.json` contains the complete returned result and can -contain sensitive output. One-shot results are finalized after runtime shutdown; -persistent-runtime results describe the invocation before a later shutdown. -An invocation end record never establishes the outcome of a later shutdown. -Saved results are created exclusively and never replace an existing file or -symlink. Recording failures add diagnostics without retrying paid work or -replacing the execution outcome. No result file is promised when execution -raises before returning a result. - -The start/end record's elapsed time covers its invocation window, including start -record I/O and core preparation. It excludes end-record writes, full-result -persistence and one-shot runtime startup/shutdown. It is not pure model time or -the caller's task schedule interval. Record I/O is synchronous and best effort. -Each record line is limited to 64 KiB; oversized lines stop that record without -replacing the available execution result. - -### Inspect a saved run +Recording supports the local environment provider with Python or process adapters. It is disabled by default. Enabling it without an artifact root or at an unsupported execution location fails validation. Recording requires neither a benchmark framework nor NeMo Relay. + +### One Run Journal + +Each recorded request writes one journal at `/.fabric/requests//run.jsonl`. The recording ID identifies this journal; it is separate from the caller's request ID. Reusing a request ID does not combine separate requests into one file. + +JSONL means one JSON object per line. New events are appended during execution. Each record has a schema and version, recording and request identities, an append sequence, a full UTC timestamp, monotonic elapsed seconds, a source, a kind and a typed payload. Runtime and invocation IDs appear after those identities are known. Nested objects remain structured data inside each line. + +The final record retains the full returned result or a typed error when no result was returned. A one-shot request finishes after its owned runtime shutdown. An invocation on a persistent runtime finishes before any later shutdown; its final record does not establish that later outcome. Configuration resolution occurs before request recording begins. + +The journal can contain output and other sensitive result data. Treat it with the same access controls as the returned result. Native harness artifacts and task work products remain separate and can be referenced by the result. Native resource collection continues to publish its own sample and summary artifacts separately from invocation recording. + +Recording appends to an exclusively created private file and never replaces an existing file or symlink. Writes are synchronous and best effort. A storage failure stops further writes and produces a diagnostic without retrying model work or replacing the execution outcome. A process crash can leave a journal with no final summary. That establishes retained evidence, not completion or zero consumption. + +### Inspect a Saved Run + +Use the saved journal path with the inspector: ```bash -python -m nemo_fabric.reporting runs/.fabric///run-result.json -python -m nemo_fabric.reporting runs/.fabric///run-result.json --details -python -m nemo_fabric.reporting runs/.fabric///run-result.json --json +python -m nemo_fabric.reporting "runs/.fabric/requests//run.jsonl" +python -m nemo_fabric.reporting "runs/.fabric/requests//run.jsonl" --details +python -m nemo_fabric.reporting "runs/.fabric/requests//run.jsonl" --json ``` -Replace the placeholder IDs with the returned `run_result` artifact path. The -default overview separates execution, scoped elapsed time, complete token totals, -observed subtotals, cache breakdown and each cost amount. It names accounting -gaps and exposes diagnostic paths. Estimates retain their pricing basis and are -not invoices. An execution failure can retain complete usage; successful -execution does not establish task correctness or complete accounting. - -Use `--details` for the final resource snapshot and pricing evidence. Resource -snapshots describe the runtime process tree, not inference GPU usage. The CPU -counter is cumulative since host start; endpoint RSS is not a peak. Rounded -currency and human-readable units affect text only. JSON retains exact values, -and positive amounts never display as zero. - -The inspector accepts an explicit `run-record.jsonl` to show the invocation -boundary when a full result is unavailable. It retains complete lines before a -truncated tail and rejects duplicate or conflicting lines. The bounded record preserves validated normalized accounting, including complete -and observed counts, when present in the invocation result. Legacy numeric usage -remains unverified diagnostics. Missing evidence never becomes zero. - -A full result remains authoritative over its matching sibling record. The reader -checks identities, skips sibling symlinks, and never opens referenced logs or -telemetry payloads or uses them to fill missing counts. Each inspected file is -limited to 16 MiB; larger saved results remain unchanged and can be inspected -separately. Recording requires no benchmark framework, task labels or Relay. +Replace `` with the recorded directory name or use the returned `run_log` artifact path. The default view shows harness and model identities, execution status, start and finish times, and a two-column measurement table. Durations use seconds with three decimal places. The readable view shows the UTC date and timezone once when both times fall on the same date; stored timestamps always include the full date and UTC timezone. + +Token values describe the invocation's complete counts unless marked `recorded; total unknown`. `Unavailable` is not zero. Cache-read and cache-write tokens are included in input and must not be added again. Execution failure can retain complete usage; execution success does not establish task correctness or complete accounting. + +Provider-reported cost and catalog estimates remain separate. An estimate is not an invoice. A partial estimate is labeled `recorded; full estimate unknown`. Text formatting does not change stored values, and a positive cost never displays as zero. + +Use `--details` for a bounded execution history, relevant diagnostic messages, resource scope, pricing evidence and artifact references. Individual resource samples do not flood the timeline. CPU and memory describe the measured agent-side processes, not remote model-server resources. Endpoint memory is not a peak; a sampled peak is the highest collected reading and can miss shorter spikes. Availability depends on the evidence retained for that request. + +`--json` prints the inspector's structured summary to stdout. It does not create another file or copy the entire journal. The terminal record retains the returned result, while earlier records retain the execution history. The inspector does not open referenced artifacts or use logs and traces to fill missing counts. + +### Interrupted and Older Records + +The reader accepts the legacy saved-result and invocation-record formats. A valid final result remains authoritative; unknown final values are never filled from earlier events. An unfinished trailing line can be ignored. Malformed complete records, changed identities and duplicate terminal summaries are errors. + +The reader processes journals incrementally rather than loading the entire event stream. It limits retained history and rejects a terminal record larger than its supported 16 MiB record size with an explicit error. The saved evidence is not silently truncated or modified. Diagnostic records have a separate 64 KiB limit. Structured invocation evidence and terminal results are not discarded by that diagnostic limit. + +The inspector retains up to 1 MiB of display history, with a 64 KiB reserve for lifecycle milestones. It marks shortened fields and reports omitted entries. Unknown event kinds retain their envelope in this view without copying their payload. The saved journal and final result remain unchanged. diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index 44c6b23aa..37ffb8173 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -129,7 +129,7 @@ descriptor schema. ### Results, Artifacts, and Diagnostics -- `sdk/run-record-line`: bounded opt-in invocation start/end evidence, including validated normalized accounting when available. +- `sdk/run-record-line`: typed request journal events with UTC timestamps, monotonic elapsed seconds and a terminal result or no-result error. Diagnostic records have a separate size bound; terminal results retain their full output and available accounting. - `sdk/run-result`: normalized consumer-facing invocation result. - `sdk/artifact-manifest`: normalized artifact references. - `sdk/error-info`: structured runtime or adapter error metadata. diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index 8c43c2fb4..1c9fa06bb 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -574,10 +574,10 @@ }, "RecordConfig": { "additionalProperties": false, - "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "description": "Opt-in local request journal settings.\n\nWhen enabled, core appends lifecycle events and a terminal summary for each\nrequest to `/.fabric/requests//run.jsonl`.\nThe journal can contain sensitive result data and error messages.", "properties": { "enabled": { - "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", + "description": "Whether to write a journal for each request. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", "type": "boolean" } }, @@ -1366,7 +1366,7 @@ "type": "null" } ], - "description": "Opt-in per-invocation run record. Core-only; never sent to adapters." + "description": "Opt-in local request journal. Core-only; never sent to adapters." }, "timeout_seconds": { "description": "Maximum duration of one invocation in seconds.", diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index 7bc566c42..4c3950384 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -2134,10 +2134,10 @@ }, "RecordConfig": { "additionalProperties": false, - "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "description": "Opt-in local request journal settings.\n\nWhen enabled, core appends lifecycle events and a terminal summary for each\nrequest to `/.fabric/requests//run.jsonl`.\nThe journal can contain sensitive result data and error messages.", "properties": { "enabled": { - "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", + "description": "Whether to write a journal for each request. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", "type": "boolean" } }, @@ -3001,7 +3001,7 @@ "type": "null" } ], - "description": "Opt-in per-invocation run record. Core-only; never sent to adapters." + "description": "Opt-in local request journal. Core-only; never sent to adapters." }, "timeout_seconds": { "description": "Maximum duration of one invocation in seconds.", diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json index c35c9a6c9..070bb3eb5 100644 --- a/schemas/sdk/run-record-line.schema.json +++ b/schemas/sdk/run-record-line.schema.json @@ -1,5 +1,30 @@ { "$defs": { + "AdapterKind": { + "description": "Adapter implementation kind.", + "oneOf": [ + { + "const": "process", + "description": "Launch and supervise a persistent adapter process.", + "type": "string" + }, + { + "const": "http", + "description": "Connect to a service or HTTP-backed harness.", + "type": "string" + }, + { + "const": "python", + "description": "Launch and supervise a persistent Python adapter host.", + "type": "string" + }, + { + "const": "native_plugin", + "description": "Delegate to a harness-native plugin package.", + "type": "string" + } + ] + }, "AppliedPricing": { "additionalProperties": false, "description": "Persisted entry and selected rates, so saved prices remain reproducible.", @@ -39,235 +64,325 @@ ], "type": "object" }, - "ContextValue": { - "anyOf": [ - { - "description": "An allowlisted key's string value, cut to 256 characters.", - "type": "string" - }, - { - "description": "An allowlisted key's integer value.", - "format": "int64", - "type": "integer" - }, - { - "$ref": "#/$defs/NotRecorded", - "description": "Any other key or value: the key name only." - } - ], - "description": "A request context value as the start line keeps it." - }, - "CostAmount": { - "description": "Complete and observed amounts in integer nano-USD.", + "ArtifactManifest": { + "additionalProperties": false, + "description": "Manifest of run artifacts.", "properties": { - "amount_nano_usd": { - "description": "Complete amount for this invocation/model scope.", - "format": "uint64", - "maximum": 18446744073709551615, - "minimum": 0, - "type": [ - "integer", - "null" - ] + "artifacts": { + "description": "Artifact entries.", + "items": { + "$ref": "#/$defs/ArtifactRef" + }, + "type": "array" }, - "observed_nano_usd": { - "description": "Valid observed contributions, not a guaranteed bound or invoice.", - "format": "uint64", - "maximum": 18446744073709551615, - "minimum": 0, + "root": { + "description": "Artifact root directory.", "type": [ - "integer", + "string", "null" ] } }, "type": "object" }, - "EndArtifact": { - "description": "One result artifact on the end line.", + "ArtifactRef": { + "additionalProperties": false, + "description": "Reference to one artifact.", "properties": { "kind": { "description": "Artifact kind.", "type": "string" }, "media_type": { - "description": "Media type.", + "description": "Optional media type.", "type": [ "string", "null" ] }, + "metadata": { + "additionalProperties": true, + "description": "Artifact-specific metadata preserved across the Rust and Python SDK boundary.", + "type": "object" + }, "name": { "description": "Logical artifact name.", "type": "string" }, "path": { - "description": "Path relative to the artifact root; absent for a path outside it.", - "type": [ - "string", - "null" - ] + "description": "Artifact path.", + "type": "string" } }, "required": [ "name", - "kind" + "kind", + "path" ], "type": "object" }, - "EndError": { - "description": "The end line's error.", + "CgroupObservation": { + "description": "A cgroup reading, or why the correctly targeted scope was unavailable.", + "oneOf": [ + { + "description": "All cgroup groups were explicitly excluded from collection.", + "properties": { + "status": { + "const": "not_collected", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "The adapter host's cgroup was located and read.", + "properties": { + "snapshot": { + "$ref": "#/$defs/CgroupSnapshot", + "description": "Snapshot of the resolved cgroup identity." + }, + "status": { + "const": "available", + "type": "string" + } + }, + "required": [ + "status", + "snapshot" + ], + "type": "object" + }, + { + "description": "No cgroup v2 reader is supported for this platform/provider.", + "properties": { + "reason": { + "description": "Missing platform or provider capability.", + "type": "string" + }, + "status": { + "const": "unsupported", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + }, + { + "description": "The target moved, exited, or could not be resolved safely.", + "properties": { + "reason": { + "description": "Safe reason targeting or collection failed.", + "type": "string" + }, + "status": { + "const": "unavailable", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + } + ] + }, + "CgroupSnapshot": { + "description": "One observation of the cgroup containing the adapter host. Its counters\ninclude the cgroup's descendants and may include other workloads. Limits\ncover only visible ancestors; ancestors hidden by namespaces are unknown.", "properties": { - "code": { - "description": "Error code matching `^[a-z0-9_.:-]{1,128}$`, else `unrecognized`.", - "pattern": "^[a-z0-9_.:-]{1,128}$", + "cpu_throttled_microseconds": { + "$ref": "#/$defs/Measurement2", + "description": "Cumulative CPU throttled time, in microseconds." + }, + "cpu_usage_microseconds": { + "$ref": "#/$defs/Measurement2", + "description": "Cumulative cgroup CPU time, in microseconds." + }, + "memory_current_bytes": { + "$ref": "#/$defs/Measurement2", + "description": "Hierarchically accounted memory, distinct from process RSS, in bytes." + }, + "memory_peak_bytes": { + "$ref": "#/$defs/Measurement2", + "description": "Provider high-water mark in bytes, since cgroup creation or the last\nexternal reset. Never an invocation-specific peak." + }, + "observed_at": { + "description": "UTC observation timestamp.", + "format": "date-time", "type": "string" }, - "retryable": { - "description": "Whether a consumer-level retry is considered safe.", - "type": "boolean" + "oom_kills": { + "$ref": "#/$defs/Measurement2", + "description": "Cumulative OOM kills in this cgroup hierarchy." }, - "stage": { - "$ref": "#/$defs/ErrorStage", - "description": "Lifecycle stage where the failure surfaced." + "pressure_stall_microseconds": { + "additionalProperties": { + "$ref": "#/$defs/Measurement2" + }, + "description": "Cumulative pressure-stall microseconds: cpu_some, cpu_full,\nmemory_some, memory_full, io_some, and io_full.", + "type": "object" + }, + "scope": { + "description": "Fixed scope identifier, `adapter_host_cgroup`.", + "type": "string" + }, + "source": { + "description": "Fixed source identifier, `linux_cgroup_v2`.", + "type": "string" + }, + "swap_current_bytes": { + "$ref": "#/$defs/Measurement2", + "description": "Current swap usage in bytes." + }, + "target": { + "description": "Device/inode identity. A different identity invalidates counter deltas.", + "type": "string" + }, + "visible_cpu_limit_cores": { + "$ref": "#/$defs/Measurement", + "description": "Minimum of effective cpuset size and visible ancestor CPU quotas.\nThis is a visible upper bound, not reserved or exclusively usable CPU." + }, + "visible_memory_limit_bytes": { + "$ref": "#/$defs/Measurement2", + "description": "Minimum memory.max among visible ancestors, in bytes." } }, "required": [ - "code", - "stage", - "retryable" + "source", + "scope", + "target", + "observed_at", + "visible_cpu_limit_cores", + "visible_memory_limit_bytes", + "memory_current_bytes", + "memory_peak_bytes", + "swap_current_bytes", + "cpu_usage_microseconds", + "cpu_throttled_microseconds", + "oom_kills", + "pressure_stall_microseconds" ], "type": "object" }, - "EndLine": { - "description": "Fields of the end line.", + "CgroupStats": { + "description": "Native cgroup observations and counter deltas over an invocation. Values\nidentify the adapter host's cgroup, without claiming container exclusivity.", "properties": { - "artifacts": { - "description": "Result artifacts, with paths relative to the artifact root.", - "items": { - "$ref": "#/$defs/EndArtifact" - }, - "type": "array" + "after": { + "$ref": "#/$defs/CgroupObservation", + "description": "Reading immediately after invocation." }, - "elapsed_ms": { - "description": "Monotonic time from the start line to the end line, in milliseconds.", - "format": "uint64", - "minimum": 0, - "type": "integer" + "before": { + "$ref": "#/$defs/CgroupObservation", + "description": "Reading immediately before invocation." }, - "error": { - "anyOf": [ - { - "$ref": "#/$defs/EndError" - }, - { - "type": "null" - } - ], - "description": "Error code, stage and retryable flag, without the message." + "cpu_throttled_microseconds": { + "$ref": "#/$defs/Measurement2", + "description": "CPU throttled time delta in microseconds, or an explicit unknown reason." }, - "model_usage": { - "anyOf": [ - { - "$ref": "#/$defs/ModelUsage" - }, - { - "type": "null" - } - ], - "description": "Validated normalized accounting frozen at the invocation boundary." + "cpu_usage_microseconds": { + "$ref": "#/$defs/Measurement2", + "description": "CPU usage delta in microseconds, or an explicit unknown reason." }, - "outcome_scope": { - "const": "invocation", - "description": "This line observes the invocation, before one-shot runtime shutdown.", + "oom_kills": { + "$ref": "#/$defs/Measurement2", + "description": "OOM-kill counter delta, or an explicit unknown reason." + } + }, + "required": [ + "before", + "after", + "cpu_usage_microseconds", + "cpu_throttled_microseconds", + "oom_kills" + ], + "type": "object" + }, + "ContextValue": { + "anyOf": [ + { "type": "string" }, - "result": { - "description": "Expected full-result file next to this record; absence means not saved.", - "type": [ - "string", - "null" - ] - }, - "status": { - "$ref": "#/$defs/RunStatus", - "description": "Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`." + { + "format": "int64", + "type": "integer" }, - "usage": { - "anyOf": [ - { - "$ref": "#/$defs/EndUsage" - }, - { - "type": "null" + { + "properties": { + "recorded": { + "type": "boolean" } + }, + "required": [ + "recorded" ], - "description": "Numeric usage counters, without adapter metadata." + "type": "object" } - }, - "required": [ - "outcome_scope", - "status", - "elapsed_ms" ], - "type": "object" + "description": "Unknown context values are intentionally not copied into request evidence." }, - "EndUsage": { - "description": "The end line's usage: the numeric `RunUsage` fields.", + "CostAmount": { + "description": "Complete and observed amounts in integer nano-USD.", "properties": { - "cached_input_tokens": { - "description": "Cached input tokens.", + "amount_nano_usd": { + "description": "Complete amount for this invocation/model scope.", "format": "uint64", + "maximum": 18446744073709551615, "minimum": 0, "type": [ "integer", "null" ] }, - "cost_usd": { - "description": "Reported cost in US dollars.", - "format": "double", - "minimum": 0.0, - "type": [ - "number", - "null" - ] - }, - "input_tokens": { - "description": "Input tokens.", + "observed_nano_usd": { + "description": "Valid observed contributions, not a guaranteed bound or invoice.", "format": "uint64", + "maximum": 18446744073709551615, "minimum": 0, "type": [ "integer", "null" ] + } + }, + "type": "object" + }, + "ErrorInfo": { + "description": "Normalized error metadata.", + "properties": { + "code": { + "description": "Stable error code.", + "type": "string" }, - "input_tokens_include_cache": { - "description": "Whether `input_tokens` includes `cached_input_tokens`; absent means unknown.", - "type": [ - "boolean", - "null" - ] + "message": { + "description": "Human-readable error message.", + "type": "string" }, - "output_tokens": { - "description": "Output tokens.", - "format": "uint64", - "minimum": 0, - "type": [ - "integer", - "null" - ] + "metadata": { + "additionalProperties": true, + "description": "Adapter or runtime metadata useful for diagnostics.", + "type": "object" }, - "total_tokens": { - "description": "Total tokens reported by the provider.", - "format": "uint64", - "minimum": 0, - "type": [ - "integer", - "null" - ] + "retryable": { + "default": false, + "description": "Whether NeMo Fabric considers this failure safe for a consumer-level retry.", + "type": "boolean" + }, + "stage": { + "$ref": "#/$defs/ErrorStage", + "description": "NeMo Fabric lifecycle stage where the failure surfaced." } }, + "required": [ + "stage", + "code", + "message" + ], "type": "object" }, "ErrorStage": { @@ -315,6 +430,41 @@ } ] }, + "FabricEvent": { + "description": "NeMo Fabric lifecycle or progress event.", + "properties": { + "event_id": { + "description": "Event id.", + "type": "string" + }, + "kind": { + "description": "Event kind.", + "type": "string" + }, + "message": { + "description": "Event message.", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "description": "Event metadata.", + "type": "object" + }, + "timestamp_millis": { + "description": "Unix timestamp in milliseconds.", + "format": "uint128", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "event_id", + "timestamp_millis", + "kind", + "message" + ], + "type": "object" + }, "GapReason": { "description": "Categories that affect whether a consumer can treat a value as complete.", "oneOf": [ @@ -335,19 +485,379 @@ } ] }, - "ModelName": { - "description": "One configured model.", + "GaugeSummary": { + "description": "A sampled gauge over the invocation window. Min/max cover retained valid\nsamples. Mean holds the previous value only across adjacent valid samples\nno more than twice the requested cadence apart. Missing intervals never\ncontribute zero. No sampled maximum claims a true process high-water mark.", "properties": { - "model": { - "description": "Provider model identifier.", - "type": "string" + "covered_fraction": { + "description": "Covered seconds divided by invocation duration.", + "format": "double", + "type": "number" }, - "provider": { - "description": "Model provider name.", - "type": "string" - } - }, - "required": [ + "covered_seconds": { + "description": "Seconds covered by valid adjacent readings.", + "format": "double", + "type": "number" + }, + "final_value": { + "description": "Final endpoint, absent if its reading failed.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "min": { + "description": "Minimum retained value.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "sampled_peak": { + "description": "Maximum retained value, not an unsampled high-water mark.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "time_weighted_mean": { + "description": "Hold-last-value integral divided by covered seconds, or unknown.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "unit": { + "description": "`byte` for RSS or `process` for process count.", + "type": "string" + }, + "valid_samples": { + "description": "Number of valid retained readings for this metric.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "unit", + "valid_samples", + "covered_seconds", + "covered_fraction" + ], + "type": "object" + }, + "InvocationFinished": { + "description": "Evidence at invocation return is independent from later owned shutdown.", + "properties": { + "adapter_exchange_elapsed_seconds": { + "format": "double", + "type": [ + "number", + "null" + ] + }, + "elapsed_seconds": { + "format": "double", + "type": "number" + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/ErrorInfo" + }, + { + "type": "null" + } + ] + }, + "model_usage": { + "anyOf": [ + { + "$ref": "#/$defs/ModelUsage" + }, + { + "type": "null" + } + ] + }, + "status": { + "$ref": "#/$defs/RunStatus" + }, + "system_stats": { + "anyOf": [ + { + "$ref": "#/$defs/InvocationSystemStats" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "status", + "elapsed_seconds" + ], + "type": "object" + }, + "InvocationSystemStats": { + "description": "System measurements around one invocation, collected inside the executing\nrunner. Process-tree observations cover its adapter host and descendants,\nexcluding the orchestrator and remote model/service processes. The separate\ncgroup observations may include other workloads in the same cgroup.\n\nEndpoints are instantaneous readings, not peaks or time averages. CPU time\nis cumulative since runtime startup; its difference is only an estimate\nbecause process reaping and reparenting can change which work is visible.", + "properties": { + "after": { + "$ref": "#/$defs/ResourceObservation", + "description": "Observation immediately after the adapter exchange, before another\ninvocation can acquire this runtime's host lock." + }, + "before": { + "$ref": "#/$defs/ResourceObservation", + "description": "Observation immediately before the adapter exchange." + }, + "cgroup": { + "anyOf": [ + { + "$ref": "#/$defs/CgroupStats" + }, + { + "type": "null" + } + ], + "description": "Before/after readings for the adapter host's cgroup. This scope may\ncontain other workloads and overlaps the process tree; never add them." + }, + "collected_groups": { + "default": [ + "process", + "cgroup.cpu", + "cgroup.memory", + "cgroup.pressure" + ], + "description": "Requested groups, including selected groups unavailable on this platform.", + "items": { + "$ref": "#/$defs/SystemStatsMetricGroup" + }, + "type": "array" + }, + "cpu_time_seconds_estimate": { + "description": "Nonnegative CPU counter difference, in seconds. Absent if an endpoint\nis missing, the runtime identity differs, or its CPU counter decreased.\nIt can undercount work whose processes leave the tree between reads.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "elapsed_seconds": { + "description": "Monotonic elapsed seconds for the adapter exchange. Excludes runtime\nstartup, endpoint collection, artifact preparation, and teardown.\nOptional sampling runs concurrently; its effect is not subtracted.", + "format": "double", + "minimum": 0.0, + "type": "number" + }, + "sampled": { + "anyOf": [ + { + "$ref": "#/$defs/SampleSummary" + }, + { + "type": "null" + } + ], + "description": "Bounded sample reductions and collection quality when sampling is enabled." + } + }, + "required": [ + "elapsed_seconds", + "before", + "after" + ], + "type": "object" + }, + "Measurement": { + "description": "A measured value or its availability. Unlimited is distinct from unknown\nand is used only for limits. No missing measurement is replaced with zero.", + "oneOf": [ + { + "description": "Its metric group was explicitly excluded from collection.", + "properties": { + "status": { + "const": "not_collected", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "A value was read successfully.", + "properties": { + "status": { + "const": "available", + "type": "string" + }, + "value": { + "description": "Observed numeric value, with units defined by its containing field.", + "format": "double", + "type": "number" + } + }, + "required": [ + "status", + "value" + ], + "type": "object" + }, + { + "description": "No limit is configured in the visible hierarchy.", + "properties": { + "status": { + "const": "unlimited", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "The source could not be read or its counter changed epoch.", + "properties": { + "reason": { + "description": "Safe reason the reading is unknown.", + "type": "string" + }, + "status": { + "const": "unavailable", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + }, + { + "description": "The operating system or controller does not expose this measurement.", + "properties": { + "reason": { + "description": "Missing platform or controller capability.", + "type": "string" + }, + "status": { + "const": "unsupported", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + } + ] + }, + "Measurement2": { + "description": "A measured value or its availability. Unlimited is distinct from unknown\nand is used only for limits. No missing measurement is replaced with zero.", + "oneOf": [ + { + "description": "Its metric group was explicitly excluded from collection.", + "properties": { + "status": { + "const": "not_collected", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "A value was read successfully.", + "properties": { + "status": { + "const": "available", + "type": "string" + }, + "value": { + "description": "Observed numeric value, with units defined by its containing field.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "status", + "value" + ], + "type": "object" + }, + { + "description": "No limit is configured in the visible hierarchy.", + "properties": { + "status": { + "const": "unlimited", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "The source could not be read or its counter changed epoch.", + "properties": { + "reason": { + "description": "Safe reason the reading is unknown.", + "type": "string" + }, + "status": { + "const": "unavailable", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + }, + { + "description": "The operating system or controller does not expose this measurement.", + "properties": { + "reason": { + "description": "Missing platform or controller capability.", + "type": "string" + }, + "status": { + "const": "unsupported", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + } + ] + }, + "ModelName": { + "properties": { + "model": { + "type": "string" + }, + "provider": { + "type": "string" + } + }, + "required": [ "provider", "model" ], @@ -441,19 +951,13 @@ ], "type": "object" }, - "NotRecorded": { - "description": "Marks a key whose value the record leaves out.", - "properties": { - "recorded": { - "const": false, - "description": "Always `false`.", - "type": "boolean" - } - }, - "required": [ - "recorded" + "OutcomeScope": { + "description": "The operation whose return boundary closes this journal.", + "enum": [ + "invocation", + "one_shot_lifecycle" ], - "type": "object" + "type": "string" }, "PerModelUsage": { "description": "Normalized usage for one literal model/provider identity.", @@ -555,176 +1059,701 @@ "description": "`catalog`; provider-reported cost has a different basis.", "type": "string" }, - "bundled_catalog_sha256": { - "description": "Exact bundled catalog bytes.", - "type": "string" + "bundled_catalog_sha256": { + "description": "Exact bundled catalog bytes.", + "type": "string" + }, + "currency": { + "description": "`USD` in this implementation.", + "type": "string" + }, + "normalization_version": { + "description": "Version of normalization and monetary rounding.", + "type": "string" + }, + "user_catalog_sha256": { + "description": "Exact override bytes, when supplied.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "basis", + "currency", + "normalization_version", + "assumption_policy", + "bundled_catalog_sha256" + ], + "type": "object" + }, + "RateSnapshot": { + "additionalProperties": false, + "description": "Integer rates selected for a model, in nano-USD per million tokens.", + "properties": { + "cache_read": { + "description": "Cache read rate; absent is not zero.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cache_write": { + "description": "Cache write rate; absent is not zero.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input": { + "description": "Uncached input rate.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, + "output": { + "description": "Reasoning-inclusive output rate.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "input", + "output" + ], + "type": "object" + }, + "RecordSchema": { + "enum": [ + "nemo.fabric.run" + ], + "type": "string" + }, + "RecordSource": { + "enum": [ + "fabric" + ], + "type": "string" + }, + "RecordVersion": { + "enum": [ + "v1alpha1" + ], + "type": "string" + }, + "RequestStarted": { + "properties": { + "agent_name": { + "type": "string" + }, + "context": { + "additionalProperties": { + "$ref": "#/$defs/ContextValue" + }, + "type": "object" + }, + "harness": { + "type": "string" + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelName" + }, + "type": "object" + }, + "outcome_scope": { + "$ref": "#/$defs/OutcomeScope" + }, + "override_keys": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "outcome_scope", + "agent_name", + "harness", + "models", + "context", + "override_keys" + ], + "type": "object" + }, + "ResourceObservation": { + "description": "One attempt to observe the adapter host process tree. Missing readings are\nexplicit; an unavailable observation never stands for zero resource usage.", + "oneOf": [ + { + "description": "The process group was explicitly excluded from collection.", + "properties": { + "status": { + "const": "not_collected", + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + { + "description": "The process tree was read successfully.", + "properties": { + "snapshot": { + "$ref": "#/$defs/RuntimeResourceSnapshot", + "description": "Timestamped cumulative counters and instantaneous gauges." + }, + "status": { + "const": "available", + "type": "string" + } + }, + "required": [ + "status", + "snapshot" + ], + "type": "object" + }, + { + "description": "The reader supports this runtime, but could not read its process tree.", + "properties": { + "reason": { + "description": "Why this observation could not be collected.", + "type": "string" + }, + "status": { + "const": "unavailable", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + }, + { + "description": "The runtime's provider or operating system has no reader.", + "properties": { + "reason": { + "description": "Which collection capability is missing.", + "type": "string" + }, + "status": { + "const": "unsupported", + "type": "string" + } + }, + "required": [ + "status", + "reason" + ], + "type": "object" + } + ] + }, + "RunResult": { + "description": "Result from a NeMo Fabric-managed harness invocation.", + "properties": { + "adapter_id": { + "description": "Adapter implementation id when an adapter descriptor was resolved.", + "type": [ + "string", + "null" + ] + }, + "adapter_kind": { + "$ref": "#/$defs/AdapterKind", + "description": "Adapter used for this run." + }, + "agent_name": { + "description": "Stable agent name.", + "type": "string" + }, + "artifacts": { + "$ref": "#/$defs/ArtifactManifest", + "default": {}, + "description": "Artifacts produced or collected by this run." + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/ErrorInfo" + }, + { + "type": "null" + } + ], + "description": "Error metadata when applicable." + }, + "events": { + "description": "NeMo Fabric lifecycle/progress events emitted during the run.", + "items": { + "$ref": "#/$defs/FabricEvent" + }, + "type": "array" + }, + "harness": { + "description": "Stable machine-readable harness identifier used for this run.", + "type": "string" + }, + "invocation_id": { + "description": "Invocation handle id.", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "description": "Adapter-specific metadata.", + "type": "object" + }, + "model_usage": { + "anyOf": [ + { + "$ref": "#/$defs/ModelUsage" + }, + { + "type": "null" + } + ], + "description": "Complete and observed usage normalized from supported accounting evidence." + }, + "output": { + "default": null, + "description": "Primary output." + }, + "request_id": { + "description": "Request id.", + "type": "string" + }, + "resource_snapshot": { + "anyOf": [ + { + "$ref": "#/$defs/RuntimeResourceSnapshot" + }, + { + "type": "null" + } + ], + "description": "Resource snapshot of the runtime, read once after the adapter returned this\nresult. Absent when the runtime's resources could not be observed." + }, + "runtime_id": { + "description": "Runtime handle id.", + "type": "string" + }, + "status": { + "$ref": "#/$defs/RunStatus", + "description": "Runtime status." + }, + "system_stats": { + "anyOf": [ + { + "$ref": "#/$defs/InvocationSystemStats" + }, + { + "type": "null" + } + ], + "description": "Invocation duration and endpoint observations from the executing runner.\nAbsent in older results and when an execution backend does not collect them." + }, + "telemetry": { + "description": "Ordered telemetry references when available.", + "items": { + "$ref": "#/$defs/TelemetryRef" + }, + "type": "array" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/$defs/RunUsage" + }, + { + "type": "null" + } + ], + "description": "Normalized invocation usage when reported by the adapter target." + } + }, + "required": [ + "agent_name", + "harness", + "adapter_kind", + "runtime_id", + "invocation_id", + "request_id", + "status" + ], + "type": "object" + }, + "RunStatus": { + "description": "Runtime completion status.", + "oneOf": [ + { + "const": "succeeded", + "description": "The harness completed successfully.", + "type": "string" + }, + { + "const": "failed", + "description": "The harness completed with a non-zero status.", + "type": "string" + }, + { + "const": "cancelled", + "description": "The invocation or runtime was cancelled.", + "type": "string" + } + ] + }, + "RunUsage": { + "additionalProperties": false, + "description": "Normalized invocation usage exposed to NeMo Fabric consumers.", + "properties": { + "cached_input_tokens": { + "description": "Cached input tokens consumed by the invocation, when reported.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Invocation cost in US dollars when reported by the provider.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Input tokens consumed by the invocation.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether input_tokens already includes cached_input_tokens; absent means unknown.", + "type": [ + "boolean", + "null" + ] + }, + "metadata": { + "additionalProperties": true, + "description": "Adapter-owned usage metadata.", + "type": "object" + }, + "output_tokens": { + "description": "Output tokens produced by the invocation.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Total tokens reported by the provider.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "RuntimeResourceSnapshot": { + "description": "Timestamped resource usage of one runtime's local adapter host process tree.\n\nMemory, process count, and CPU time cover the adapter host process and its\nlive descendants. For a runtime started on a service, the shared service\nprocess is not included. Values NeMo Fabric cannot read are returned as an\nerror, never as zero.", + "properties": { + "cpu_time_seconds": { + "description": "CPU time consumed by the tree since the adapter host started, in seconds,\nincluding exited descendants. It can drop when a descendant leaves the\ntree.", + "format": "double", + "minimum": 0.0, + "type": "number" + }, + "observed_at": { + "description": "When NeMo Fabric read these values, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + }, + "process_count": { + "description": "Live processes in the tree, including the adapter host. Exited processes\nthat are not yet reaped are not counted.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "resident_memory_bytes": { + "description": "Sum of the resident set sizes of the live processes, in bytes. Pages\nshared between processes count once per process.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "runtime_id": { + "description": "Runtime this snapshot describes.", + "type": "string" + }, + "started_at": { + "description": "When NeMo Fabric started the runtime's adapter host, as an RFC 3339 UTC timestamp.", + "format": "date-time", + "type": "string" + } + }, + "required": [ + "runtime_id", + "started_at", + "observed_at", + "resident_memory_bytes", + "process_count", + "cpu_time_seconds" + ], + "type": "object" + }, + "SampleSummary": { + "description": "Resource series retention, reductions, and collection quality. Resource\nsamples remain in the referenced JSONL artifact for interval selection.", + "properties": { + "cgroup_target_continuous": { + "description": "Whether retained readings kept one cgroup identity with nondecreasing\ncounters and no gaps over twice cadence. This does not prove exclusivity.", + "type": "boolean" + }, + "collection_seconds": { + "description": "Total time spent collecting retained observations, including endpoints.", + "format": "double", + "type": "number" + }, + "cpu_percent_one_core": { + "$ref": "#/$defs/Measurement", + "description": "100% means one fully occupied CPU core; this is not normalized by quota." + }, + "cpu_time_seconds_estimate": { + "$ref": "#/$defs/Measurement", + "description": "Estimated CPU counter delta, invalidated by observed resets, missing\nendpoints, target changes, or collection gaps larger than twice cadence." + }, + "diagnostics": { + "description": "Safe collection/persistence diagnostics. They never change agent status.", + "items": { + "type": "string" + }, + "type": "array" + }, + "dropped_samples": { + "description": "Samples skipped because collection fell behind or retention was full.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "largest_gap_seconds": { + "description": "Maximum observed interval between retained readings.", + "format": "double", + "type": "number" + }, + "max_samples": { + "description": "Retention bound, including endpoints.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "process_count": { + "anyOf": [ + { + "$ref": "#/$defs/GaugeSummary" + }, + { + "type": "null" + } + ], + "description": "Adapter-host tree live-process-count reductions." + }, + "resident_memory_bytes": { + "anyOf": [ + { + "$ref": "#/$defs/GaugeSummary" + }, + { + "type": "null" + } + ], + "description": "Adapter-host tree resident-memory reductions." + }, + "sample_count": { + "description": "Retained samples, including unavailable readings and endpoints.", + "format": "uint32", + "minimum": 0, + "type": "integer" }, - "currency": { - "description": "`USD` in this implementation.", - "type": "string" + "samples": { + "anyOf": [ + { + "$ref": "#/$defs/ArtifactRef" + }, + { + "type": "null" + } + ], + "description": "Series artifact, absent when its file could not be created or fully written." }, - "normalization_version": { - "description": "Version of normalization and monetary rounding.", - "type": "string" + "target_interval_ms": { + "description": "Requested cadence; actual timing is retained in sample offsets.", + "format": "uint32", + "minimum": 0, + "type": "integer" }, - "user_catalog_sha256": { - "description": "Exact override bytes, when supplied.", - "type": [ - "string", - "null" - ] + "truncated": { + "description": "Whether collection or retention prevented a complete series.", + "type": "boolean" } }, "required": [ - "basis", - "currency", - "normalization_version", - "assumption_policy", - "bundled_catalog_sha256" + "target_interval_ms", + "max_samples", + "sample_count", + "dropped_samples", + "largest_gap_seconds", + "collection_seconds", + "truncated", + "diagnostics", + "cgroup_target_continuous", + "cpu_time_seconds_estimate", + "cpu_percent_one_core" ], "type": "object" }, - "RateSnapshot": { - "additionalProperties": false, - "description": "Integer rates selected for a model, in nano-USD per million tokens.", + "Summary": { + "description": "Authoritative final evidence for this recorded operation.", "properties": { - "cache_read": { - "description": "Cache read rate; absent is not zero.", - "format": "uint64", - "maximum": 18446744073709551615, - "minimum": 0, + "adapter_exchange_elapsed_seconds": { + "format": "double", "type": [ - "integer", + "number", "null" ] }, - "cache_write": { - "description": "Cache write rate; absent is not zero.", - "format": "uint64", - "maximum": 18446744073709551615, - "minimum": 0, - "type": [ - "integer", - "null" + "elapsed_seconds": { + "format": "double", + "type": "number" + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/ErrorInfo" + }, + { + "type": "null" + } ] }, - "input": { - "description": "Uncached input rate.", - "format": "uint64", - "maximum": 18446744073709551615, - "minimum": 0, - "type": "integer" + "outcome_scope": { + "$ref": "#/$defs/OutcomeScope" }, - "output": { - "description": "Reasoning-inclusive output rate.", - "format": "uint64", - "maximum": 18446744073709551615, - "minimum": 0, - "type": "integer" + "request_finished_at": { + "type": "string" + }, + "request_started_at": { + "type": "string" + }, + "result": { + "anyOf": [ + { + "$ref": "#/$defs/RunResult" + }, + { + "type": "null" + } + ] } }, "required": [ - "input", - "output" + "outcome_scope", + "request_started_at", + "request_finished_at", + "elapsed_seconds" ], "type": "object" }, - "RecordSchema": { - "description": "The record's schema name.", - "oneOf": [ - { - "const": "nemo.fabric.run_record", - "description": "`nemo.fabric.run_record`.", - "type": "string" - } - ] - }, - "RecordVersion": { - "description": "The record's line format version.", + "SystemStatsMetricGroup": { + "description": "Resource groups supported by the native invocation collector.", "oneOf": [ { - "const": "v1alpha1", - "description": "`v1alpha1`.", + "const": "process", + "description": "Adapter host process-tree CPU, resident memory, and process count.", "type": "string" - } - ] - }, - "RunStatus": { - "description": "Runtime completion status.", - "oneOf": [ + }, { - "const": "succeeded", - "description": "The harness completed successfully.", + "const": "cgroup.cpu", + "description": "Cgroup CPU capacity, usage, and throttling.", "type": "string" }, { - "const": "failed", - "description": "The harness completed with a non-zero status.", + "const": "cgroup.memory", + "description": "Cgroup memory capacity, usage, swap, and OOM kills.", "type": "string" }, { - "const": "cancelled", - "description": "The invocation or runtime was cancelled.", + "const": "cgroup.pressure", + "description": "Cgroup CPU, memory, and I/O pressure counters.", "type": "string" } ] }, - "StartLine": { - "description": "Fields of the start line.", + "TelemetryRef": { + "description": "Reference to telemetry emitted by Relay or another configured telemetry path.", "properties": { - "adapter_id": { - "description": "Adapter implementation id.", + "kind": { + "description": "Reference category in the public SDK representation.", "type": [ "string", "null" ] }, - "agent_name": { - "description": "Config name (`metadata.name`).", - "type": "string" - }, - "context": { - "additionalProperties": { - "$ref": "#/$defs/ContextValue" - }, - "description": "`RunRequest.context`. Allowlisted identity and Harbor link keys keep a\nstring or integer value; every other key is listed by name only.", + "metadata": { + "additionalProperties": true, + "description": "Telemetry metadata.", "type": "object" }, - "fabric_version": { - "description": "NeMo Fabric version that wrote the record.", - "type": "string" - }, - "harness": { - "description": "Harness identifier.", - "type": "string" + "provider": { + "description": "Provider in the public SDK reference representation.", + "type": [ + "string", + "null" + ] }, - "models": { - "additionalProperties": { - "$ref": "#/$defs/ModelName" - }, - "description": "Configured models by role: provider and model name only.", - "type": "object" + "relay_enabled": { + "default": false, + "description": "Whether Relay was enabled for this run; retained for native compatibility.", + "type": "boolean" }, - "override_keys": { - "description": "Top-level key names of `RunRequest.overrides`, sorted. Values are never\nrecorded.", - "items": { - "type": "string" - }, - "type": "array" + "trace_id": { + "description": "Provider trace identity.", + "type": [ + "string", + "null" + ] }, - "timeout_ms": { - "description": "Host invocation timeout that applied, in milliseconds.", - "format": "uint64", - "minimum": 0, - "type": "integer" + "uri": { + "description": "Location of the persisted telemetry.", + "type": [ + "string", + "null" + ] } }, - "required": [ - "agent_name", - "harness", - "fabric_version", - "timeout_ms" - ], "type": "object" }, "TokenCost": { @@ -892,71 +1921,256 @@ } }, "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "One line of a run record.", + "description": "One independently parseable line of a recorded request.", "oneOf": [ { - "$ref": "#/$defs/StartLine", - "description": "Written once, before core sends the request to the adapter host.", "properties": { + "data": { + "$ref": "#/$defs/RequestStarted" + }, "kind": { - "const": "start", + "const": "request_started", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "properties": { + "runtime_id": { + "type": "string" + } + }, + "required": [ + "runtime_id" + ], + "type": "object" + }, + "kind": { + "const": "runtime_ready", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "properties": { + "timeout_seconds": { + "format": "double", + "type": "number" + } + }, + "required": [ + "timeout_seconds" + ], + "type": "object" + }, + "kind": { + "const": "invocation_started", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "$ref": "#/$defs/InvocationFinished" + }, + "kind": { + "const": "invocation_finished", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "properties": { + "error": { + "anyOf": [ + { + "$ref": "#/$defs/ErrorInfo" + }, + { + "type": "null" + } + ] + }, + "succeeded": { + "type": "boolean" + } + }, + "required": [ + "succeeded" + ], + "type": "object" + }, + "kind": { + "const": "runtime_stopped", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "properties": { + "message": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "stream": { + "type": "string" + } + }, + "required": [ + "stream", + "scope", + "message" + ], + "type": "object" + }, + "kind": { + "const": "diagnostic", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "properties": { + "data": { + "properties": { + "omitted_bytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "read_error": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "omitted_bytes" + ], + "type": "object" + }, + "kind": { + "const": "diagnostic_retention", "type": "string" } }, "required": [ - "kind" + "kind", + "data" ], "type": "object" }, { - "$ref": "#/$defs/EndLine", - "description": "Written once, after the invocation returned a result or failed.", "properties": { + "data": { + "$ref": "#/$defs/Summary" + }, "kind": { - "const": "end", + "const": "summary", "type": "string" } }, "required": [ - "kind" + "kind", + "data" ], "type": "object" } ], "properties": { + "elapsed_seconds": { + "format": "double", + "minimum": 0.0, + "type": "number" + }, "invocation_id": { - "description": "Invocation id.", + "type": [ + "string", + "null" + ] + }, + "recording_id": { "type": "string" }, "request_id": { - "description": "Request id.", "type": "string" }, "runtime_id": { - "description": "Runtime id.", - "type": "string" + "type": [ + "string", + "null" + ] }, "schema": { - "$ref": "#/$defs/RecordSchema", - "description": "Always `nemo.fabric.run_record`." + "$ref": "#/$defs/RecordSchema" + }, + "sequence": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "source": { + "$ref": "#/$defs/RecordSource" }, "time": { - "description": "UTC wall-clock time the line was written, RFC 3339 with milliseconds.", "format": "date-time", "type": "string" }, "version": { - "$ref": "#/$defs/RecordVersion", - "description": "Line format version. `v1alpha1` may change until a record reader ships." + "$ref": "#/$defs/RecordVersion" } }, "required": [ "schema", "version", - "runtime_id", - "invocation_id", + "recording_id", "request_id", - "time" + "sequence", + "time", + "elapsed_seconds", + "source" ], "title": "RunRecordLine", "type": "object" diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_report_format.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_report_format.py index d9784a7de..658ff7bbc 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_report_format.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/_report_format.py @@ -13,7 +13,7 @@ def format_money(value: int | None) -> str: them as free execution. JSON retains the original nanoUSD quantity. """ if value is None: - return "unavailable" + return "Unavailable" decimals = 2 if value == 0 else min(9, max(2, 12 - len(str(value)))) quantum = 10 ** (9 - decimals) rounded = (value + quantum // 2) // quantum @@ -23,16 +23,7 @@ def format_money(value: int | None) -> str: def format_duration_ms(value: int | float) -> str: """Format a measured duration without changing its measurement scope.""" - if value < 1000: - return f"{value:g} ms" - seconds = value / 1000 - if seconds < 60: - return f"{seconds:.3f}".rstrip("0").rstrip(".") + " s" - minutes, seconds = divmod(seconds, 60) - if minutes < 60: - return f"{int(minutes)} min {seconds:g} s" - hours, minutes = divmod(int(minutes), 60) - return f"{hours} h {minutes} min {seconds:g} s" + return f"{value / 1000:.3f} s" def format_bytes(value: int) -> str: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index 8da24932f..f9e314c81 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -189,12 +189,14 @@ class InstructionsConfig(FabricBaseModel): class RecordConfig(FabricBaseModel): - """Opt-in per-invocation run record. - - When ``enabled`` is true, NeMo Fabric writes - ``/.fabric///run-record.jsonl`` - for each invocation. It requires an artifact root, the ``local`` - environment provider, and a ``process`` or ``python`` adapter. + """Opt-in local request journal. + + When ``enabled`` is true, NeMo Fabric appends lifecycle events and + a terminal summary to + ``/.fabric/requests//run.jsonl`` + for each request. The journal can contain sensitive result data and error + messages. It requires an artifact root, the ``local`` environment provider, + and a ``process`` or ``python`` adapter. """ model_config = ConfigDict(extra="forbid") diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py index 9c6cf6948..ba3ed473a 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/reporting.py @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Inspect saved invocation evidence without a benchmark manifest or trace import.""" +"""Inspect saved run journals and legacy results without importing traces.""" from __future__ import annotations import argparse from collections.abc import Mapping -from datetime import datetime +from datetime import datetime, timezone import json import math from pathlib import Path @@ -25,6 +25,22 @@ ) MAX_RESULT_BYTES = 16 * 1024 * 1024 +MAX_HISTORY_EVENTS = 256 +MAX_HISTORY_BYTES = 1024 * 1024 +MAX_HISTORY_MILESTONE_RESERVE_BYTES = 64 * 1024 +MAX_HISTORY_FIELD_CHARS = 256 +_HISTORY_MILESTONES = frozenset( + { + "request_started", + "runtime_ready", + "invocation_started", + "invocation_finished", + "runtime_stopped", + "summary", + } +) +MAX_EVENT_BYTES = 64 * 1024 +MAX_HISTORY_MESSAGE_CHARS = 4096 def _text(value: Any, field: str) -> str: @@ -178,7 +194,7 @@ def summarize_result(result: Mapping[str, Any]) -> dict[str, Any]: ( ref.get("metadata", {}).get("boundary") for ref in artifacts - if ref.get("kind") == "run_result" + if ref.get("kind") in {"run_result", "run_log"} and isinstance(ref.get("metadata"), Mapping) ), None, @@ -249,10 +265,10 @@ def _attach_invocation_interval(summary: dict[str, Any], result_path: Path) -> N def read_evidence(path: Path) -> dict[str, Any]: - """Read a full result or invocation record; never follow references.""" + """Read a run journal or legacy result without opening referenced artifacts.""" path = Path(path) if path.suffix == ".jsonl": - return read_invocation_record(path) + return _read_journal(path) document = _load(path) summary = summarize_result(document) summary["result_path"] = str(path) @@ -260,6 +276,380 @@ def read_evidence(path: Path) -> dict[str, Any]: return summary +def _utc_time(value: Any) -> datetime: + text = _text(value, "time") + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError("invalid UTC timestamp") from error + if parsed.tzinfo is None or parsed.utcoffset().total_seconds() != 0: + raise ValueError("timestamps must include UTC timezone") + return parsed.astimezone(timezone.utc) + + +def _read_journal(path: Path) -> dict[str, Any]: + """Read a unified run log or the original two-line invocation journal.""" + with path.open("rb") as stream: + first = stream.readline(MAX_RESULT_BYTES + 1) + if len(first) > MAX_RESULT_BYTES: + raise ValueError("run record exceeds the reader's 16 MiB per-record limit") + try: + document = json.loads( + first, parse_constant=_invalid_constant, parse_float=_finite_float + ) + except (ValueError, RecursionError) as error: + raise ValueError( + f"run log must begin with a complete JSON record: {error}" + ) from error + if ( + isinstance(document, dict) + and document.get("schema") == "nemo.fabric.run_record" + ): + return read_invocation_record(path) + return _read_run_log(path) + + +def _journal_seconds(value: Any) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0: + raise ValueError("elapsed seconds must be finite and nonnegative") + try: + finite = math.isfinite(value) + except OverflowError as error: + raise ValueError("elapsed seconds exceed the supported range") from error + if not finite: + raise ValueError("elapsed seconds must be finite and nonnegative") + return value + + +def _journal_models(value: Any) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValueError("request models must be an object") + for role, model in value.items(): + _text(role, "model role") + if not isinstance(model, dict): + raise ValueError("requested model must be an object") + _text(model.get("model"), "model") + _text(model.get("provider"), "provider") + return value + + +def _journal_error(value: Any) -> dict[str, Any] | None: + if value is None: + return None + if not isinstance(value, dict): + raise ValueError("execution error must be an object or null") + for field in ("stage", "code"): + _text(value.get(field), "error " + field) + return { + field: value[field] + for field in ("stage", "code", "retryable") + if field in value + } + + +def _history_projection(row: Mapping[str, Any]) -> dict[str, Any]: + """Retain bounded display fields, not arbitrary event payloads.""" + item = {key: row[key] for key in ("kind", "time", "elapsed_seconds", "source")} + shortened = False + + def text(value: str, limit: int = MAX_HISTORY_FIELD_CHARS) -> str: + nonlocal shortened + shortened |= len(value) > limit + return value[:limit] + + if len(item["time"]) > MAX_HISTORY_FIELD_CHARS: + item["time"] = _utc_time(item["time"]).isoformat().replace("+00:00", "Z") + shortened = True + kind, data = row["kind"], row["data"] + projected = {} + keys = { + "invocation_started": ("timeout_seconds",), + "invocation_finished": ("status",), + "runtime_stopped": ("succeeded",), + "diagnostic": ("level", "severity", "stream", "scope"), + }.get(kind, ()) + for key in keys: + value = data.get(key) + if isinstance(value, str): + projected[key] = text(value) + elif isinstance(value, bool): + projected[key] = value + elif isinstance(value, (int, float)): + try: + projected[key] = _journal_seconds(value) + except ValueError: + shortened = True + if ( + kind in {"invocation_finished", "runtime_stopped"} + and data.get("error") is not None + ): + error = _journal_error(data["error"]) + projected["error"] = {key: text(error[key]) for key in ("stage", "code")} + if type(error.get("retryable")) is bool: + projected["error"]["retryable"] = error["retryable"] + elif "retryable" in error: + shortened = True + if kind == "diagnostic" and isinstance(data.get("message"), str): + projected["message"] = text(data["message"], MAX_HISTORY_MESSAGE_CHARS) + projected["message_truncated"] = ( + len(data["message"]) > MAX_HISTORY_MESSAGE_CHARS + ) + item["data"] = projected + if shortened: + item["projection_truncated"] = True + return item + + +def _read_run_log(path: Path) -> dict[str, Any]: + """Stream journal events with bounded display history; retain terminal evidence.""" + first = None + final = None + invocation = None + history = [] + history_bytes = 0 + history_truncated = 0 + omitted = 0 + truncated = False + identities: dict[str, str] = {} + previous_elapsed = 0.0 + milestones = set() + diagnostics = {"omitted_bytes": 0, "read_errors": []} + with path.open("rb") as stream: + sequence = 0 + while line := stream.readline(MAX_RESULT_BYTES + 1): + if len(line) > MAX_RESULT_BYTES: + raise ValueError( + "run record exceeds the reader's 16 MiB per-record limit; the original file is unchanged" + ) + if not line.endswith(b"\n"): + truncated = True + break + try: + row = json.loads( + line, parse_constant=_invalid_constant, parse_float=_finite_float + ) + except (ValueError, RecursionError) as error: + raise ValueError("invalid complete run log record") from error + if not isinstance(row, dict) or (row.get("schema"), row.get("version")) != ( + "nemo.fabric.run", + "v1alpha1", + ): + raise ValueError("unsupported run log schema or version") + if final is not None: + raise ValueError("run log contains records after its terminal summary") + if type(row.get("sequence")) is not int or row["sequence"] != sequence: + raise ValueError( + "run log sequence must be contiguous and start at zero" + ) + sequence += 1 + for field in ("recording_id", "request_id"): + value = _text(row.get(field), field) + if field in identities and identities[field] != value: + raise ValueError("run log identity changed") + identities[field] = value + for field in ("runtime_id", "invocation_id"): + if field in row: + value = _text(row[field], field) + if field in identities and identities[field] != value: + raise ValueError("run log identity changed") + identities[field] = value + _utc_time(row.get("time")) + elapsed = _journal_seconds(row.get("elapsed_seconds")) + if elapsed < previous_elapsed: + raise ValueError( + "run elapsed seconds must be finite, nonnegative and ordered" + ) + previous_elapsed = elapsed + if len(_text(row.get("source"), "source")) > 256: + raise ValueError("run source exceeds 256 characters") + kind = _text(row.get("kind"), "kind") + if len(kind) > 128: + raise ValueError("run kind exceeds 128 characters") + if ( + kind in {"diagnostic", "resource_sample"} + and len(line) > MAX_EVENT_BYTES + ): + raise ValueError("run event exceeds the reader's 64 KiB event limit") + if kind in {"runtime_ready", "invocation_started", "runtime_stopped"}: + if kind in milestones: + raise ValueError("duplicate lifecycle milestone") + milestones.add(kind) + data = row.get("data") + if not isinstance(data, dict): + raise ValueError("run record data must be an object") + if first is None: + if kind != "request_started": + raise ValueError("run log must begin with request_started") + override_keys = data.get("override_keys", []) + if not isinstance(override_keys, list) or any( + not isinstance(key, str) for key in override_keys + ): + raise ValueError("override_keys must be a list of strings") + _journal_models(data.get("models", {})) + _text(data.get("harness"), "harness") + if data.get("outcome_scope") not in { + "invocation", + "one_shot_lifecycle", + }: + raise ValueError("invalid request outcome scope") + first = row + elif kind == "request_started": + raise ValueError("duplicate request_started record") + if kind == "runtime_ready" and data.get("runtime_id") != identities.get( + "runtime_id" + ): + raise ValueError("runtime_ready identity differs from envelope") + if kind == "invocation_finished": + if invocation is not None: + raise ValueError("duplicate invocation_finished record") + invocation = row + elif kind == "summary": + final = row + if kind == "diagnostic_retention": + omitted_bytes = data.get("omitted_bytes") + read_error = data.get("read_error") + if type(omitted_bytes) is not int or omitted_bytes < 0: + raise ValueError("diagnostic omitted_bytes must be nonnegative") + if read_error is not None and not isinstance(read_error, str): + raise ValueError("diagnostic read_error must be text or null") + diagnostics["omitted_bytes"] += omitted_bytes + if read_error and len(diagnostics["read_errors"]) < 16: + diagnostics["read_errors"].append( + read_error[:MAX_HISTORY_MESSAGE_CHARS] + ) + if kind not in {"resource_sample", "diagnostic_retention"}: + item = _history_projection(row) + item_bytes = len( + json.dumps(item, ensure_ascii=True, separators=(",", ":")).encode( + "utf-8" + ) + ) + milestone = kind in _HISTORY_MILESTONES + if milestone or ( + len(history) < MAX_HISTORY_EVENTS + and history_bytes + item_bytes + <= MAX_HISTORY_BYTES - MAX_HISTORY_MILESTONE_RESERVE_BYTES + ): + history.append(item) + history_bytes += item_bytes + history_truncated += item.get("projection_truncated", False) + else: + omitted += 1 + if first is None: + raise ValueError("run log has no complete request_started record") + payload = final["data"] if final else {} + result = payload.get("result") + if result is not None: + if not isinstance(result, dict): + raise ValueError("summary result must be an object or null") + summary = summarize_result(result) + for field in ("request_id", "runtime_id", "invocation_id"): + if field not in identities or summary[field] != identities[field]: + raise ValueError("terminal result identity differs from run log") + elif final is None and invocation is not None: + evidence = invocation["data"] + if not all(field in identities for field in ("runtime_id", "invocation_id")): + raise ValueError( + "invocation evidence has no runtime or invocation identity" + ) + if any( + field in evidence and evidence[field] != identities[field] + for field in ("request_id", "runtime_id", "invocation_id") + ): + raise ValueError("invocation evidence identity differs from run log") + summary = summarize_result({**evidence, **identities}) + summary.update( + schema="nemo.fabric.invocation_record_summary", + outcome_boundary="invocation", + state="record_only", + elapsed_ms=None, + exchange_elapsed_ms=None, + ) + else: + summary = { + "schema": "nemo.fabric.request_summary", + "version": "v1alpha1", + "request_id": identities["request_id"], + "status": None, + "state": "no_result" if final else "registered", + "usage": None, + "model_usage": None, + "execution_error": payload.get("error"), + } + if final is not None and result is None and invocation is not None: + evidence = invocation["data"] + if not all(field in identities for field in ("runtime_id", "invocation_id")): + raise ValueError( + "invocation evidence has no runtime or invocation identity" + ) + if any( + field in evidence and evidence[field] != identities[field] + for field in ("request_id", "runtime_id", "invocation_id") + ): + raise ValueError("invocation evidence identity differs from run log") + retained = summarize_result({**evidence, **identities}) + retained.update( + schema="nemo.fabric.invocation_record_summary", + outcome_boundary="invocation", + ) + summary["invocation_summary"] = retained + for field in ( + "usage", + "model_usage", + "system_stats", + "system_stats_error", + "resource_window", + "accounting_error", + ): + summary[field] = retained[field] + if invocation is not None or final is not None: + exchange = ( + payload.get("adapter_exchange_elapsed_seconds") + if final is not None + else invocation["data"].get("adapter_exchange_elapsed_seconds") + ) + if exchange is not None: + summary["adapter_exchange"] = { + "elapsed_ms": _journal_seconds(exchange) * 1000 + } + if final is not None: + if payload.get("outcome_scope") not in {"invocation", "one_shot_lifecycle"}: + raise ValueError("invalid final outcome scope") + if payload["outcome_scope"] != first["data"]["outcome_scope"]: + raise ValueError("final outcome scope differs from request") + _utc_time(payload.get("request_started_at")) + _utc_time(payload.get("request_finished_at")) + duration = _journal_seconds(payload.get("elapsed_seconds")) + error = _journal_error(payload.get("error")) + if result is None and error is None: + raise ValueError("final no-result summary must contain an execution error") + summary.update( + outcome_boundary=payload["outcome_scope"], + started_at=payload["request_started_at"], + finished_at=payload["request_finished_at"], + request_elapsed_seconds=duration, + ) + if payload.get("error") is not None: + summary["execution_error"] = error + summary["stage"] = error["stage"] + else: + summary["started_at"] = first["time"] + summary.update( + recording_id=identities["recording_id"], + harness=summary.get("harness") or first["data"].get("harness"), + requested_models=first["data"].get("models", {}), + model_override_unrecorded="models" in first["data"].get("override_keys", []), + history=history, + diagnostics=diagnostics, + history_projection_truncated_count=history_truncated, + history_omitted_count=omitted, + truncated_tail=truncated, + terminal_summary_present=final is not None, + evidence_path=str(path), + ) + return summary + + RESOURCE_SCOPE = "runtime_adapter_host_process_tree" @@ -356,27 +746,8 @@ def parse_time(value: str) -> datetime: return window -def _render_resource_window(window: Mapping[str, Any]) -> list[str]: - lines = [] - if window["cpu_counter_change_seconds"] is not None: - lines.append( - f"Runtime process-tree CPU counter change: {window['cpu_counter_change_seconds']:.3f} s between snapshots (not invocation CPU)." - ) - elif window["delta_reason"]: - lines.append( - "Runtime process-tree CPU counter change unavailable: " - + window["delta_reason"] - + "." - ) - if window["start"] is not None and window["end"] is not None: - lines.append( - f"RSS observations: {format_bytes(window['start']['resident_memory_bytes'])} → {format_bytes(window['end']['resident_memory_bytes'])} (not a peak)." - ) - return lines - - def read_invocation_record(path: Path) -> dict[str, Any]: - """Read bounded start/end evidence; a truncated tail never creates a result. + """Read legacy start/end evidence; a truncated tail never creates a result. This reports the core invocation window before one-shot shutdown. Only the typed model_usage projection supplies normalized accounting; legacy counters @@ -455,6 +826,10 @@ def read_invocation_record(path: Path) -> dict[str, Any]: "version": "v1alpha1", **identity, "status": end["status"] if end else None, + "execution_error": end.get("error") if end else None, + "harness": rows[0].get("harness"), + "started_at": rows[0].get("time"), + "finished_at": end.get("time") if end else None, "model_usage": normalized, "usage": usage, "accounting_error": accounting_error, @@ -472,309 +847,601 @@ def read_invocation_record(path: Path) -> dict[str, Any]: def render_invocation_record_summary( summary: Mapping[str, Any], *, details: bool = False ) -> str: - """Describe the invocation interval without assuming whole-runtime success.""" - if summary.get("usage") is not None: - display = dict(summary) + """Render legacy invocation evidence without claiming one-shot completion.""" + display = dict(summary) + if summary.get("elapsed_ms") is not None: display["execution_interval"] = {"elapsed_ms": summary["elapsed_ms"]} - if summary["exchange_elapsed_ms"] is not None: - display["adapter_exchange"] = {"elapsed_ms": summary["exchange_elapsed_ms"]} - lines = render_result_summary(display, details=details).splitlines() - lines[0] = "Invocation outcome before runtime shutdown: " + summary["status"] - lines.append( - "The separate full result is authoritative for the one-shot outcome." - ) - if summary["truncated_tail"]: - lines.append( - "A truncated final line was ignored; complete lines were retained." - ) - lines.append( - "Saved evidence: " + json.dumps(summary["evidence_path"], ensure_ascii=True) - ) - return "\n".join(lines) - lines = [ - "Invocation: " - + json.dumps( - f"{summary['runtime_id']}/{summary['invocation_id']}", ensure_ascii=True - ), - "Invocation outcome before runtime shutdown: " - + (summary["status"] or "unavailable"), - ] - if summary.get("provenance_error"): - lines.append( - "Saved execution facts were unreadable; status and accounting are retained." - ) if summary.get("exchange_elapsed_ms") is not None: - lines.append( - f"Adapter exchange: {summary['exchange_elapsed_ms']:,} ms (excludes queue and result persistence)." - ) - elif summary["elapsed_ms"] is not None: - lines.append(f"Core invocation interval: {summary['elapsed_ms']:,} ms") - else: - lines.append( - "No complete end record; elapsed time and final usage are unknown." - ) - if summary["truncated_tail"]: - lines.append( - "A truncated final line was ignored; complete lines were retained." - ) - if summary.get("accounting_error"): - lines.append( - "Saved model accounting was invalid; execution evidence is retained." - ) - lines.append("Inspect the full result for accounting and the one-shot outcome.") - lines.append( - "Saved evidence: " + json.dumps(summary["evidence_path"], ensure_ascii=True) + display["adapter_exchange"] = {"elapsed_ms": summary["exchange_elapsed_ms"]} + return render_result_summary(display, details=details) + ( + "\nThis record covers the agent call; it does not establish the one-shot shutdown outcome." ) - return "\n".join(lines) -def _render_tokens(dimensions: Mapping[str, Any]) -> list[str]: - totals = ("input_tokens", "output_tokens", "total_tokens") - caches = ("cached_input_tokens", "cache_write_input_tokens") +def _safe(value: Any) -> str: + """Escape control characters without adding JSON quotes around plain text.""" + return json.dumps(str(value), ensure_ascii=True)[1:-1] - def coverage(names: tuple[str, ...]) -> str: - if all(dimensions[name]["complete_total"] is not None for name in names): - return "complete" - if any( - dimensions[name]["observed_sum"] is not None - or dimensions[name]["complete_total"] is not None - for name in names - ): - return "incomplete" - return "unavailable" - total_coverage, cache_coverage = coverage(totals), coverage(caches) +def _rows(items: list[tuple[str, str]]) -> list[str]: + width = max(len(label) for label, _ in items) + return [f"{label:<{width}} {value}" for label, value in items] + + +def _dimension_value( + field: Mapping[str, Any], *, money: bool = False, estimate: bool = False +) -> str: + complete = field.get("complete_total") + observed = field.get("observed_sum") + value = complete if complete is not None else observed + if value is None: + return "Unavailable" + rendered = format_money(value) if money else f"{value:,}" + if complete is None: + rendered += " recorded; " + ( + "full estimate unknown" if estimate else "total unknown" + ) + return rendered + + +def _time_rows(summary: Mapping[str, Any]) -> list[tuple[str, str]]: + start_value, end_value = summary.get("started_at"), summary.get("finished_at") + if not start_value: + return [] + start = _utc_time(start_value) + end = _utc_time(end_value) if end_value else None + same_day = end is None or start.date() == end.date() + fmt = "%H:%M:%S.%f" if same_day else "%Y-%m-%d %H:%M:%S.%f" + rows = [("Date:", start.date().isoformat())] if same_day else [] + rows.extend( + [ + ("Timezone:", "UTC"), + ("Started:", start.strftime(fmt)[:-3]), + ("Finished:", end.strftime(fmt)[:-3] if end else "Unavailable"), + ] + ) + return rows + - def count(name: str, label: str, group_coverage: str) -> str: - field = dimensions[name] - if field["complete_total"] is not None: - suffix = "" if group_coverage == "complete" else " (complete)" - return f"{label} {field['complete_total']:,}{suffix}" - if field["observed_sum"] is not None: - return f"{label} {field['observed_sum']:,} (observed, incomplete)" - return f"{label} unavailable" +def _models(summary: Mapping[str, Any]) -> tuple[str, str]: + facts = summary.get("what_ran") or {} + roles = facts.get("models", {}) + requested = summary.get("requested_models", {}) + names = set(roles) | set(requested) + if "default" in names: + role = "default" + elif len(names) == 1: + role = next(iter(names)) + elif names: + return f"Multiple roles ({len(names)})", "See details" + else: + return "Unavailable", "Not reported" + model_facts = roles.get(role, {}) + name = model_facts.get("requested", {}).get("value") + served = model_facts.get("served", {}).get("value") + if name is None and summary.get("model_override_unrecorded"): + requested_text = "Unavailable (model override not retained)" + else: + name = name or requested.get(role, {}).get("model") + requested_text = _safe(name) if name else "Unavailable" + return requested_text, _safe(served) if served else "Not reported" + +def _resource_rows(summary: Mapping[str, Any]) -> list[tuple[str, str]]: + stats = summary.get("system_stats") or {} + cpu = stats.get("cpu_time_seconds_estimate") + after = stats.get("after", {}) + memory = ( + after.get("snapshot", {}).get("resident_memory_bytes") + if after.get("status") == "available" + else None + ) + if not stats and summary.get("resource_snapshot") is not None: + memory = summary["resource_snapshot"]["resident_memory_bytes"] + sampled = stats.get("sampled") + peak = (sampled or {}).get("resident_memory_bytes") or {} return [ - "Tokens: " - + "; ".join( - count(name, label, total_coverage) - for name, label in zip(totals, ("input", "output", "total")) + ("CPU time, estimated", f"{cpu:.3f} s" if cpu is not None else "Unavailable"), + ( + "Memory after execution", + format_bytes(memory) if memory is not None else "Unavailable", ), - "Cache (included in input): " - + "; ".join( - count(name, label, cache_coverage) - for name, label in zip(caches, ("reads", "writes")) + ( + "Peak memory, sampled", + format_bytes(peak["sampled_peak"]) + if peak.get("sampled_peak") is not None + else "Unavailable" + + (" (sampling disabled)" if stats and sampled is None else ""), ), - f"Token coverage: input/output/total {total_coverage}; cache breakdown {cache_coverage}", ] -def render_result_summary(summary: Mapping[str, Any], *, details: bool = False) -> str: - """Render a concise overview; details add resource evidence. +def _gap_lines(summary: Mapping[str, Any]) -> list[str]: + explanations = set() + labels = { + "input_tokens": "input tokens", + "cached_input_tokens": "cache read tokens", + "cache_write_input_tokens": "cache write tokens", + "output_tokens": "output tokens", + "total_tokens": "total tokens", + "estimated_cost": "estimated cost", + "reported_cost": "provider-reported cost", + } + messages = { + "unfinished_work": "some model work was unfinished at return", + "scope_unverified": "the source could not establish coverage of all model work", + "missing_usage": "the source did not report usable usage", + } + for gap in (summary.get("model_usage") or {}).get("gaps", []): + dimensions = gap["dimensions"] + # An absent provider cost is routine and already represented by omission. + if dimensions == ["reported_cost"]: + continue + names = ", ".join(labels.get(name, name) for name in dimensions) + reason = ( + "the applicable cache-write price was not established" + if gap.get("code") == "cache_write_tariff_unverified" + else messages.get(gap["reason"], gap["reason"].replace("_", " ")) + ) + if dimensions == ["estimated_cost"] and gap["reason"] != "unfinished_work": + reason = "the estimate could not be established from available usage and pricing evidence" + explanations.add(f"Accounting gap: {names}: {reason}.") + for key, message in ( + ( + "accounting_error", + "Saved model accounting is invalid; execution evidence is retained.", + ), + ( + "system_stats_error", + "Saved resource measurements are invalid; execution and accounting are retained.", + ), + ( + "resource_error", + "Saved resource snapshot is invalid; accounting is retained.", + ), + ( + "record_error", + "The sibling invocation record could not be used; result accounting is retained.", + ), + ( + "provenance_error", + "Saved execution facts are unreadable; measurements are retained.", + ), + ): + if summary.get(key): + explanations.add(message) + return sorted(explanations) - Token and cost completeness remain separate. JSON quantities and accounting - semantics are unchanged by this presentation. - """ - dimensions = summary["usage"]["dimensions"] - lines = [f"Execution: {summary['status']}"] - if details: - lines.append( - "Invocation: " - + json.dumps( - f"{summary['runtime_id']}/{summary['invocation_id']}", ensure_ascii=True + +def _history_lines(summary: Mapping[str, Any]) -> list[str]: + history = summary.get("history", []) + if not history: + return [] + labels = { + "request_started": "Request recording started", + "runtime_ready": "Agent runtime ready", + "invocation_started": "Invocation started", + "invocation_finished": "Invocation finished", + "runtime_stopped": "Agent runtime stopped", + "summary": "Final summary", + } + lines = ["Execution history"] + day = None + for event in history: + kind, data = event["kind"], event["data"] + if kind == "summary": + continue + time = _utc_time(event["time"]) + if time.date() != day: + day = time.date() + lines.extend( + [ + f"Date: {day.isoformat()} · Timezone: UTC", + "Time Elapsed (s) Event", + ] ) - ) - if summary.get("harness"): - lines.append("Harness: " + json.dumps(summary["harness"], ensure_ascii=True)) - if summary.get("execution_error"): - error = summary["execution_error"] - lines.append( - "Execution error: " - + json.dumps( - {field: error[field] for field in ("stage", "code") if field in error}, - ensure_ascii=True, + label = labels.get(kind, _safe(kind.replace("_", " "))) + if kind == "invocation_finished" and data.get("status"): + label += ": " + _safe(data["status"]) + elif kind == "diagnostic": + label = _safe(event["source"]) + ": " + _safe(data.get("message", "")) + if data.get("message_truncated"): + label += " (message shortened; full event retained in file)" + if kind == "runtime_stopped" and data.get("succeeded") is False: + label = "Agent runtime shutdown failed" + if data.get("error"): + label += ( + ": " + + _safe(data["error"]["stage"]) + + "/" + + _safe(data["error"]["code"]) ) - ) - if details and summary.get("outcome_boundary"): - lines.append( - "Outcome boundary: " - + json.dumps(summary["outcome_boundary"], ensure_ascii=True) - ) - interval = summary.get("execution_interval") - if summary.get("adapter_exchange") is not None: + if event.get("projection_truncated"): + label += " (history details shortened; full event retained in file)" lines.append( - f"Elapsed (adapter exchange): {format_duration_ms(summary['adapter_exchange']['elapsed_ms'])} (excludes queue and result persistence)." + f"{time.strftime('%H:%M:%S.%f')[:-3]} {event['elapsed_seconds']:11.3f} {label}" ) - elif summary.get("system_stats") is not None: + if summary.get("history_omitted_count"): lines.append( - f"Elapsed (adapter exchange): {format_duration_ms(summary['system_stats']['elapsed_seconds'] * 1000)} (excludes queue and result persistence)." + f"Additional history entries retained in file: {summary['history_omitted_count']:,}" ) - elif interval is not None and interval["elapsed_ms"] is not None: - lines.append( - f"Core invocation interval before shutdown: {format_duration_ms(interval['elapsed_ms'])}" - ) - else: - lines.append("Elapsed: unavailable") - if details and summary.get("resource_window") is not None: - lines.extend(_render_resource_window(summary["resource_window"])) - resource = summary.get("resource_snapshot") - if details and resource is not None: - lines.append( - f"Runtime process-tree snapshot: CPU {resource['cpu_time_seconds']:.3f} s since host start; " - f"RSS {format_bytes(resource['resident_memory_bytes'])}; live processes {resource['process_count']}" - ) - normalized = summary.get("model_usage") - models = normalized.get("models", []) if normalized is not None else [] - if models: - shown = models if details else models[:3] - lines.append( - "Accounted models: " - + ", ".join( - json.dumps( - f"{model.get('provider') or 'unspecified'}/{model['model']}", - ensure_ascii=True, + return lines + + +def _details_lines(summary: Mapping[str, Any]) -> list[str]: + sections = [] + history = _history_lines(summary) + if history: + sections.append(history) + stats = summary.get("system_stats") + if stats: + lines = ["Resource details"] + if "process" in stats["collected_groups"]: + lines.extend( + [ + "Measured processes: adapter host and its child processes", + "Endpoint memory is not a peak or time average.", + "Remote model inference is not measured.", + "Concurrent work can contribute to CPU time; exited processes can cause undercounting.", + ] + ) + else: + lines.append("Process measurements: Not collected") + sampled = stats.get("sampled") + if sampled: + memory = sampled.get("resident_memory_bytes") + lines.extend( + _rows( + [ + ( + "Requested sample period", + format_duration_ms(sampled["target_interval_ms"]), + ), + ("Samples retained", f"{sampled['sample_count']:,}"), + ("Samples dropped", f"{sampled['dropped_samples']:,}"), + ( + "Largest sample gap", + f"{sampled['largest_gap_seconds']:.3f} s", + ), + ] ) - for model in shown ) - + ( - f"; {len(models) - len(shown)} more (see --details)" - if len(shown) < len(models) - else "" + if memory: + mean = memory.get("time_weighted_mean") + lines.extend( + _rows( + [ + ( + "Sampled memory mean", + format_bytes(mean) + if mean is not None + else "Unavailable", + ), + ( + "Time covered by mean", + f"{memory['covered_fraction']:.1%}", + ), + ] + ) + ) + lines.append("Short memory spikes between samples can be missed.") + if sampled["truncated"]: + lines.append("Sampling retention was truncated.") + lines.extend( + "Sampling diagnostic: " + _safe(value) + for value in sampled["diagnostics"] ) + cgroup = stats.get("cgroup") + if cgroup: + lines.append("Cgroup measurements (may include other workloads)") + for key, label in ( + ("cpu_usage_microseconds", "CPU time"), + ("cpu_throttled_microseconds", "CPU throttled time"), + ("oom_kills", "OOM kills"), + ): + field = cgroup[key] + value = field.get("value") + rendered = ( + ( + f"{value / 1_000_000:.3f} s" + if key.endswith("microseconds") + else f"{value:,}" + ) + if field["status"] == "available" + else "Unavailable" + ) + lines.extend(_rows([(label, rendered)])) + sections.append(lines) + elif summary.get("resource_snapshot") is not None: + snapshot = summary["resource_snapshot"] + sections.append( + [ + "Runtime snapshot", + "Observed at: " + _safe(snapshot["observed_at"]), + "CPU counter since host start: " + + f"{snapshot['cpu_time_seconds']:.3f} s", + "Resident memory: " + format_bytes(snapshot["resident_memory_bytes"]), + "This snapshot does not establish invocation resource consumption.", + ] ) - if not normalized["models_exhaustive"]: - lines.append("Per-model accounting is not exhaustive.") - lines.extend(_render_tokens(dimensions)) - gaps = normalized.get("gaps", []) if normalized is not None else [] - not_reported = { - dimension - for gap in gaps - if gap["reason"] == "missing_usage" - for dimension in gap["dimensions"] - } - for dimension, label in ( - ("reported_cost", "Provider-reported cost"), - ("estimated_cost", "Catalog estimate"), - ): - field = dimensions[dimension] - if field["complete_total"] is not None: - text = format_money(field["complete_total"]) + " (complete)" - elif field["observed_sum"] is not None: - text = format_money(field["observed_sum"]) + " (observed, incomplete)" - else: - text = "unavailable" + ( - " (not reported)" if dimension in not_reported else "" + facts = summary.get("what_ran") or {} + roles = facts.get("models", {}) + configured = summary.get("requested_models", {}) + if len(set(roles) | set(configured)) > 1: + lines = ["Model roles"] + for role in sorted(set(roles) | set(configured)): + facts_for_role = roles.get(role, {}) + requested = facts_for_role.get("requested", {}).get("value") + if requested is None and not summary.get("model_override_unrecorded"): + requested = configured.get(role, {}).get("model") + served = facts_for_role.get("served", {}).get("value") + lines.append( + _safe(role) + + ": requested " + + (_safe(requested) if requested else "Unavailable") + + "; served " + + (_safe(served) if served else "Not reported") ) - lines.append(f"{label}: {text}") - pricing = summary["usage"].get("pricing") - if pricing: - if details: + sections.append(lines) + normalized = summary.get("model_usage") or {} + models = normalized.get("models", []) + if models: + lines = ["Per-model accounting"] + for model in models: lines.append( - "Pricing basis: " - + json.dumps(pricing, ensure_ascii=True, sort_keys=True) + _safe((model.get("provider") or "unspecified") + "/" + model["model"]) ) - else: - catalogs = [ - f"{label} {pricing[key].removeprefix('sha256:')[:12]}" - for key, label in ( - ("bundled_catalog_sha256", "bundled"), - ("user_catalog_sha256", "user"), + lines.extend( + _rows( + [ + ( + label, + _dimension_value( + { + "complete_total": model.get(key), + "observed_sum": model.get("observed", {}).get(key), + } + ), + ) + for key, label in ( + ("input_tokens", "Input tokens"), + ("output_tokens", "Output tokens"), + ("total_tokens", "Total tokens"), + ) + ] ) - if pricing.get(key) - ] - lines.append( - "Pricing basis: catalog" - + ("; " + "; ".join(catalogs) if catalogs else "") - + " (full basis in --details or --json)." ) - if summary.get("recording_failed"): - lines.append( - "Recording failed; execution and available accounting are retained. " - "Inspect record_write_failed events in the original result." + if not normalized.get("models_exhaustive"): + lines.append("This model breakdown does not cover all invocation usage.") + sections.append(lines) + pricing = (summary.get("usage") or {}).get("pricing") + if pricing: + lines = [ + "Pricing", + "Price source: " + + ( + "Fabric packaged price table" + if not pricing.get("user_catalog_sha256") + else "Fabric and user price tables" + ), + ] + for key, label in ( + ("bundled_catalog_sha256", "Packaged table fingerprint"), + ("user_catalog_sha256", "User table fingerprint"), + ): + if pricing.get(key): + lines.append(label + ": " + _safe(pricing[key])) + lines.extend( + [ + "Input includes cache; output includes reasoning.", + "The estimate requires known token counts and applicable prices.", + ] ) - explanations: set[str] = set() - missing: set[str] = set() - for gap in gaps: - if gap.get("code") == "cache_write_tariff_unverified": - explanations.add("The applicable cache-write price was not established.") - elif gap["reason"] == "unfinished_work": - explanations.add( - "Some invocation-owned model work was unfinished at return." + applied = list(normalized.get("cost", {}).get("applied_rates", [])) + for model in models: + applied.extend(model.get("cost", {}).get("applied_rates", [])) + seen = set() + for rate in applied: + key = json.dumps(rate, sort_keys=True) + if key in seen: + continue + seen.add(key) + lines.extend( + [ + "Rates for " + + _safe(rate["provider"] + "/" + rate["model_id"]) + + " (USD per 1,000,000 tokens)", + " Pricing date: " + _safe(rate["pricing_as_of"]), + " Source: " + _safe(rate["pricing_source"]), + ] ) - elif gap["reason"] == "scope_unverified": - explanations.add( - "The source did not establish complete invocation coverage." + for name, label in ( + ("input", "Uncached input"), + ("cache_read", "Cache reads"), + ("cache_write", "Cache writes"), + ("output", "Output"), + ): + value = rate["rates"].get(name) + if value is not None: + lines.append(" " + label + ": " + format_money(value)) + sections.append(lines) + refs = [] + for artifact in summary.get("artifacts", []): + if artifact.get("kind") in {"run_log", "run_record", "run_result"}: + continue + if isinstance(artifact.get("path"), str): + name = { + "stdout.txt": "Adapter output", + "stderr.txt": "Adapter diagnostics", + }.get( + Path(artifact["path"]).name, + artifact.get("name") or artifact.get("kind") or "Artifact", ) - else: - labels = { - "reported_cost": "provider-reported cost", - "estimated_cost": "catalog estimate", - } - missing.update( - labels.get(dimension, dimension.replace("_", " ")) - for dimension in gap["dimensions"] - if dimension not in {"reported_cost", "estimated_cost"} + refs.append((_safe(name), _safe(artifact["path"]))) + for reference in summary.get("telemetry", []): + if isinstance(reference, Mapping) and isinstance(reference.get("uri"), str): + refs.append(("Trace reference", _safe(reference["uri"]))) + if refs: + sections.append(["Related artifacts", *_rows(refs)]) + return [line for section in sections for line in ["", *section]] + + +def render_result_summary(summary: Mapping[str, Any], *, details: bool = False) -> str: + """Render concise measurement values; details add retained history and sources.""" + requested, served = _models(summary) + status = summary.get("status") + execution = ( + status.capitalize() + if status + else ( + "No result returned" + if summary.get("state") == "no_result" + else "Completion unconfirmed" + ) + ) + if summary.get("terminal_summary_present") is False: + execution = "Completion unconfirmed" + lines = _rows( + [ + ( + "Harness:", + _safe(summary["harness"]) if summary.get("harness") else "Unavailable", + ), + ("Requested model:", requested), + ("Served model:", served), + ("Execution:", execution), + *_time_rows(summary), + ] + ) + error = summary.get("execution_error") + if isinstance(error, Mapping): + lines.append( + "Execution error: " + + "; ".join( + _safe(error[key]) + for key in ("stage", "code") + if error.get(key) is not None ) - if summary.get("accounting_error"): - explanations.add( - "Saved model accounting was invalid; execution evidence is retained." ) - if missing: - explanations.add("Not reported: " + ", ".join(sorted(missing)) + ".") - for explanation in sorted(explanations): - lines.append(explanation) - if summary.get("legacy_usage") is not None and normalized is None: - lines.append("Raw harness usage is retained in JSON as unverified diagnostics.") - if summary.get("record_error"): + stats = summary.get("system_stats") or {} + exchange = summary.get("adapter_exchange") or {} + execution_ms = exchange.get("elapsed_ms") + if execution_ms is None and "elapsed_seconds" in stats: + execution_ms = stats["elapsed_seconds"] * 1000 + request_seconds = summary.get("request_elapsed_seconds") + measurements = [ + ( + "Request elapsed", + f"{request_seconds:.3f} s" + if request_seconds is not None + else "Unavailable", + ), + ( + "Request execution time", + format_duration_ms(execution_ms) + if execution_ms is not None + else "Unavailable", + ), + *_resource_rows(summary), + ] + dimensions = (summary.get("usage") or {}).get("dimensions", {}) + for key, label in ( + ("input_tokens", "Input tokens"), + ("output_tokens", "Output tokens"), + ("total_tokens", "Total tokens"), + ("cached_input_tokens", "Cache read tokens"), + ("cache_write_input_tokens", "Cache write tokens"), + ("estimated_cost", "Estimated cost, USD"), + ): + measurements.append( + ( + label, + _dimension_value( + dimensions.get(key, {}), + money=key == "estimated_cost", + estimate=key == "estimated_cost", + ), + ) + ) + reported = dimensions.get("reported_cost", {}) + if ( + reported.get("complete_total") is not None + or reported.get("observed_sum") is not None + ): + measurements.append( + ("Provider-reported cost, USD", _dimension_value(reported, money=True)) + ) + lines.extend( + [ + "", + *_rows([("Measurement", "Value"), *measurements]), + "", + "Cache tokens are included in input.", + ] + ) + lines.extend(_gap_lines(summary)) + diagnostics = summary.get("diagnostics") or {} + if diagnostics.get("omitted_bytes"): lines.append( - "The sibling invocation record could not be used; result accounting is retained." + f"Diagnostic capture omitted {diagnostics['omitted_bytes']:,} bytes." ) - if details and summary.get("system_stats") is not None: - stats = summary["system_stats"] + for read_error in diagnostics.get("read_errors", []): + lines.append("Diagnostic capture error: " + _safe(read_error)) + if summary.get("recording_failed"): lines.append( - "System stats collected groups: " - + json.dumps(stats["collected_groups"], ensure_ascii=True) + "Recording failed; execution and available accounting are retained. Inspect record_write_failed events." ) - sampled = stats.get("sampled") - if sampled is not None: - lines.append( - f"Resource samples: {sampled['sample_count']:,}; dropped {sampled['dropped_samples']:,}; truncated {str(sampled['truncated']).lower()} (sampled peaks are not guaranteed true peaks)." - ) - if summary.get("system_stats_error"): + if summary.get("legacy_usage") is not None and summary.get("model_usage") is None: + lines.append("Raw harness usage is retained in JSON as unverified diagnostics.") + if summary.get("truncated_tail"): lines.append( - "Saved system stats were invalid; execution and accounting are retained." + "An unfinished final line was ignored; earlier complete records were retained." ) - if summary.get("resource_error"): + if summary.get("terminal_summary_present") is False: lines.append( - "The saved resource snapshot is invalid; result accounting is retained." + "No final summary was saved; retained events do not establish the request outcome." ) - for artifact in summary.get("artifacts", []): - if artifact.get("kind") == "log" and isinstance(artifact.get("path"), str): + if summary.get("invocation_summary") is not None: + lines.append( + "Usage and resources describe the recorded agent call; no final result was returned." + ) + if details: + lines.extend(_details_lines(summary)) + interval = summary.get("execution_interval") + if interval is not None and interval.get("elapsed_ms") is not None: lines.append( - "Log artifact: " + json.dumps(artifact["path"], ensure_ascii=True) + "Core invocation interval: " + + format_duration_ms(interval["elapsed_ms"]) ) - for reference in summary.get("telemetry", []): - if isinstance(reference, Mapping) and isinstance(reference.get("uri"), str): + boundary = summary.get("outcome_boundary") + if boundary in {"one_shot_lifecycle", "invocation"}: + label = ( + "request through runtime shutdown" + if boundary == "one_shot_lifecycle" + else "this invocation; later shutdown excluded" + ) + lines.extend(["", "Outcome covers: " + label]) + if summary.get("runtime_id") and summary.get("invocation_id"): lines.append( - "Telemetry reference: " - + json.dumps(reference["uri"], ensure_ascii=True) + "Invocation: " + + _safe(summary["runtime_id"] + "/" + summary["invocation_id"]) ) - if summary.get("result_path"): - lines.append( - "Saved result: " + json.dumps(summary["result_path"], ensure_ascii=True) - ) - lines.append("Execution status is not a task verdict. Estimates are not invoices.") + path = summary.get("evidence_path") or summary.get("result_path") + if path: + lines.extend(["", "Run file: " + _safe(path)]) return "\n".join(lines) def _render(summary: Mapping[str, Any], *, details: bool = False) -> str: + if summary.get("recording_id") is not None: + return render_result_summary(summary, details=details) if summary["schema"] == "nemo.fabric.invocation_record_summary": return render_invocation_record_summary(summary, details=details) return render_result_summary(summary, details=details) def main() -> None: - """Inspect a saved invocation result as a concise summary or JSON.""" + """Inspect a saved run journal or legacy result as readable text or JSON.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("result", type=Path) parser.add_argument("--json", action="store_true", dest="as_json") diff --git a/tests/python/test_invocation_reporting.py b/tests/python/test_invocation_reporting.py index a1e640545..e16365bde 100644 --- a/tests/python/test_invocation_reporting.py +++ b/tests/python/test_invocation_reporting.py @@ -3,6 +3,7 @@ from copy import deepcopy import json +import re import pytest @@ -15,6 +16,11 @@ from nemo_fabric.types import RunResult +def table_value(text, label): + rows = [re.split(r"\s{2,}", row, maxsplit=1) for row in text.splitlines()] + return dict(row for row in rows if len(row) == 2)[label] + + @pytest.fixture(name="result") def result_fixture(): return { @@ -47,10 +53,10 @@ def test_one_off_result_needs_no_task_labels_or_benchmark_manifest(tmp_path, res summary = read_result(path) text = render_result_summary(summary) assert summary["status"] == "succeeded" - assert "input 100" in text - assert "reads 0" in text - assert "Provider-reported cost: $0.000000010 (complete)" in text - assert "Catalog estimate: unavailable" in text + assert table_value(text, "Input tokens") == "100" + assert table_value(text, "Cache read tokens") == "0" + assert table_value(text, "Provider-reported cost, USD") == "$0.000000010" + assert table_value(text, "Estimated cost, USD") == "Unavailable" assert summary["result_path"] == str(path) @@ -75,9 +81,11 @@ def test_partial_input_preserves_observations_and_independent_output(result): assert dimensions["input_tokens"]["observed_sum"] == 50 assert dimensions["output_tokens"]["complete_total"] == 2 text = render_result_summary(summary) - assert "input 50 (observed, incomplete)" in text - assert "output 2 (complete)" in text - assert "Some invocation-owned model work was unfinished at return." in text + assert table_value(text, "Input tokens") == "50 recorded; total unknown" + assert table_value(text, "Output tokens") == "2" + assert ( + "input tokens, total tokens: some model work was unfinished at return" in text + ) def test_legacy_usage_does_not_become_normalized_or_zero(result): @@ -128,7 +136,7 @@ def test_result_outcome_boundary_survives_the_reader(result): ] summary = summarize_result(result) assert summary["outcome_boundary"] == "one_shot_lifecycle" - assert 'Outcome boundary: "one_shot_lifecycle"' in render_result_summary( + assert "Outcome covers: request through runtime shutdown" in render_result_summary( summary, details=True ) @@ -215,7 +223,9 @@ def test_result_reads_only_matching_core_sibling_for_timing( summary = read_result(path) assert summary["execution_interval"]["elapsed_ms"] == 123 assert summary["execution_interval"]["scope"] == "core_invocation" - assert "before shutdown: 123 ms" in render_result_summary(summary) + assert "Core invocation interval: 0.123 s" in render_result_summary( + summary, details=True + ) assert summary["status"] == result["status"] record_lines[1]["runtime_id"] = "other" record.write_text("".join(json.dumps(row) + "\n" for row in record_lines)) @@ -267,7 +277,7 @@ def test_failed_result_shows_structured_error_without_losing_usage_or_dumping_me } assert "adapter_failure" in text assert "private model request details" not in text - assert "input 100" in text + assert table_value(text, "Input tokens") == "100" def test_foreign_runtime_snapshot_is_rejected_without_erasing_usage(result): @@ -340,18 +350,19 @@ def test_core_accounting_failure_event_is_visible_without_replacing_execution(re assert summary["model_usage"] is None assert summary["accounting_error"] == "invalid_model_usage" assert summary["legacy_usage"] == {"input_tokens": 100} - assert "accounting was invalid" in render_result_summary(summary) + assert "accounting is invalid" in render_result_summary(summary) def test_overview_separates_token_and_cost_coverage_and_keeps_exact_json(result): summary = summarize_result(result) text = render_result_summary(summary) - assert ( - "Token coverage: input/output/total complete; cache breakdown complete" in text - ) - assert "Tokens: input 100; output 2; total 102" in text - assert "Provider-reported cost: $0.000000010 (complete)" in text - assert "Catalog estimate: unavailable" in text + assert "Token coverage" not in text + assert text.count("Input tokens") == 1 + assert table_value(text, "Input tokens") == "100" + assert table_value(text, "Output tokens") == "2" + assert table_value(text, "Total tokens") == "102" + assert table_value(text, "Provider-reported cost, USD") == "$0.000000010" + assert table_value(text, "Estimated cost, USD") == "Unavailable" assert "Invocation:" not in text assert summary["usage"]["dimensions"]["reported_cost"]["complete_total"] == 10 @@ -364,10 +375,10 @@ def test_overview_names_missing_token_dimensions_without_hiding_complete_output( {"dimensions": ["input_tokens", "total_tokens"], "reason": "unfinished_work"} ) text = render_result_summary(summarize_result(result)) - assert "input 50 (observed, incomplete)" in text - assert "total unavailable" in text - assert "output 2 (complete)" in text - assert "Token coverage: input/output/total incomplete" in text + assert table_value(text, "Input tokens") == "50 recorded; total unknown" + assert table_value(text, "Total tokens") == "Unavailable" + assert table_value(text, "Output tokens") == "2" + assert "Token coverage" not in text assert "unfinished at return" in text @@ -378,7 +389,10 @@ def test_accounted_models_are_not_presented_as_served_model_provenance(result): {"provider": "anthropic", "model": "helper\x1b[2J"}, ] text = render_result_summary(summary) - assert 'Accounted models: "openai/main-model", "anthropic/helper\\u001b[2J"' in text + assert "Accounted models:" not in text + detail = render_result_summary(summary, details=True) + assert "openai/main-model" in detail + assert "anthropic/helper\\u001b[2J" in detail assert '"served":' not in text assert "\x1b" not in text @@ -410,12 +424,12 @@ def test_missing_cache_breakdown_does_not_label_complete_token_totals_incomplete } ) text = render_result_summary(summarize_result(result)) - assert "Tokens: input 100; output 2; total 102" in text - assert ( - "Token coverage: input/output/total complete; cache breakdown unavailable" - in text - ) - assert "Cache (included in input): reads unavailable; writes unavailable" in text + assert table_value(text, "Input tokens") == "100" + assert table_value(text, "Output tokens") == "2" + assert table_value(text, "Total tokens") == "102" + assert "Token coverage" not in text + assert table_value(text, "Cache read tokens") == "Unavailable" + assert table_value(text, "Cache write tokens") == "Unavailable" def test_catalog_identity_is_visible_without_dumping_the_basis(result): @@ -435,10 +449,11 @@ def test_catalog_identity_is_visible_without_dumping_the_basis(result): summary = summarize_result(result) overview = render_result_summary(summary) detail = render_result_summary(summary, details=True) - assert "Catalog estimate: $0.000280 (complete)" in overview - assert "Pricing basis: catalog; bundled aaaaaaaaaaaa" in overview + assert table_value(overview, "Estimated cost, USD") == "$0.000280" + assert "aaaaaaaaaaaa" not in overview + assert "Packaged table fingerprint: sha256:" + "a" * 64 in detail assert "applicable_tariff_no_missing_counts_v1" not in overview - assert "applicable_tariff_no_missing_counts_v1" in detail + assert "The estimate requires known token counts and applicable prices." in detail assert summary["usage"]["pricing"] == pricing @@ -472,8 +487,10 @@ def test_record_projection_retains_normalized_accounting_without_full_result( from nemo_fabric.reporting import render_invocation_record_summary text = render_invocation_record_summary(summary) - assert "input 50 (observed, incomplete)" in text if partial else "input 100" in text - assert "Invocation outcome before runtime shutdown: succeeded" in text + assert table_value(text, "Input tokens") == ( + "50 recorded; total unknown" if partial else "100" + ) + assert "does not establish the one-shot shutdown outcome" in text assert not (tmp_path / "run-result.json").exists() @@ -527,9 +544,9 @@ def test_ordinary_full_result_preserves_system_stats_without_recording(result): assert summary["system_stats"]["after"] == result["system_stats"]["after"] assert summary["resource_window"]["cpu_counter_change_seconds"] == 2 text = render_result_summary(summary, details=True) - assert "Elapsed (adapter exchange): 123 ms" in text - assert "not invocation CPU" in text - assert "System stats collected groups" in text + assert table_value(text, "Request execution time") == "0.123 s" + assert "Concurrent work can contribute" in text + assert "Measured processes: adapter host and its child processes" in text def test_process_stats_not_collected_never_implies_zero(result): @@ -561,10 +578,10 @@ def test_details_retain_resource_scope_while_overview_stays_compact(result): summary = summarize_result(result) overview = render_result_summary(summary) detail = render_result_summary(summary, details=True) - assert "Elapsed (adapter exchange): 123 ms" in overview + assert table_value(overview, "Request execution time") == "0.123 s" assert "process-tree" not in overview - assert "not invocation CPU" in detail - assert "not a peak" in detail + assert "Concurrent work can contribute" in detail + assert "Endpoint memory is not a peak or time average." in detail assert "Invocation:" in detail - assert "System stats collected groups" in detail + assert "Measured processes: adapter host and its child processes" in detail assert "bytes" not in detail # Human-readable memory uses binary units. diff --git a/tests/python/test_report_format.py b/tests/python/test_report_format.py index c1215a8c7..615f986bf 100644 --- a/tests/python/test_report_format.py +++ b/tests/python/test_report_format.py @@ -9,7 +9,7 @@ @pytest.mark.parametrize( ("value", "expected"), [ - (None, "unavailable"), + (None, "Unavailable"), (0, "$0.00"), (1, "$0.000000001"), (280000, "$0.000280"), @@ -25,11 +25,11 @@ def test_money_preserves_positive_spend_with_readable_precision(value, expected) @pytest.mark.parametrize( ("value", "expected"), [ - (0, "0 ms"), - (0.25, "0.25 ms"), - (6000, "6 s"), - (61000, "1 min 1 s"), - (3661000, "1 h 1 min 1 s"), + (0, "0.000 s"), + (0.25, "0.000 s"), + (6000, "6.000 s"), + (61000, "61.000 s"), + (3661000, "3661.000 s"), ], ) def test_duration_uses_explicit_readable_units(value, expected): diff --git a/tests/python/test_run_log_reporting.py b/tests/python/test_run_log_reporting.py new file mode 100644 index 000000000..adabd1cd3 --- /dev/null +++ b/tests/python/test_run_log_reporting.py @@ -0,0 +1,622 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from copy import deepcopy +import json + +import pytest + +from nemo_fabric.reporting import read_evidence, render_result_summary + + +@pytest.fixture(name="run_result") +def run_result_fixture(): + return { + "request_id": "request", + "runtime_id": "runtime", + "invocation_id": "invocation", + "status": "succeeded", + "model_usage": { + "input_tokens": 100, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + "total_tokens": 102, + "observed": {}, + "gaps": [], + "models": [], + "models_exhaustive": False, + "cost": {"reported": {"amount_nano_usd": 1}}, + }, + "artifacts": {"artifacts": []}, + } + + +@pytest.fixture(name="run_rows") +def run_rows_fixture(run_result): + def row(sequence, kind, seconds, data): + return { + "schema": "nemo.fabric.run", + "version": "v1alpha1", + "recording_id": "recording", + "request_id": "request", + "sequence": sequence, + "time": f"2026-10-25T19:01:{10 + int(seconds):02}.000Z", + "elapsed_seconds": seconds, + "source": "fabric", + "kind": kind, + "data": data, + **( + {"runtime_id": "runtime", "invocation_id": "invocation"} + if sequence + else {} + ), + } + + return [ + row( + 0, + "request_started", + 0, + { + "harness": "Claude Code", + "agent_name": "example", + "outcome_scope": "one_shot_lifecycle", + "models": { + "default": {"provider": "example", "model": "example-model"} + }, + }, + ), + row(1, "runtime_ready", 1, {"runtime_id": "runtime"}), + row(2, "invocation_started", 2, {"timeout_seconds": 300}), + row( + 3, + "invocation_finished", + 43, + {key: run_result[key] for key in ("status", "model_usage")}, + ), + row( + 4, + "summary", + 44, + { + "outcome_scope": "one_shot_lifecycle", + "result": run_result, + "error": None, + "request_started_at": "2026-10-25T19:01:10.000Z", + "request_finished_at": "2026-10-25T19:01:54.000Z", + "elapsed_seconds": 44, + }, + ), + ] + + +def write_log(path, rows): + path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + + +def test_run_summary_and_timeline_keep_same_utc_times_and_seconds(tmp_path, run_rows): + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["schema"] == "nemo.fabric.invocation_summary" + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + text = render_result_summary(summary) + assert "19:01:10.000" in text and "19:01:54.000" in text + assert "44.000 s" in text + assert "example-model" in text + assert "Token coverage" not in text and "Accounting: " not in text + assert "$0.000000001" in text + assert "2026-10-25T" not in text + details = render_result_summary(summary, details=True) + assert ( + "Execution history" in details and "Invocation finished: succeeded" in details + ) + assert "43.000" in details + assert details.count("Input tokens") == 1 + + +def test_cross_midnight_summary_keeps_each_date(tmp_path, run_rows): + run_rows[-1]["data"].update( + request_started_at="2026-10-25T23:59:59.000Z", + request_finished_at="2026-10-26T00:00:43.000Z", + ) + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + text = render_result_summary(read_evidence(path)) + assert "2026-10-25 23:59:59.000" in text + assert "2026-10-26 00:00:43.000" in text + + +def test_missing_final_summary_retains_invocation_accounting_without_outer_success( + tmp_path, run_rows +): + path = tmp_path / "run.jsonl" + write_log(path, run_rows[:-1]) + with path.open("a") as stream: + stream.write('{"kind":') + summary = read_evidence(path) + assert summary["schema"] == "nemo.fabric.invocation_record_summary" + assert summary["state"] == "record_only" + assert summary["truncated_tail"] + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + assert "Completion unconfirmed" in render_result_summary(summary) + + +def test_final_unknown_usage_is_not_repaired_from_invocation_event(tmp_path, run_rows): + result = deepcopy(run_rows[-1]["data"]["result"]) + result.pop("model_usage") + run_rows[-1]["data"]["result"] = result + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["model_usage"] is None + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] is None + + +def test_final_no_result_retains_error_and_independent_invocation_evidence( + tmp_path, run_rows +): + final = run_rows[-1]["data"] + final["result"] = None + final["error"] = { + "stage": "stop", + "code": "shutdown_failed", + "retryable": False, + "message": "private diagnostic", + } + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["schema"] == "nemo.fabric.request_summary" + assert summary["state"] == "no_result" + assert ( + summary["invocation_summary"]["usage"]["dimensions"]["input_tokens"][ + "complete_total" + ] + == 100 + ) + text = render_result_summary(summary) + assert "No result returned" in text + assert "shutdown_failed" in text and "stop" in text + assert "private diagnostic" not in text + + +@pytest.mark.parametrize( + "mutation", + [ + "sequence", + "identity", + "result_identity", + "duplicate_final", + "after_final", + "naive_time", + "negative_elapsed", + "reverse_elapsed", + "nonfinite", + "malformed_complete", + ], +) +def test_conflicting_or_malformed_complete_run_evidence_is_rejected( + tmp_path, run_rows, mutation +): + if mutation == "sequence": + run_rows[1]["sequence"] = 4 + elif mutation == "identity": + run_rows[2]["request_id"] = "other" + elif mutation == "result_identity": + run_rows[-1]["data"]["result"]["runtime_id"] = "other" + elif mutation in {"duplicate_final", "after_final"}: + row = deepcopy(run_rows[-1]) + row["sequence"] += 1 + if mutation == "after_final": + row["kind"] = "diagnostic" + run_rows.append(row) + elif mutation == "naive_time": + run_rows[1]["time"] = "2026-10-25T19:01:11" + elif mutation == "negative_elapsed": + run_rows[0]["elapsed_seconds"] = -1 + elif mutation == "reverse_elapsed": + run_rows[3]["elapsed_seconds"] = 0 + elif mutation == "nonfinite": + run_rows[1]["elapsed_seconds"] = float("inf") + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + if mutation == "malformed_complete": + with path.open("a") as stream: + stream.write('{"kind":\n') + with pytest.raises(ValueError): + read_evidence(path) + + +def test_journal_streams_files_larger_than_one_result_limit(tmp_path, run_rows): + path = tmp_path / "run.jsonl" + rows = run_rows[:-1] + for sequence in range(4, 304): + rows.append( + { + **run_rows[0], + "sequence": sequence, + "elapsed_seconds": 43, + "kind": "resource_sample", + "data": {"diagnostic": "x" * 60_000}, + } + ) + final = deepcopy(run_rows[-1]) + final["sequence"] = len(rows) + rows.append(final) + write_log(path, rows) + assert path.stat().st_size > 16 * 1024 * 1024 + summary = read_evidence(path) + assert summary["terminal_summary_present"] + assert all(row["kind"] != "resource_sample" for row in summary["history"]) + + +def test_oversized_record_is_explicit_error_without_modifying_file(tmp_path, run_rows): + path = tmp_path / "run.jsonl" + run_rows[-1]["data"]["result"]["output"] = "x" * (16 * 1024 * 1024) + write_log(path, run_rows) + original_size = path.stat().st_size + with pytest.raises(ValueError, match="per-record limit"): + read_evidence(path) + assert path.stat().st_size == original_size + + +def test_diagnostics_and_names_cannot_inject_terminal_controls(tmp_path, run_rows): + run_rows[0]["data"]["harness"] = "agent\x1b[2J\nforged" + diagnostic = { + **run_rows[2], + "sequence": 3, + "kind": "diagnostic", + "data": {"message": "message\x1b[2J\nforged"}, + } + run_rows.insert(3, diagnostic) + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + text = render_result_summary(read_evidence(path), details=True) + assert "\x1b" not in text + assert "\\u001b[2J\\nforged" in text + + +@pytest.mark.parametrize( + "models", + ["invalid", {"default": "invalid"}, {"default": {"model": "missing-provider"}}], +) +def test_malformed_request_model_shapes_are_rejected(tmp_path, run_rows, models): + run_rows[0]["data"]["models"] = models + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + with pytest.raises(ValueError, match="model|provider"): + read_evidence(path) + + +@pytest.mark.parametrize("seconds", [True, 10**400]) +def test_invalid_elapsed_values_have_explicit_validation_errors( + tmp_path, run_rows, seconds +): + run_rows[1]["elapsed_seconds"] = seconds + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + with pytest.raises(ValueError, match="seconds"): + read_evidence(path) + + +def test_history_does_not_duplicate_terminal_result_or_full_diagnostics( + tmp_path, run_rows +): + run_rows[-1]["data"]["result"]["output"] = {"text": "x" * 100000} + diagnostic = { + **run_rows[2], + "sequence": 3, + "kind": "diagnostic", + "data": {"message": "m" * 5000}, + } + run_rows.insert(3, diagnostic) + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert "result" not in summary["history"][-1]["data"] + assert "model_usage" not in summary["history"][-2]["data"] + message = summary["history"][3]["data"] + assert len(message["message"]) == 4096 + assert message["message_truncated"] + assert len(json.dumps(summary["history"])) < 10000 + assert "message shortened" in render_result_summary(summary, details=True) + + +def test_pricing_details_show_actual_rates_and_basis_without_overview_noise(run_result): + from nemo_fabric.reporting import summarize_result + + fingerprint = "sha256:" + "a" * 64 + cost = run_result["model_usage"]["cost"] + cost["pricing"] = { + "basis": "catalog", + "currency": "USD", + "normalization_version": "cache_reasoning_inclusive_v1", + "assumption_policy": "applicable_tariff_no_missing_counts_v1", + "bundled_catalog_sha256": fingerprint, + } + cost["estimated"] = {"amount_nano_usd": 1000} + cost["applied_rates"] = [ + { + "catalog_sha256": fingerprint, + "provider": "example", + "model_id": "example-model", + "pricing_as_of": "2026-10-25", + "pricing_source": "https://example.com/pricing", + "rates": { + "input": 5_000_000_000, + "output": 8_000_000_000, + "cache_read": 1_000_000_000, + }, + } + ] + summary = summarize_result(run_result) + overview = render_result_summary(summary) + details = render_result_summary(summary, details=True) + assert "sha256" not in overview + assert "USD per 1,000,000 tokens" in details + assert "Uncached input: $5.00" in details + assert "Cache reads: $1.00" in details + assert "Output: $8.00" in details + assert "2026-10-25" in details and "https://example.com/pricing" in details + + +def test_recorded_provider_cost_does_not_claim_to_be_an_estimate(run_result): + from nemo_fabric.reporting import summarize_result + + cost = run_result["model_usage"]["cost"]["reported"] + cost.update(amount_nano_usd=None, observed_nano_usd=1) + run_result["model_usage"]["gaps"] = [ + {"dimensions": ["reported_cost"], "reason": "unfinished_work"} + ] + text = render_result_summary(summarize_result(run_result)) + provider_row = next( + line for line in text.splitlines() if line.startswith("Provider-reported cost") + ) + assert "$0.000000001 recorded; total unknown" in provider_row + assert "estimate" not in provider_row + + +def test_runtime_snapshot_supplies_memory_but_never_cumulative_cpu(run_result): + from nemo_fabric.reporting import summarize_result + + run_result["resource_snapshot"] = { + "runtime_id": "runtime", + "started_at": "2026-10-25T19:01:10Z", + "observed_at": "2026-10-25T19:01:54Z", + "resident_memory_bytes": 1024, + "process_count": 1, + "cpu_time_seconds": 100, + } + text = render_result_summary(summarize_result(run_result)) + memory = next( + line for line in text.splitlines() if line.startswith("Memory after execution") + ) + cpu = next( + line for line in text.splitlines() if line.startswith("CPU time, estimated") + ) + assert "1.0 KiB" in memory + assert "Unavailable" in cpu + + +def test_multiple_nondefault_roles_are_not_presented_as_one_model(tmp_path, run_rows): + run_rows[0]["data"]["models"] = { + "planner": {"provider": "example", "model": "planner-model"}, + "coder": {"provider": "example", "model": "coder-model"}, + } + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + text = render_result_summary(summary) + assert "Multiple roles (2)" in text + assert "planner-model" not in text and "coder-model" not in text + details = render_result_summary(summary, details=True) + assert "planner: requested planner-model" in details + assert "coder: requested coder-model" in details + + +def test_unretained_model_override_does_not_turn_config_into_requested_fact( + tmp_path, run_rows +): + run_rows[0]["data"]["override_keys"] = ["models"] + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + text = render_result_summary(read_evidence(path)) + assert "Unavailable (model override not retained)" in text + assert "example-model" not in text + + +@pytest.mark.parametrize("keys", [None, 1, [None]]) +def test_malformed_override_key_lists_are_rejected(tmp_path, run_rows, keys): + run_rows[0]["data"]["override_keys"] = keys + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + with pytest.raises(ValueError, match="override_keys"): + read_evidence(path) + + +def test_shutdown_failure_is_visible_even_after_successful_invocation( + tmp_path, run_rows +): + error = {"stage": "stop", "code": "shutdown_failed", "retryable": False} + run_rows.insert( + -1, + { + **run_rows[-1], + "kind": "runtime_stopped", + "data": {"succeeded": False, "error": error}, + }, + ) + run_rows[-1]["data"]["result"].update(status="failed", error=error) + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + text = render_result_summary(read_evidence(path), details=True) + assert "Invocation finished: succeeded" in text + assert "Agent runtime shutdown failed: stop/shutdown_failed" in text + assert "Execution:" in text and "Failed" in text + + +def test_diagnostic_retention_does_not_hide_terminal_milestones(tmp_path, run_rows): + diagnostics = [ + {**run_rows[2], "kind": "diagnostic", "data": {"message": "progress"}} + for _ in range(300) + ] + run_rows[3:3] = diagnostics + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["history_omitted_count"] > 0 + assert len(summary["history"]) < 265 + text = render_result_summary(summary, details=True) + assert "Invocation finished: succeeded" in text + assert "Additional history entries retained in file:" in text + + +def test_streamed_history_has_byte_budget_and_keeps_terminal_evidence( + tmp_path, run_rows +): + diagnostics = [ + { + **run_rows[2], + "kind": "diagnostic", + "data": { + "message": "\U0001f680" * 4096, + "level": "L" * 4000, + "severity": "S" * 4000, + "stream": "stderr", + "scope": "runtime", + }, + } + for _ in range(40) + ] + # Future kinds keep their envelope; their unknown payload is not copied. + future = { + **run_rows[2], + "kind": "future_event", + "data": {"status": "X" * (2 * 1024 * 1024)}, + } + run_rows[3:3] = [future, *diagnostics] + stopped = { + **run_rows[-2], + "kind": "runtime_stopped", + "data": {"succeeded": False, "error": {"stage": "stop", "code": "E" * 100_000}}, + } + run_rows.insert(-1, stopped) + final_result = run_rows[-1]["data"]["result"] + final_result["status"] = "failed" + final_result["error"] = {"stage": "stop", "code": "shutdown_failed"} + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert len(json.dumps(summary["history"]).encode("utf-8")) <= 1024 * 1024 + assert summary["history_omitted_count"] > 0 + assert summary["history_projection_truncated_count"] > 0 + assert summary["status"] == "failed" + assert summary["execution_error"]["code"] == "shutdown_failed" + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + assert [row["kind"] for row in summary["history"]][-3:] == [ + "invocation_finished", + "runtime_stopped", + "summary", + ] + future_projection = next( + row for row in summary["history"] if row["kind"] == "future_event" + ) + assert future_projection["data"] == {} + stopped_projection = summary["history"][-2] + assert len(stopped_projection["data"]["error"]["code"]) == 256 + text = render_result_summary(summary, details=True) + assert "history details shortened; full event retained in file" in text + assert "Agent runtime shutdown failed" in text + + +def test_diagnostic_retention_sums_markers_after_the_display_history_is_full( + tmp_path, run_rows +): + events = [ + { + **run_rows[2], + "kind": "diagnostic", + "data": {"stream": "stderr", "scope": "runtime", "message": "progress"}, + } + for _ in range(260) + ] + events.extend( + { + **run_rows[2], + "kind": "diagnostic_retention", + "data": {"omitted_bytes": count, "read_error": error}, + } + for count, error in ((5, None), (6, "read_failed")) + ) + run_rows[3:3] = events + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + path = tmp_path / "run.jsonl" + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["diagnostics"] == { + "omitted_bytes": 11, + "read_errors": ["read_failed"], + } + assert summary["history_omitted_count"] > 0 + assert summary["terminal_summary_present"] + assert summary["usage"]["dimensions"]["input_tokens"]["complete_total"] == 100 + text = render_result_summary(summary) + assert "Diagnostic capture omitted 11 bytes." in text + assert "Diagnostic capture error: read_failed" in text + assert "diagnostic retention" not in render_result_summary(summary, details=True) + + +def test_diagnostic_retention_errors_are_bounded_and_zero_loss_is_quiet( + tmp_path, run_rows +): + from nemo_fabric.reporting import MAX_HISTORY_MESSAGE_CHARS + + markers = [ + { + **run_rows[2], + "kind": "diagnostic_retention", + "data": {"omitted_bytes": 0, "read_error": None}, + } + ] + path = tmp_path / "run.jsonl" + run_rows[3:3] = markers + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["diagnostics"] == {"omitted_bytes": 0, "read_errors": []} + assert "Diagnostic capture" not in render_result_summary(summary) + + markers = [ + { + **run_rows[2], + "kind": "diagnostic_retention", + "data": { + "omitted_bytes": 0, + "read_error": "e" * (MAX_HISTORY_MESSAGE_CHARS + 1), + }, + } + for _ in range(17) + ] + run_rows[3:3] = markers + for sequence, row in enumerate(run_rows): + row["sequence"] = sequence + write_log(path, run_rows) + summary = read_evidence(path) + assert summary["diagnostics"]["omitted_bytes"] == 0 + assert ( + summary["diagnostics"]["read_errors"] == ["e" * MAX_HISTORY_MESSAGE_CHARS] * 16 + ) From 4c0dbc0039a4e13fbfa93af57170b59b127e76c5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 08:24:58 -0700 Subject: [PATCH 151/159] docs: qualify normalized model usage availability Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 724418569..31c84a57a 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1217,11 +1217,7 @@ the SDK raises a `FabricError` subclass instead of returning a partial ### Complete counts and useful observations -`result.model_usage` is present when the adapter supplies the declared -`nemo_fabric.usage_accounting.v1` evidence. Its token fields are independently -nullable counts: input includes cache reads and writes, output includes -reasoning, and total requires known input plus output. The independent raw -provider total remains in `result.usage`. +`result.model_usage` is present when the adapter supplies valid `nemo_fabric.usage_accounting.v1` evidence and NeMo Fabric successfully normalizes it. Its token fields are independently nullable counts: input includes cache reads and writes, output includes reasoning, and total requires known input plus output. The independent raw provider total remains in `result.usage`. Typed access to `model_usage` requires an updated SDK. Permissive readers can retain or ignore the new field; closed validators must use the updated result schema. Current readers still accept older results without this field. From b30b122cec9be7c1bef791353fbcde292c1c41a4 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 15:21:29 -0700 Subject: [PATCH 152/159] fix: enforce usage accounting parity across bindings Signed-off-by: mnajafian-nv --- .github/workflows/ci_python.yml | 16 +++++++ .../nemo_fabric_adapter_contract/models.py | 9 ++++ .../adapter_contract/test_agent_execution.py | 47 +++++++++++++++++++ tests/python/test_usage_reducer_parity.py | 3 ++ 4 files changed, 75 insertions(+) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index d73bbce66..633e34d54 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -84,6 +84,14 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + - name: Set up Node.js for reducer parity + if: matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: 24 + cache: npm + cache-dependency-path: adapter-contract/typescript/package-lock.json + - name: Cache cargo build uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -100,7 +108,15 @@ jobs: run: | uv sync --no-default-groups --group adapter-tests --group collector --group test --extra claude --extra codex --extra deepagents --extra mini-swe-agent --extra openclaw --extra openhands --extra remote-agent ${{ (matrix.python-version == '3.12' || matrix.python-version == '3.13') && '--extra nooa' || '' }} ${{ matrix.python-version != '3.11' && '--extra harbor' || '' }} --extra relay + - name: Build TypeScript contract for reducer parity + if: matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' + run: | + just install-typescript-contract + npm run build --prefix adapter-contract/typescript + - name: Run pytest + env: + FABRIC_REQUIRE_TYPESCRIPT_REDUCER_PARITY: ${{ matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' && '1' || '0' }} run: | set -euo pipefail just --set no_uv true test-python diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py index 68104e1ab..1280e2a93 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py @@ -637,6 +637,15 @@ class UsageAccounting(ContractModel): models: list[AccountedModel] = _empty_list() def _validate(self) -> None: + if self.models_exhaustive and not self.models and self.observed is not None: + for dimension in UsageDimension: + if dimension is UsageDimension.COST_USD: + continue + count = getattr(self.observed, dimension.value) + if count is not None and count > 0: + raise ContractValidationError( + "exhaustive empty model list conflicts with positive tokens" + ) identities = [(model.provider, model.model) for model in self.models] if len(set(identities)) != len(identities): raise ContractValidationError("duplicate model identity") diff --git a/tests/adapter_contract/test_agent_execution.py b/tests/adapter_contract/test_agent_execution.py index d0e706480..89aa75cd8 100644 --- a/tests/adapter_contract/test_agent_execution.py +++ b/tests/adapter_contract/test_agent_execution.py @@ -22,6 +22,7 @@ from nemo_fabric_adapter_contract.models import EnvironmentHandle from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapter_contract.models import RuntimeTelemetryContext +from nemo_fabric_adapter_contract.models import UsageAccounting from nemo_fabric_adapter_contract.codec import ContractValidationError from nemo_fabric_adapter_contract.pydantic_support import type_adapter @@ -54,6 +55,52 @@ def test_usage_rejects_invalid_cache_semantics(value): AgentUsage(input_tokens_include_cache=value) +@pytest.mark.parametrize( + "field", + [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "reasoning_tokens", + "total_tokens", + "peak_request_input_tokens", + ], +) +def test_exhaustive_empty_models_reject_positive_observed_tokens(field): + with pytest.raises(ContractValidationError, match="exhaustive empty model list"): + UsageAccounting.from_mapping( + { + "models_exhaustive": True, + "models": [], + "gaps": [], + "observed": {field: 1}, + } + ) + + +@pytest.mark.parametrize("observed", [{}, {"input_tokens": 0}, {"cost_usd": 0.25}]) +def test_exhaustive_empty_models_preserve_zero_unknown_and_cost(observed): + accounting = UsageAccounting.from_mapping( + {"models_exhaustive": True, "models": [], "gaps": [], "observed": observed} + ) + assert accounting.observed is not None + assert accounting.observed.to_mapping() == observed + + +def test_nonexhaustive_empty_models_can_observe_positive_tokens(): + accounting = UsageAccounting.from_mapping( + { + "models_exhaustive": False, + "models": [], + "gaps": [], + "observed": {"input_tokens": 1}, + } + ) + assert accounting.observed is not None + assert accounting.observed.input_tokens == 1 + + def test_agent_run_request_contains_only_southbound_request_fields(): request = AgentRunRequest( input={"messages": [{"role": "user", "content": "hello"}]}, diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py index ccdf1628c..4dea09d37 100644 --- a/tests/python/test_usage_reducer_parity.py +++ b/tests/python/test_usage_reducer_parity.py @@ -8,6 +8,7 @@ """ import json +import os from pathlib import Path import shutil import subprocess @@ -71,6 +72,8 @@ def test_python_and_typescript_reducers_emit_the_same_wire_evidence(reduced): node = shutil.which("node") module = ROOT / "adapter-contract/typescript/dist/usage.js" if node is None or not module.exists(): + if os.environ.get("FABRIC_REQUIRE_TYPESCRIPT_REDUCER_PARITY") == "1": + pytest.fail("TypeScript reducer parity requires Node and the built contract") pytest.skip("build the TypeScript contract to run cross-binding reducer parity") source = """ import { pathToFileURL } from 'node:url'; From 87568beefa550c5ff4de15e6c647b45d4e878741 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 15:32:34 -0700 Subject: [PATCH 153/159] fix: enforce usage accounting parity across bindings Signed-off-by: mnajafian-nv --- .github/workflows/ci_python.yml | 16 +++++++ .../nemo_fabric_adapter_contract/models.py | 9 ++++ .../adapter_contract/test_agent_execution.py | 47 +++++++++++++++++++ tests/python/test_usage_reducer_parity.py | 3 ++ 4 files changed, 75 insertions(+) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index d73bbce66..633e34d54 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -84,6 +84,14 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8 + - name: Set up Node.js for reducer parity + if: matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: 24 + cache: npm + cache-dependency-path: adapter-contract/typescript/package-lock.json + - name: Cache cargo build uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -100,7 +108,15 @@ jobs: run: | uv sync --no-default-groups --group adapter-tests --group collector --group test --extra claude --extra codex --extra deepagents --extra mini-swe-agent --extra openclaw --extra openhands --extra remote-agent ${{ (matrix.python-version == '3.12' || matrix.python-version == '3.13') && '--extra nooa' || '' }} ${{ matrix.python-version != '3.11' && '--extra harbor' || '' }} --extra relay + - name: Build TypeScript contract for reducer parity + if: matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' + run: | + just install-typescript-contract + npm run build --prefix adapter-contract/typescript + - name: Run pytest + env: + FABRIC_REQUIRE_TYPESCRIPT_REDUCER_PARITY: ${{ matrix.platform == 'linux-amd64' && matrix.python-version == '3.12' && '1' || '0' }} run: | set -euo pipefail just --set no_uv true test-python diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py index 68104e1ab..1280e2a93 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py @@ -637,6 +637,15 @@ class UsageAccounting(ContractModel): models: list[AccountedModel] = _empty_list() def _validate(self) -> None: + if self.models_exhaustive and not self.models and self.observed is not None: + for dimension in UsageDimension: + if dimension is UsageDimension.COST_USD: + continue + count = getattr(self.observed, dimension.value) + if count is not None and count > 0: + raise ContractValidationError( + "exhaustive empty model list conflicts with positive tokens" + ) identities = [(model.provider, model.model) for model in self.models] if len(set(identities)) != len(identities): raise ContractValidationError("duplicate model identity") diff --git a/tests/adapter_contract/test_agent_execution.py b/tests/adapter_contract/test_agent_execution.py index d0e706480..89aa75cd8 100644 --- a/tests/adapter_contract/test_agent_execution.py +++ b/tests/adapter_contract/test_agent_execution.py @@ -22,6 +22,7 @@ from nemo_fabric_adapter_contract.models import EnvironmentHandle from nemo_fabric_adapter_contract.models import RuntimeContext from nemo_fabric_adapter_contract.models import RuntimeTelemetryContext +from nemo_fabric_adapter_contract.models import UsageAccounting from nemo_fabric_adapter_contract.codec import ContractValidationError from nemo_fabric_adapter_contract.pydantic_support import type_adapter @@ -54,6 +55,52 @@ def test_usage_rejects_invalid_cache_semantics(value): AgentUsage(input_tokens_include_cache=value) +@pytest.mark.parametrize( + "field", + [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "reasoning_tokens", + "total_tokens", + "peak_request_input_tokens", + ], +) +def test_exhaustive_empty_models_reject_positive_observed_tokens(field): + with pytest.raises(ContractValidationError, match="exhaustive empty model list"): + UsageAccounting.from_mapping( + { + "models_exhaustive": True, + "models": [], + "gaps": [], + "observed": {field: 1}, + } + ) + + +@pytest.mark.parametrize("observed", [{}, {"input_tokens": 0}, {"cost_usd": 0.25}]) +def test_exhaustive_empty_models_preserve_zero_unknown_and_cost(observed): + accounting = UsageAccounting.from_mapping( + {"models_exhaustive": True, "models": [], "gaps": [], "observed": observed} + ) + assert accounting.observed is not None + assert accounting.observed.to_mapping() == observed + + +def test_nonexhaustive_empty_models_can_observe_positive_tokens(): + accounting = UsageAccounting.from_mapping( + { + "models_exhaustive": False, + "models": [], + "gaps": [], + "observed": {"input_tokens": 1}, + } + ) + assert accounting.observed is not None + assert accounting.observed.input_tokens == 1 + + def test_agent_run_request_contains_only_southbound_request_fields(): request = AgentRunRequest( input={"messages": [{"role": "user", "content": "hello"}]}, diff --git a/tests/python/test_usage_reducer_parity.py b/tests/python/test_usage_reducer_parity.py index ccdf1628c..4dea09d37 100644 --- a/tests/python/test_usage_reducer_parity.py +++ b/tests/python/test_usage_reducer_parity.py @@ -8,6 +8,7 @@ """ import json +import os from pathlib import Path import shutil import subprocess @@ -71,6 +72,8 @@ def test_python_and_typescript_reducers_emit_the_same_wire_evidence(reduced): node = shutil.which("node") module = ROOT / "adapter-contract/typescript/dist/usage.js" if node is None or not module.exists(): + if os.environ.get("FABRIC_REQUIRE_TYPESCRIPT_REDUCER_PARITY") == "1": + pytest.fail("TypeScript reducer parity requires Node and the built contract") pytest.skip("build the TypeScript contract to run cross-binding reducer parity") source = """ import { pathToFileURL } from 'node:url'; From ff89776210b7cb38de5745fe98ca81ee02e98402 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 16:20:26 -0700 Subject: [PATCH 154/159] fix: preserve runtime version and Claude turn attribution Signed-off-by: mnajafian-nv --- adapters/python/claude/README.md | 3 + .../nemo_fabric_adapters/claude/adapter.py | 93 +++++++++++++++---- .../nemo_fabric_adapters/common/provenance.py | 4 +- .../src/nemo_fabric_adapters/nooa/adapter.py | 3 +- .../nooa/bench_adapter.py | 3 +- .../nemo_fabric_adapters/openclaw/adapter.py | 3 +- .../nemo_fabric_adapters/openhands/adapter.py | 3 +- .../remote_agent/adapter.py | 3 +- docs/sdk/python.mdx | 2 +- .../test_adapters_common_provenance.py | 3 +- tests/adapters/test_claude_adapter.py | 85 +++++++++++++++++ tests/adapters/test_openclaw.py | 17 ++++ 12 files changed, 197 insertions(+), 25 deletions(-) diff --git a/adapters/python/claude/README.md b/adapters/python/claude/README.md index bfa13847c..dbd76d1ef 100644 --- a/adapters/python/claude/README.md +++ b/adapters/python/claude/README.md @@ -135,6 +135,9 @@ by the SDK and is not persisted as a NeMo Fabric artifact. The result's `usage` carries Claude Code's `input_tokens`, `output_tokens`, and reported `cost_usd`. Claude Code counts tokens for its main agent loop only: its usage leaves out Task subagent, sidechain, and auxiliary model calls. +If Claude Code handles an injected turn during a Fabric invocation, the adapter +waits for the caller's result. It leaves the invocation's complete token counts +and cost unknown because it cannot prove that the totals exclude the injected work. Claude Code's `input_tokens` excludes cache reads and cache writes, so the adapter also returns the `nemo_fabric.detailed_token_usage.v1` usage extension, diff --git a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py index 30d1972c8..1652da6fe 100644 --- a/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py +++ b/adapters/python/claude/src/nemo_fabric_adapters/claude/adapter.py @@ -33,6 +33,7 @@ from claude_agent_sdk import ProcessError from claude_agent_sdk import ResultMessage from claude_agent_sdk import SystemMessage +from claude_agent_sdk import UserMessage from claude_agent_sdk import HookMatcher from claude_agent_sdk._errors import MessageParseError from nemo_fabric_adapter_contract.codec import ContractValidationError @@ -842,12 +843,53 @@ def sdk_failure(error: BaseException) -> dict[str, Any]: _SYNTHETIC_MODEL = "" +def _injected_turn(message: UserMessage | ResultMessage) -> bool: + """Whether the SDK attributes this turn to a source other than the caller.""" + + origin = message.origin + return origin is not None and ( + not isinstance(origin, dict) or origin.get("kind") != "human" + ) + + @dataclass class InvocationMessages: - """One invocation's messages, and why collecting them failed, if it did.""" + """Caller-owned messages and runtime initialization facts for one invocation.""" items: list[Message] = field(default_factory=list) failure: str | None = None + skipped_injected_result: bool = False + _injected: bool = field(default=False, init=False) + + def observe(self, message: Message) -> ResultMessage | None: + """Collect owned evidence and return only the caller's terminal result.""" + + if ( + isinstance(message, UserMessage) + and message.parent_tool_use_id is None + and message.origin is not None + ): + self._injected = _injected_turn(message) + if isinstance(message, ResultMessage): + was_injected = self._injected + self._injected = False + if _injected_turn(message): + self.skipped_injected_result = True + if not was_injected: + # Without a user echo, preceding model messages have no + # reliable owner. Keep runtime initialization evidence. + self.items[:] = [ + item + for item in self.items + if isinstance(item, SystemMessage) and item.subtype == "init" + ] + return None + return message + if not self._injected or ( + isinstance(message, SystemMessage) and message.subtype == "init" + ): + self.items.append(message) + return None def served_model_fact(messages: InvocationMessages | None) -> dict[str, Any]: @@ -955,6 +997,7 @@ def _detailed_token_usage(usage: dict[str, Any]) -> dict[str, int | bool]: def _agent_run_result(output: dict[str, Any]) -> AgentRunResult: normalized = dict(output) + usage_unattributed = normalized.pop("_usage_unattributed", False) failed = bool(normalized.pop("failed", False)) reported_error = normalized.pop("error", None) error = None @@ -967,7 +1010,7 @@ def _agent_run_result(output: dict[str, Any]) -> AgentRunResult: retryable=bool(reported.get("retryable", False)), extensions=metadata if isinstance(metadata, dict) else {}, ) - raw_usage = normalized.get("usage") + raw_usage = None if usage_unattributed else normalized.get("usage") usage = raw_usage if isinstance(raw_usage, dict) else {} tokens = { name: value @@ -1332,23 +1375,25 @@ async def _run_query( ) -> dict[str, Any]: """Run one SDK query and normalize its terminal result. - Every message other than the terminal result is appended to - ``collected``, which also records why the stream failed, if it did. + The same deadline bounds all response reads, including injected turns. """ collected = InvocationMessages() if collected is None else collected self._invocation_messages = collected - messages = collected.items result: ResultMessage | None = None deadline = asyncio.timeout(remaining_timeout) try: async with deadline: await client.query(prompt) - async for message in client.receive_response(): - if isinstance(message, ResultMessage): - result = message - else: - messages.append(message) + while result is None: + received = False + async for message in client.receive_response(): + received = True + terminal = collected.observe(message) + if terminal is not None: + result = terminal + if not received: + break except (TimeoutError, ClaudeSDKError) as error: self._unusable = True collected.failure = "timeout" if deadline.expired() else "no_result" @@ -1365,7 +1410,7 @@ async def _run_query( if result is None or not _result_failed(result): raise LOGGER.exception("Claude SDK stream raised after a failed terminal result") - output = self._normalize_invocation(messages, result) + output = self._normalize_invocation(collected, result) else: if result is None: self._unusable = True @@ -1374,17 +1419,25 @@ async def _run_query( "claude_missing_result", "Claude returned no terminal result" ) else: - output = self._normalize_invocation(messages, result) + output = self._normalize_invocation(collected, result) return output def _normalize_invocation( self, - messages: list[Message], + collected: InvocationMessages, result: ResultMessage, ) -> dict[str, Any]: try: - output = normalize_result(messages, result) - output["cost_usd"] = self._invocation_cost(messages, result) + output = normalize_result(collected.items, result) + output["cost_usd"] = self._invocation_cost( + collected.items, + result, + skipped_injected_result=collected.skipped_injected_result, + ) + if collected.skipped_injected_result: + # The terminal usage may include or omit work before the + # injected turn, so it cannot establish a complete total. + output["_usage_unattributed"] = True if not output["failed"]: invalid_session = _validate_result_session( self._claude_session_id, result @@ -1402,6 +1455,8 @@ def _invocation_cost( self, messages: list[Message], result: ResultMessage, + *, + skipped_injected_result: bool = False, ) -> float | None: """Convert Claude Code's running cost total into this invocation's cost. @@ -1427,7 +1482,13 @@ def _invocation_cost( reset = any( isinstance(message, ConversationResetMessage) for message in messages ) - if total is None or previous is None or reset or total < previous: + if ( + skipped_injected_result + or total is None + or previous is None + or reset + or total < previous + ): return None # Drop float subtraction noise such as 0.19999999999999998. A tenth of a # nanodollar is far below the price of a single token. diff --git a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py index b8db9a95d..3cbbdc026 100644 --- a/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py +++ b/adapters/python/common/src/nemo_fabric_adapters/common/provenance.py @@ -165,7 +165,7 @@ def versions_report( def package_version_report( - distribution: str, model_roles: Iterable[str] + package_version: Mapping[str, Any], model_roles: Iterable[str] ) -> dict[str, Any]: """The report of an adapter that has a source only for its package version. @@ -174,7 +174,7 @@ def package_version_report( """ return versions_report( - package_version=package_version_fact(distribution), + package_version=package_version, harness_name=no_source_fact(), harness_version=no_source_fact(), served_models={role: no_source_fact() for role in model_roles}, diff --git a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py index 670e075fd..731f3ab2e 100644 --- a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py +++ b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/adapter.py @@ -526,6 +526,7 @@ def __init__(self) -> None: self._models: dict[str, Any] = {} self._telemetry: RelayTelemetry | None = None self._model_roles: list[str] = [] + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: if self._target is not None: @@ -596,7 +597,7 @@ async def invoke( result = await self._invoke(request, runtime_context) result.extensions[provenance.VERSIONS_REPORT_KEY] = ( - provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + provenance.package_version_report(self._package_version, self._model_roles) ) return result diff --git a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py index a4175e94a..26842bdcf 100644 --- a/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py +++ b/adapters/python/nooa/src/nemo_fabric_adapters/nooa/bench_adapter.py @@ -205,6 +205,7 @@ def __init__(self) -> None: self._instruction: str | None = None self._telemetry: RelayTelemetry | None = None self._model_roles: list[str] = [] + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: if self._bench_agent_factory is not None: @@ -270,7 +271,7 @@ async def invoke( result = await self._invoke(request, runtime_context) result.extensions[provenance.VERSIONS_REPORT_KEY] = ( - provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + provenance.package_version_report(self._package_version, self._model_roles) ) return result diff --git a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py index 9f05edd16..e026f7064 100644 --- a/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py +++ b/adapters/python/openclaw/src/nemo_fabric_adapters/openclaw/adapter.py @@ -809,6 +809,7 @@ def __init__(self) -> None: self._harness_version = provenance.missing_fact( "unavailable", "not_collected", _METHOD_VERSION_PROBE ) + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> dict[str, Any] | None: config: contract.AgentConfig = payload["config"] @@ -1452,7 +1453,7 @@ def _versions_extensions(self) -> dict[str, Any]: models = self._config.models if self._config is not None else {} return { provenance.VERSIONS_REPORT_KEY: provenance.versions_report( - package_version=provenance.package_version_fact(_PACKAGE_DISTRIBUTION), + package_version=self._package_version, harness_name=provenance.known_fact( _HARNESS_NAME, provenance.METHOD_ADAPTER_REPORT ), diff --git a/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py b/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py index 646192bb5..c9867fc05 100644 --- a/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py +++ b/adapters/python/openhands/src/nemo_fabric_adapters/openhands/adapter.py @@ -337,6 +337,7 @@ def __init__(self) -> None: self._profile_store_temp: tempfile.TemporaryDirectory[str] | None = None self._runtime_id: str | None = None self._model_roles: list[str] = [] + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] @@ -411,7 +412,7 @@ async def invoke( result = await self._invoke(request, context) result.extensions[provenance.VERSIONS_REPORT_KEY] = ( - provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + provenance.package_version_report(self._package_version, self._model_roles) ) return result diff --git a/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py b/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py index 9425fec36..58fbe91e4 100644 --- a/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py +++ b/adapters/python/remote-agent/src/nemo_fabric_adapters/remote_agent/adapter.py @@ -131,6 +131,7 @@ def __init__(self) -> None: self._messages: list[dict[str, str]] = [] self._relay_streaming = False self._model_roles: list[str] = [] + self._package_version = provenance.package_version_fact(_PACKAGE_DISTRIBUTION) async def start(self, payload: dict[str, Any]) -> None: config: contract.AgentConfig = payload["config"] @@ -220,7 +221,7 @@ async def invoke( result = await self._invoke(request, context) result.extensions[provenance.VERSIONS_REPORT_KEY] = ( - provenance.package_version_report(_PACKAGE_DISTRIBUTION, self._model_roles) + provenance.package_version_report(self._package_version, self._model_roles) ) return result diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index fc4a6d892..092e1cb98 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1308,7 +1308,7 @@ adapters also report their harness name and version. Other adapters that emit a versions report mark those harness facts `unsupported` with reason `no_source`. The Claude Code adapter also reports the served model of the role it runs: the one model that this invocation's main-thread assistant messages name, skipping subagent messages and Claude Code's `` API-error -messages. Two models give `unavailable` with reason `not_attributed`, and none +messages. The adapter also excludes messages from injected turns. Two models give `unavailable` with reason `not_attributed`, and none gives `unavailable` with reason `empty_response`. The Cline adapter reports the served model of the role it runs as `unavailable` with reason `not_attributed`, because nothing yet shows that the model its SDK names served this invocation. diff --git a/tests/adapters/test_adapters_common_provenance.py b/tests/adapters/test_adapters_common_provenance.py index 34ab857ab..03b2ac7eb 100644 --- a/tests/adapters/test_adapters_common_provenance.py +++ b/tests/adapters/test_adapters_common_provenance.py @@ -227,7 +227,8 @@ def refuse_to_start(_thread: threading.Thread) -> None: def test_package_version_report_marks_harness_and_served_models_no_source(): report = provenance.package_version_report( - "nemo-fabric-adapters-common", ["judge", "default"] + provenance.package_version_fact("nemo-fabric-adapters-common"), + ["judge", "default"], ) no_source = missing("unsupported", "no_source", "adapter_report") diff --git a/tests/adapters/test_claude_adapter.py b/tests/adapters/test_claude_adapter.py index cd40aa1dd..7f239f330 100644 --- a/tests/adapters/test_claude_adapter.py +++ b/tests/adapters/test_claude_adapter.py @@ -31,6 +31,7 @@ from claude_agent_sdk import ResultMessage from claude_agent_sdk import SystemMessage from claude_agent_sdk import TextBlock +from claude_agent_sdk import UserMessage from claude_agent_sdk._errors import MessageParseError from claude_agent_sdk._internal.message_parser import parse_message from nemo_fabric_adapter_contract.models import AgentConfig @@ -2347,6 +2348,90 @@ async def test_claude_served_model_is_isolated_per_invocation( ] +@pytest.mark.parametrize("echoed", [False, True], ids=["terminal-only", "user-echo"]) +async def test_claude_injected_turn_does_not_end_or_bill_the_caller_turn( + claude_payload, monkeypatch, echoed +): + init, _, injected = stream_messages("normal.jsonl") + injected.origin = {"kind": "task-notification"} + injected.total_cost_usd = 0.10 + owned = running_total_result(0.20) + owned.result = "owned answer" + owned.origin = {"kind": "human"} + first: list[Message] = [init] + if echoed: + first.extend( + [ + parse_message(assistant_line("claude-before")), + UserMessage( + content="notification", origin={"kind": "task-notification"} + ), + UserMessage(content="child result", parent_tool_use_id="child-1"), + ] + ) + first.extend([parse_message(assistant_line("claude-injected")), injected]) + install_scripted_client( + monkeypatch, + [ + first, + [parse_message(assistant_line("claude-after")), owned], + [running_total_result(0.25)], + ], + ) + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(claude_payload)) + try: + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + following = await runtime.invoke(*lifecycle_invocation(claude_payload)) + finally: + await runtime.stop() + + assert result.output["response"] == "owned answer" + assert result.output["cost_usd"] is None + assert result.output["usage"] == {"input_tokens": 1, "output_tokens": 1} + assert [ + event["message"]["model"] + for event in result.output["events"] + if event["type"] == "AssistantMessage" + ] == (["claude-before"] if echoed else []) + ["claude-after"] + assert result.usage is None + served = result.extensions[VERSIONS_REPORT_KEY]["models"]["default"]["served"] + expected = ( + served_state("unavailable", "not_attributed") + if echoed + else reported("claude-after", "harness_api") + ) + assert served == expected + assert result.extensions[VERSIONS_REPORT_KEY]["harness"]["version"] == reported( + "2.1.294", "harness_init_message" + ) + assert following.usage is not None + assert following.usage.cost_usd == 0.05 + + +async def test_claude_injected_result_without_caller_result_fails_closed( + claude_payload, monkeypatch +): + init, assistant, injected = stream_messages("normal.jsonl") + injected.origin = {"kind": "task-notification"} + install_scripted_client(monkeypatch, [[init, assistant, injected], []]) + runtime = adapter.ClaudeRuntime() + await runtime.start(lifecycle_start(claude_payload)) + try: + result = await runtime.invoke(*lifecycle_invocation(claude_payload)) + finally: + await runtime.stop() + + assert result.status is AgentRunStatus.FAILED + assert result.error.code == "claude_missing_result" + assert result.usage is None + report = result.extensions[VERSIONS_REPORT_KEY] + assert report["models"]["default"]["served"] == served_state( + "collection_failed", "no_result" + ) + assert report["harness"]["version"] == reported("2.1.294", "harness_init_message") + + async def test_claude_served_model_reads_collection_failed_when_the_stream_fails( claude_payload, monkeypatch ): diff --git a/tests/adapters/test_openclaw.py b/tests/adapters/test_openclaw.py index b1f930979..adf7c3010 100644 --- a/tests/adapters/test_openclaw.py +++ b/tests/adapters/test_openclaw.py @@ -181,6 +181,23 @@ def test_openclaw_accepts_supported_version(): adapter._validate_openclaw_version("OpenClaw 2026.9.4 (3a9d69d)") +def test_openclaw_runtime_keeps_its_loaded_adapter_version(monkeypatch): + installed_version = "1.0" + monkeypatch.setattr( + adapter.provenance.importlib.metadata, + "version", + lambda _distribution: installed_version, + ) + runtime = adapter.OpenClawRuntime() + + first = runtime._versions_extensions()[adapter.provenance.VERSIONS_REPORT_KEY] + installed_version = "2.0" + second = runtime._versions_extensions()[adapter.provenance.VERSIONS_REPORT_KEY] + + assert first["adapter"]["package_version"]["value"] == "1.0" + assert second["adapter"]["package_version"] == first["adapter"]["package_version"] + + def test_openclaw_rejects_unsupported_version(): with pytest.raises(adapter.lifecycle.LifecycleError) as caught: adapter._validate_openclaw_version("OpenClaw 2026.9.5 (abcdef0)") From 4524f5552a36320c9396532b445b966834a0b072 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 17:18:16 -0700 Subject: [PATCH 155/159] fix: preserve adapter health checks in version wrapper Signed-off-by: mnajafian-nv --- adapters/typescript/common/src/provenance.ts | 6 +++++- .../typescript/common/test/provenance.test.mjs | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/adapters/typescript/common/src/provenance.ts b/adapters/typescript/common/src/provenance.ts index bcfe8ecce..8f53b34eb 100644 --- a/adapters/typescript/common/src/provenance.ts +++ b/adapters/typescript/common/src/provenance.ts @@ -122,7 +122,7 @@ export function withPackageVersion( servedModel?: (input: AdapterStartInput) => ServedModelFact | undefined, ): AdapterRuntime { let report: JsonObject | undefined; - return { + const wrapped: AdapterRuntime = { async start(input: AdapterStartInput): Promise { report = packageVersionReport( packageVersionFact(packageJson), @@ -140,4 +140,8 @@ export function withPackageVersion( return runtime.stop(); }, }; + if (runtime.check !== undefined) { + wrapped.check = runtime.check.bind(runtime); + } + return wrapped; } diff --git a/adapters/typescript/common/test/provenance.test.mjs b/adapters/typescript/common/test/provenance.test.mjs index dce4d4d02..93269bd56 100644 --- a/adapters/typescript/common/test/provenance.test.mjs +++ b/adapters/typescript/common/test/provenance.test.mjs @@ -76,6 +76,24 @@ test("the wrapper adds the versions report and keeps the result", async () => { } }); +test("the wrapper preserves runtime health checks", async () => { + const inner = { + levels: [], + async start() {}, + async invoke() { return {status: "succeeded", output: "done"}; }, + async stop() {}, + async check(level) { + this.levels.push(level); + return {outcome: "passed", checks: []}; + }, + }; + const runtime = withPackageVersion(inner, new URL("../package.json", import.meta.url)); + + assert.equal(typeof runtime.check, "function"); + assert.deepEqual(await runtime.check("ready"), {outcome: "passed", checks: []}); + assert.deepEqual(inner.levels, ["ready"]); +}); + test("a served-model override reaches only the role it names", () => { const packageVersion = {value: "1.2.3", source: "reported", collected_by: "adapter", method: "package_metadata"}; const notAttributed = missingFact("unavailable", "not_attributed", "harness_api"); From 6759c8097e8ae2c5ee57c9ea7fa8c750199c08a5 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 18:03:07 -0700 Subject: [PATCH 156/159] fix: drain recorded diagnostics before host cleanup Signed-off-by: mnajafian-nv --- crates/fabric-core/src/runtime.rs | 138 ++++++++++++++++++------------ 1 file changed, 85 insertions(+), 53 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 06f036736..11e2d1450 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -951,7 +951,7 @@ pub fn run_plan_with_recording_context( let mut result = match invoke_runtime_unpersisted(plan, &runtime, request, recorder.as_ref()) { Ok(result) => result, Err(error) => { - let stopped = stop_runtime(plan, &runtime); + let stopped = stop_runtime_inner(plan, &runtime, Some(&mut diagnostics)); if let Some(recorder) = &recorder { recorder.runtime_stopped(&stopped); } @@ -967,7 +967,7 @@ pub fn run_plan_with_recording_context( )); } }; - let stopped = stop_runtime(plan, &runtime); + let stopped = stop_runtime_inner(plan, &runtime, Some(&mut diagnostics)); if let Some(recorder) = &recorder { recorder.runtime_stopped(&stopped); } @@ -1469,10 +1469,28 @@ fn validate_adapter_compatibility(plan: &RunPlan) -> Result<()> { /// Stop or detach from a harness runtime. pub fn stop_runtime(plan: &RunPlan, runtime: &RuntimeHandle) -> Result> { + stop_runtime_inner(plan, runtime, None) +} + +fn stop_runtime_inner( + plan: &RunPlan, + runtime: &RuntimeHandle, + diagnostics: Option<&mut Option>, +) -> Result> { validate_runtime_handle(plan, runtime)?; if uses_local_host(plan) { let events = if runtime.environment.provider == "openshell" { - OpenShellRuntimeAdapter.stop(runtime)? + let stopped = OpenShellRuntimeAdapter.stop(runtime); + if let Some(observer) = diagnostics.and_then(Option::take) { + observer.finish(); + } + stopped? + } else if let Some(observer) = diagnostics { + stop_local_host_with_final_drain(runtime, || { + if let Some(observer) = observer.take() { + observer.finish(); + } + })? } else { LocalHostAdapter.stop(runtime)? }; @@ -1918,58 +1936,69 @@ impl RuntimeAdapter for LocalHostAdapter { } fn stop(&self, runtime: &RuntimeHandle) -> Result> { - local_host_processes().remove(&runtime.runtime_id); - let Some(host) = local_hosts().remove(&runtime.runtime_id) else { - return Ok(vec![local_host_stop_event(runtime, true, false)]); - }; - let mut host = host.lock().unwrap_or_else(|error| error.into_inner()); - let request = - AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Stop(AdapterLifecycleStop { - runtime_id: runtime.runtime_id.clone(), - })); - let result = exchange_lifecycle_message( - &mut host, - &runtime.runtime_id, - &request, - LOCAL_HOST_STOP_TIMEOUT, - ); - let termination = terminate_local_host(&mut host); - let diagnostics = local_host_diagnostics(&host); - let removal = remove_local_host_files(&host); - let host_crashed = matches!( - &result, - Err(FabricError::AdapterLifecycleOperation { code, .. }) - if code == "host_crashed" - ); - if !host_crashed { - result?; - } - termination.map_err(|source| { - lifecycle_error( - AdapterLifecycleOperation::Stop, - &runtime.runtime_id, - "host_termination_failed", - format!("persistent local adapter host could not be terminated: {source}"), - diagnostics, - ) - })?; - removal.map_err(|source| { - lifecycle_error( - AdapterLifecycleOperation::Stop, - &runtime.runtime_id, - "host_cleanup_failed", - format!("persistent local adapter host files could not be removed: {source}"), - "", - ) - })?; + stop_local_host_with_final_drain(runtime, || {}) + } +} - #[cfg(test)] - TEST_STOPPED_AGENTS - .lock() - .expect("stop tracker") - .push(runtime.agent_name.clone()); - Ok(vec![local_host_stop_event(runtime, false, host_crashed)]) +fn stop_local_host_with_final_drain( + runtime: &RuntimeHandle, + final_drain: impl FnOnce(), +) -> Result> { + local_host_processes().remove(&runtime.runtime_id); + let Some(host) = local_hosts().remove(&runtime.runtime_id) else { + final_drain(); + return Ok(vec![local_host_stop_event(runtime, true, false)]); + }; + let mut host = host.lock().unwrap_or_else(|error| error.into_inner()); + let request = + AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Stop(AdapterLifecycleStop { + runtime_id: runtime.runtime_id.clone(), + })); + let result = exchange_lifecycle_message( + &mut host, + &runtime.runtime_id, + &request, + LOCAL_HOST_STOP_TIMEOUT, + ); + let termination = terminate_local_host(&mut host); + let diagnostics = local_host_diagnostics(&host); + // Shutdown diagnostics are captured before the host directory is removed. + // The observer must release its stderr handle before cleanup on Windows. + final_drain(); + let removal = remove_local_host_files(&host); + let host_crashed = matches!( + &result, + Err(FabricError::AdapterLifecycleOperation { code, .. }) + if code == "host_crashed" + ); + if !host_crashed { + result?; } + termination.map_err(|source| { + lifecycle_error( + AdapterLifecycleOperation::Stop, + &runtime.runtime_id, + "host_termination_failed", + format!("persistent local adapter host could not be terminated: {source}"), + diagnostics, + ) + })?; + removal.map_err(|source| { + lifecycle_error( + AdapterLifecycleOperation::Stop, + &runtime.runtime_id, + "host_cleanup_failed", + format!("persistent local adapter host files could not be removed: {source}"), + "", + ) + })?; + + #[cfg(test)] + TEST_STOPPED_AGENTS + .lock() + .expect("stop tracker") + .push(runtime.agent_name.clone()); + Ok(vec![local_host_stop_event(runtime, false, host_crashed)]) } fn start_local_runtime( @@ -2043,6 +2072,9 @@ fn start_local_runtime_recorded( LOCAL_HOST_START_TIMEOUT, ) { let _ = terminate_local_host(&mut host); + if let Some(observer) = diagnostics.take() { + observer.finish(); + } let _ = remove_local_host_files(&host); return Err(error); } From 3dbdd9ca4d3d34c57da7008d17a559a80d7d219c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 18:03:21 -0700 Subject: [PATCH 157/159] fix: preserve invocation diagnostic evidence Keep stdout and stderr artifacts when the journal writer is unavailable. Limit rejected-response diagnostics to the current warm invocation instead of reading an earlier request tail. Signed-off-by: mnajafian-nv --- crates/fabric-core/src/record/invocation.rs | 5 ++ crates/fabric-core/src/runtime.rs | 88 +++++++++++++++++---- 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs index 74c3c004c..4b694b4a2 100644 --- a/crates/fabric-core/src/record/invocation.rs +++ b/crates/fabric-core/src/record/invocation.rs @@ -136,6 +136,11 @@ impl RunRecorder { (state.sequence > 0).then(|| state.path.clone()) } + pub(crate) fn is_writable(&self) -> bool { + let state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + state.writer.is_some() && !state.closed + } + pub(crate) fn runtime_ready(&self, runtime: &RuntimeHandle) { let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); state.runtime_id = Some(runtime.runtime_id.clone()); diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 11e2d1450..d4fd0706e 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -2406,15 +2406,10 @@ fn local_host_invocation_result( .map(|stats| crate::system_stats::persist_summary(stats, &invocation, &fabric_home)); match exchange { Ok(output) => { - let stderr = if recorder.is_some() { - // Live capture owns publication. Only retain the bounded - // diagnostic tail needed to explain a rejected response. - let stderr = local_host_diagnostics(&host_guard); - if let Ok(metadata) = std::fs::metadata(&host_guard.stderr_path) { - host_guard.stderr_offset = - usize::try_from(metadata.len()).unwrap_or(usize::MAX); - } - stderr + let stderr = if recorder.is_some_and(RunRecorder::is_writable) { + // The observer publishes diagnostics. Keep only this + // invocation's bounded tail for a rejected response. + take_local_host_diagnostic_tail(&mut host_guard) } else { take_local_host_stderr(&mut host_guard) }; @@ -2539,7 +2534,7 @@ fn local_host_invocation_result( ), ]), )); - if recorder.is_none() { + if !recorder.is_some_and(RunRecorder::is_writable) { let mut stdout = serde_json::to_string(&output).map_err(FabricError::SerializeJson)?; stdout.push('\n'); write_artifact( @@ -3288,6 +3283,28 @@ fn take_local_host_stderr(host: &mut LocalAdapterHost) -> String { String::from_utf8_lossy(&bytes[start..]).into_owned() } +fn take_local_host_diagnostic_tail(host: &mut LocalAdapterHost) -> String { + let Ok(mut file) = File::open(&host.stderr_path) else { + return String::new(); + }; + let Ok(metadata) = file.metadata() else { + return String::new(); + }; + let end = metadata.len(); + let start = (host.stderr_offset as u64) + .min(end) + .max(end.saturating_sub(LOCAL_HOST_DIAGNOSTIC_LIMIT as u64)); + host.stderr_offset = usize::try_from(end).unwrap_or(usize::MAX); + if file.seek(SeekFrom::Start(start)).is_err() { + return String::new(); + } + let mut bytes = Vec::with_capacity((end - start) as usize); + if file.take(end - start).read_to_end(&mut bytes).is_err() { + return String::new(); + } + String::from_utf8_lossy(&bytes).trim().to_string() +} + fn terminate_local_host(host: &mut LocalAdapterHost) -> std::io::Result<()> { if host.child.try_wait()?.is_some() { return Ok(()); @@ -4636,8 +4653,11 @@ for line in sys.stdin: sys.exit(17) elif operation in {"invoke", "invoke_openai_stream"}: invocations += 1 - if MODE == "invoke_stderr": + if MODE in {"invoke_stderr", "invoke_stderr_then_invalid"}: print(f"diagnostic-{invocations}", file=sys.stderr, flush=True) + if MODE == "invoke_stderr_then_invalid" and invocations == 3: + response(operation, output={"status": "invalid"}) + continue if MODE == "invoke_stderr_overflow": sys.stderr.write("x" * (1024 * 1024 + 31)) sys.stderr.flush() @@ -7470,8 +7490,8 @@ for line in sys.stdin: } #[test] - fn record_off_with_an_artifact_root_writes_no_record() { - let (root, plan) = local_host_plan("typed_result"); + fn recording_disabled_or_failed_keeps_log_artifacts() { + let (root, mut plan) = local_host_plan("invoke_stderr"); assert!(plan.config.runtime.artifacts.is_some()); let result = run_plan(&plan, RunRequest::text("no record")).expect("run"); @@ -7490,6 +7510,37 @@ for line in sys.stdin: .iter() .all(|event| event.kind != "record_write_failed") ); + for name in ["stdout", "stderr"] { + assert!( + result + .artifacts + .artifacts + .iter() + .any(|artifact| artifact.name == name) + ); + } + + plan.config.runtime.record = Some(crate::config::RecordConfig { enabled: true }); + crate::record::test_hooks::FAULT + .with(|slot| slot.set(Some(crate::record::test_hooks::Fault::StartAppend))); + let failed_recording = run_plan(&plan, RunRequest::text("failed journal")); + crate::record::test_hooks::FAULT.with(|slot| slot.set(None)); + let failed_recording = failed_recording.expect("execution survives journal failure"); + assert!( + failed_recording + .events + .iter() + .any(|event| event.kind == "record_write_failed") + ); + for name in ["stdout", "stderr"] { + assert!( + failed_recording + .artifacts + .artifacts + .iter() + .any(|artifact| artifact.name == name) + ); + } let _ = fs::remove_dir_all(root); } @@ -7700,7 +7751,7 @@ for line in sys.stdin: #[test] fn warm_parallel_requests_capture_only_their_serialized_host_window() { - let (root, plan) = record_plan("invoke_stderr"); + let (root, plan) = record_plan("invoke_stderr_then_invalid"); let runtime = start_runtime(&plan).unwrap(); let results = std::thread::scope(|scope| { let first = @@ -7709,8 +7760,15 @@ for line in sys.stdin: scope.spawn(|| invoke_runtime(&plan, &runtime, RunRequest::text("two")).unwrap()); [first.join().unwrap(), second.join().unwrap()] }); + let invalid = invoke_runtime(&plan, &runtime, RunRequest::text("invalid")) + .expect_err("third response is invalid"); + assert!(matches!( + invalid, + FabricError::AdapterLifecycleOperation { code, diagnostics, .. } + if code == "invalid_agent_run_result" && diagnostics == "diagnostic-3" + )); stop_runtime(&plan, &runtime).unwrap(); - assert_eq!(record_files(&root).len(), 2); + assert_eq!(record_files(&root).len(), 3); for result in results { let path = &result .artifacts From f7d7b5e0afcbb452051718ed6382c66caeb92a9c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 19:14:32 -0700 Subject: [PATCH 158/159] feat: report the Strands adapter package version Wrap the Strands runtime with the existing version reporter so its declared result key has an adapter-version producer. Keep harness and served-model facts unsupported. Signed-off-by: mnajafian-nv --- adapters/typescript/common/test/provenance.test.mjs | 2 +- adapters/typescript/strands/src/cli.ts | 4 ++-- docs/sdk/python.mdx | 5 +---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/adapters/typescript/common/test/provenance.test.mjs b/adapters/typescript/common/test/provenance.test.mjs index 93269bd56..4b9bd2301 100644 --- a/adapters/typescript/common/test/provenance.test.mjs +++ b/adapters/typescript/common/test/provenance.test.mjs @@ -31,7 +31,7 @@ test("bounded text matches core's string length rule", async () => { }); test("each TypeScript adapter reads its own package version", async () => { - for (const adapter of ["cline", "droid", "kilo", "kiro", "opencode", "pi", "qwen"]) { + for (const adapter of ["cline", "droid", "kilo", "kiro", "opencode", "pi", "qwen", "strands"]) { const packageJson = new URL(`../../${adapter}/package.json`, import.meta.url); const {version} = JSON.parse(await readFile(packageJson, "utf8")); assert.deepEqual(packageVersionFact(packageJson), { diff --git a/adapters/typescript/strands/src/cli.ts b/adapters/typescript/strands/src/cli.ts index fecc04337..eb323856f 100644 --- a/adapters/typescript/strands/src/cli.ts +++ b/adapters/typescript/strands/src/cli.ts @@ -2,10 +2,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { serve } from "nemo-fabric-adapters-common"; +import { serve, withPackageVersion } from "nemo-fabric-adapters-common"; await serve(async () => { const [{ StrandsAcpSessionFactory }, { StrandsAdapterRuntime }] = await Promise.all([import("./strands-acp.js"), import("./runtime.js")]); - return new StrandsAdapterRuntime(new StrandsAcpSessionFactory()); + return withPackageVersion(new StrandsAdapterRuntime(new StrandsAcpSessionFactory()), new URL("../package.json", import.meta.url)); }); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 37da4a7b9..2eb8a30a1 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1384,10 +1384,7 @@ carry `source`, `collected_by`, and `method`: An adapter reports its facts through the result keys `nemo_fabric.versions.v1` (package version, harness name and version, and served models) and `nemo_fabric.mcp.v1` (MCP `runtime` blocks) when its descriptor declares them. -Bundled adapters other than Strands report their package version. Strands -declares the versions report key but does not emit a report. Its adapter package -version, harness facts and served-model facts have state `unavailable` with -reason `key_omitted`. +Bundled adapters report their package version. The Claude Code, Codex, Hermes Agent, OpenClaw, Deep Agents, and mini-SWE-agent adapters also report their harness name and version. Other adapters that emit a From 2bfd53321fa8d541d42abc195be5e9dd9db4350f Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 26 Oct 2026 19:16:37 -0700 Subject: [PATCH 159/159] docs: qualify when normalized model usage is present Declared adapter evidence can fail normalization. State that condition before describing nullable usage fields; the later failure paragraph already explains the retained raw evidence. Signed-off-by: mnajafian-nv --- docs/sdk/python.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 2eb8a30a1..e8051969b 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1306,8 +1306,8 @@ the SDK raises a `FabricError` subclass instead of returning a partial ### Complete counts and useful observations -`result.model_usage` is present when the adapter supplies the declared -`nemo_fabric.usage_accounting.v1` evidence. Its token fields are independently +`result.model_usage` is present when the adapter supplies declared +`nemo_fabric.usage_accounting.v1` evidence that NeMo Fabric can normalize. Its token fields are independently nullable counts: input includes cache reads and writes, output includes reasoning, and total requires known input plus output. The independent raw provider total remains in `result.usage`.