diff --git a/crates/fabric-cli/src/app.rs b/crates/fabric-cli/src/app.rs index 032df62eb..f8ae26b91 100644 --- a/crates/fabric-cli/src/app.rs +++ b/crates/fabric-cli/src/app.rs @@ -7,7 +7,7 @@ use clap::error::ErrorKind; use clap::{ArgGroup, Args, Parser, Subcommand}; use nemo_fabric_core::{ ResolveContext, RunRequest, RunStatus, doctor_plan, resolve_diagnostic_plan_from_config, - resolve_run_plan_from_config, run_plan, + resolve_run_plan_from_config, run_plan, run_record_folder, }; use crate::examples; @@ -47,6 +47,13 @@ enum Command { /// Inline request text. #[arg(long, default_value = "")] input: String, + /// Task ID recorded on the run (sets `task_id` in the request context). + #[arg( + long, + value_name = "ID", + value_parser = clap::builder::NonEmptyStringValueParser::new() + )] + task: Option, }, /// Print the NVIDIA NeMo Fabric core version. Version, @@ -87,6 +94,45 @@ enum ExampleCommand { }, } +/// Config switches that turn on and label the run record. +#[derive(Debug, Default, Args)] +struct RecordSwitches { + /// Write a run record for each invocation (sets `runtime.record.enabled`). + /// Needs `--artifacts` unless the example sets an artifact root. + #[arg(long)] + record: bool, + /// Artifact root, resolved against the current folder (sets `runtime.artifacts`). + #[arg(long, value_name = "DIR")] + artifacts: Option, + /// Config name recorded on the run (sets `metadata.name`). + #[arg( + long, + value_name = "NAME", + value_parser = clap::builder::NonEmptyStringValueParser::new() + )] + name: Option, +} + +impl RecordSwitches { + /// Apply the switches to the selected config. + fn apply(&self, config: &mut nemo_fabric_core::FabricConfig) -> std::io::Result<()> { + if self.record { + config + .runtime + .record + .get_or_insert_with(nemo_fabric_core::RecordConfig::default) + .enabled = true; + } + if let Some(artifacts) = &self.artifacts { + config.runtime.artifacts = Some(std::path::absolute(artifacts)?); + } + if let Some(name) = &self.name { + config.metadata.name.clone_from(name); + } + Ok(()) + } +} + #[derive(Debug, Args)] #[command(group( ArgGroup::new("source") @@ -125,6 +171,8 @@ struct Selector { value_parser = parse_temperature )] temperature: Option, + #[command(flatten)] + record: RecordSwitches, } fn parse_temperature(value: &str) -> Result { @@ -150,6 +198,13 @@ impl SelectedSource { } } + fn config_mut(&mut self) -> &mut nemo_fabric_core::FabricConfig { + match self { + Self::Preset(selected) => &mut selected.config, + Self::Example(selected) => &mut selected.config, + } + } + fn base_dir(&self) -> &std::path::Path { match self { Self::Preset(selected) => selected.base_dir(), @@ -245,13 +300,26 @@ fn run(cli: Cli) -> Result<(), Box> { )?; println!("{}", serde_json::to_string_pretty(&doctor_plan(&plan))?); } - Command::Run { selector, input } => { + Command::Run { + selector, + input, + task, + } => { let selected = select_source(&selector)?; let plan = resolve_run_plan_from_config( selected.config().clone(), ResolveContext::new(selected.base_dir()), )?; - let result = match run_plan(&plan, RunRequest::text(input)) { + let mut request = RunRequest::text(input); + if let Some(task) = task { + request + .context + .insert("task_id".to_string(), serde_json::Value::from(task)); + } + if let Some(folder) = run_record_folder(&plan) { + eprintln!("nemo-fabric: run record under {}", folder.display()); + } + let result = match run_plan(&plan, request) { Ok(result) => result, Err(error) => { println!( @@ -284,17 +352,17 @@ fn require_successful_run(status: RunStatus) -> Result<(), String> { } fn select_source(selector: &Selector) -> Result> { - if let Some(name) = &selector.preset { + let mut selected = if let Some(name) = &selector.preset { let mut selected = find_preset(name)?.stage()?; apply_model_overrides(&mut selected.config, selector)?; - return Ok(SelectedSource::Preset(selected)); - } - if let Some(name) = &selector.example { - return Ok(SelectedSource::Example( - find_example(name)?.select(selector.variant.as_deref())?, - )); - } - unreachable!("Clap requires exactly one source selector") + SelectedSource::Preset(selected) + } else if let Some(name) = &selector.example { + SelectedSource::Example(find_example(name)?.select(selector.variant.as_deref())?) + } else { + unreachable!("Clap requires exactly one source selector") + }; + selector.record.apply(selected.config_mut())?; + Ok(selected) } fn apply_model_overrides( @@ -381,6 +449,7 @@ mod tests { variant: None, model: Some("meta/llama-3.3-70b-instruct".to_string()), temperature: Some(0.2), + record: RecordSwitches::default(), }; let selected = select_source(&selector).expect("select preset"); let model = selected.config().models.get("default").expect("model"); @@ -389,6 +458,7 @@ mod tests { assert_eq!(model.model, "meta/llama-3.3-70b-instruct"); assert_eq!(model.temperature, Some(0.2)); assert_eq!(model.api_key_env.as_deref(), Some("NVIDIA_API_KEY")); + assert!(selected.config().runtime.record.is_none()); } #[test] @@ -411,6 +481,7 @@ mod tests { variant: None, model: Some("other-model".to_string()), temperature: None, + record: RecordSwitches::default(), }; assert_eq!( select_source(&selector) @@ -477,4 +548,49 @@ mod tests { fn plan_requires_a_preset() { assert!(Cli::try_parse_from(["nemo-fabric", "plan"]).is_err()); } + + #[test] + fn record_switches_parse_on_plan_doctor_and_run_and_set_the_config() { + let selectors: Vec = ["plan", "doctor", "run"] + .into_iter() + .map(|command| { + let cli = Cli::try_parse_from([ + "nemo-fabric", + command, + "--preset", + "scripted", + "--record", + "--artifacts", + "runs", + "--name", + "swe-scripted", + ]) + .expect("parse record switches"); + match cli.command { + Command::Plan(selector) | Command::Doctor(selector) => selector, + Command::Run { selector, .. } => selector, + _ => panic!("selector command expected"), + } + }) + .collect(); + for selector in &selectors { + assert!(selector.record.record); + assert_eq!(selector.record.name.as_deref(), Some("swe-scripted")); + } + + let selected = select_source(&selectors[0]).expect("select"); + + let config = selected.config(); + assert!( + config + .runtime + .record + .as_ref() + .is_some_and(|record| record.enabled) + ); + let artifacts = config.runtime.artifacts.as_ref().expect("artifacts"); + assert!(artifacts.is_absolute()); + assert!(artifacts.ends_with("runs")); + assert_eq!(config.metadata.name, "swe-scripted"); + } } diff --git a/crates/fabric-cli/src/presets.rs b/crates/fabric-cli/src/presets.rs index 803c6667e..5ebaa3202 100644 --- a/crates/fabric-cli/src/presets.rs +++ b/crates/fabric-cli/src/presets.rs @@ -287,6 +287,7 @@ fn config( artifacts: None, timeout_seconds: None, max_turns: None, + record: None, extensions: BTreeMap::new(), }, environment: Some(EnvironmentConfig { diff --git a/crates/fabric-cli/templates/rust/main.rs.tmpl b/crates/fabric-cli/templates/rust/main.rs.tmpl index e7f0ccfd3..be1c9771f 100644 --- a/crates/fabric-cli/templates/rust/main.rs.tmpl +++ b/crates/fabric-cli/templates/rust/main.rs.tmpl @@ -38,6 +38,7 @@ fn build_config() -> FabricConfig { artifacts: None, timeout_seconds: {{TIMEOUT_SECONDS}}, max_turns: {{MAX_TURNS}}, + record: None, extensions: BTreeMap::new(), }, environment: Some(EnvironmentConfig { diff --git a/crates/fabric-cli/tests/run_record.rs b/crates/fabric-cli/tests/run_record.rs new file mode 100644 index 000000000..136207389 --- /dev/null +++ b/crates/fabric-cli/tests/run_record.rs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! CLI regression coverage for the run record switches. + +use std::process::Command; + +#[test] +fn run_with_record_writes_task_and_name_and_keeps_stdout_json() { + let root = std::env::temp_dir().join(format!("fabric-cli-record-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + + let output = Command::new(env!("CARGO_BIN_EXE_nemo-fabric")) + .args([ + "run", "--preset", "scripted", "--input", "hello", "--record", + ]) + .arg("--artifacts") + .arg(&root) + .args(["--name", "swe-scripted", "--task", "repo-io-fix"]) + .output() + .expect("run CLI"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "scripted run failed: {stderr}"); + + // The folder line comes first on standard error, and standard output is + // only the JSON result. + let folder = root.join(".fabric"); + assert_eq!( + stderr.lines().next(), + Some(format!("nemo-fabric: run record under {}", folder.display()).as_str()), + "{stderr}" + ); + let result: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("standard output is one JSON document"); + assert_eq!(result["status"], "succeeded"); + + let records: Vec<_> = std::fs::read_dir(&folder) + .expect("runtime folders") + .flatten() + .flat_map(|runtime| { + std::fs::read_dir(runtime.path()) + .expect("invocations") + .flatten() + }) + .map(|invocation| invocation.path().join("run-record.jsonl")) + .filter(|path| path.exists()) + .collect(); + assert_eq!(records.len(), 1, "{records:?}"); + let text = std::fs::read_to_string(&records[0]).expect("record"); + let start: serde_json::Value = + serde_json::from_str(text.lines().next().expect("start line")).expect("json"); + assert_eq!(start["context"]["task_id"], "repo-io-fix"); + assert_eq!(start["agent_name"], "swe-scripted"); + assert!( + text.lines() + .last() + .expect("end line") + .contains("\"kind\":\"end\"") + ); + let _ = std::fs::remove_dir_all(root); +} diff --git a/crates/fabric-core/src/config.rs b/crates/fabric-core/src/config.rs index 667fe951a..7de39134c 100644 --- a/crates/fabric-core/src/config.rs +++ b/crates/fabric-core/src/config.rs @@ -1057,11 +1057,34 @@ pub struct RuntimeConfig { #[serde(default, skip_serializing_if = "Option::is_none")] #[schemars(range(min = 1, max = u32::MAX))] pub max_turns: Option, + /// Opt-in per-invocation run record. Core-only; never sent to adapters. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub record: Option, /// Additive normalized runtime fields. #[serde(default, flatten)] pub extensions: BTreeMap, } +impl RuntimeConfig { + /// Whether `runtime.record.enabled` is set. + pub(crate) fn record_enabled(&self) -> bool { + self.record.as_ref().is_some_and(|record| record.enabled) + } +} + +/// Per-invocation run record settings. +/// +/// When enabled, core appends a start line and an end line for each local +/// invocation to `/.fabric///run-record.jsonl`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct RecordConfig { + /// Whether to write a run record for each invocation. Requires an + /// artifact root, the `local` environment provider, and a `process` or + /// `python` adapter. + pub enabled: bool, +} + fn default_input_schema() -> String { "text".to_string() } @@ -1928,6 +1951,7 @@ pub(crate) fn validate_config(config: &FabricConfig) -> Result<()> { "must be a finite number greater than zero", ); } + validate_record_config(config)?; if let Some(system) = config .instructions .as_ref() @@ -2379,6 +2403,52 @@ fn validate_names(field: &str, names: &[String]) -> Result<()> { Ok(()) } +/// A record needs a host-side artifact root and a local adapter host, so an +/// enabled record that could never be written fails validation instead of +/// silently writing nothing. +fn validate_record_config(config: &FabricConfig) -> Result<()> { + if !config.runtime.record_enabled() { + return Ok(()); + } + let environment = config.environment.as_ref(); + if environment.is_some_and(|environment| environment.provider != "local") { + return invalid_config( + "runtime.record.enabled", + "supported only with the local environment provider", + ); + } + let has_artifact_root = config.runtime.artifacts.is_some() + || environment.is_some_and(|environment| environment.artifacts.is_some()); + if !has_artifact_root { + return invalid_config( + "runtime.record.enabled", + "requires runtime.artifacts or an environment artifact root", + ); + } + Ok(()) +} + +/// Core writes the record from its local adapter host path, so an adapter +/// that runs elsewhere would get an enabled record that is never written. +fn validate_record_adapter( + config: &FabricConfig, + descriptor: Option<&AdapterDescriptor>, +) -> Result<()> { + let local_host = descriptor.is_none_or(|descriptor| { + matches!( + descriptor.adapter_kind, + AdapterKind::Process | AdapterKind::Python + ) + }); + if config.runtime.record_enabled() && !local_host { + return invalid_config( + "runtime.record.enabled", + "supported only with a process or python adapter", + ); + } + Ok(()) +} + fn invalid_config(field: impl Into, reason: impl Into) -> Result { Err(FabricError::InvalidConfig { field: field.into(), @@ -2579,6 +2649,7 @@ fn resolve_selected_descriptors( let resolution = resolve_resolution(&config, descriptor)?; let environment_plan = resolve_environment_plan(&config, &base_dir); validate_control_location(descriptor, environment_plan.as_ref())?; + validate_record_adapter(&config, descriptor)?; let capability_plan = resolve_capability_plan(&config, &base_dir, adapter_descriptor.as_ref()); if enforce_compatibility { validate_capability_plan_compatibility(&capability_plan, descriptor)?; @@ -7427,6 +7498,104 @@ mod tests { && paths.len() == 2 )); } + + fn record_error(config: &FabricConfig) -> Option<(String, String)> { + match validate_config(config) { + Err(FabricError::InvalidConfig { field, reason }) => Some((field, reason)), + Err(error) => panic!("unexpected error: {error}"), + Ok(()) => None, + } + } + + #[test] + fn record_enabled_without_an_artifact_root_fails_validation() { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.record = Some(RecordConfig { enabled: true }); + + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "requires runtime.artifacts or an environment artifact root".to_string() + )) + ); + config.environment.as_mut().expect("environment").artifacts = Some(PathBuf::from("runs")); + assert_eq!(record_error(&config), None); + config.environment.as_mut().expect("environment").artifacts = None; + config.runtime.artifacts = Some(PathBuf::from("runs")); + assert_eq!(record_error(&config), None); + } + + #[test] + fn record_enabled_with_a_non_local_provider_fails_validation() { + for provider in ["docker", "openshell"] { + let mut config = typed_config("nvidia.fabric.hermes"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + let environment = config.environment.as_mut().expect("environment"); + environment.provider = provider.to_string(); + // Valid OpenShell task paths, so only the record check can fail. + environment.workspace = Some(PathBuf::from("/sandbox/workspace")); + environment.artifacts = Some(PathBuf::from("/sandbox/artifacts")); + + assert_eq!( + record_error(&config), + Some(( + "runtime.record.enabled".to_string(), + "supported only with the local environment provider".to_string() + )), + "{provider}" + ); + } + } + + #[test] + fn record_enabled_with_an_adapter_outside_the_local_host_fails_validation() { + let path = repository_root().join("adapters/python/claude/claude.fabric-adapter.json"); + let mut descriptor = load_adapter_descriptor(&path).expect("Claude descriptor"); + let mut config = typed_config("nvidia.fabric.claude"); + config.runtime.artifacts = Some(PathBuf::from("runs")); + config.runtime.record = Some(RecordConfig { enabled: true }); + assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); + + for kind in [AdapterKind::Http, AdapterKind::NativePlugin] { + descriptor.adapter_kind = kind; + assert!(matches!( + validate_record_adapter(&config, Some(&descriptor)), + Err(FabricError::InvalidConfig { field, reason }) + if field == "runtime.record.enabled" + && reason == "supported only with a process or python adapter" + )); + } + config.runtime.record = None; + assert!(validate_record_adapter(&config, Some(&descriptor)).is_ok()); + } + + #[test] + fn record_disabled_or_absent_needs_no_artifact_root() { + let mut config = typed_config("nvidia.fabric.hermes"); + assert_eq!(record_error(&config), None); + config.runtime.record = Some(RecordConfig { enabled: false }); + assert_eq!(record_error(&config), None); + } + + #[test] + fn record_block_requires_enabled_and_rejects_unknown_fields() { + for runtime in [ + serde_json::json!({"record": {}}), + serde_json::json!({"record": {"enabled": true, "durable": true}}), + serde_json::json!({"record": true}), + ] { + let mut value = + serde_json::to_value(typed_config("nvidia.fabric.hermes")).expect("config JSON"); + value["runtime"] = runtime.clone(); + assert!( + serde_json::from_value::(value).is_err(), + "{runtime}" + ); + } + } + #[test] fn remote_environment_paths_are_not_resolved_on_the_host() { let mut config = typed_config("nvidia.fabric.hermes"); diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index af3493a60..3e58ac0c1 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -10,9 +10,11 @@ pub mod config; pub mod doctor; mod environment; pub mod error; +mod record; pub mod runtime; pub mod schema; mod task_command; +mod time; pub use adapter_contract::{ADAPTER_CONTRACT_VERSION, AdapterExtensionPoint}; pub use agent_execution::{ @@ -30,10 +32,11 @@ pub use config::{ EnvironmentOwnership, EnvironmentPlan, FabricConfig, HarnessConfig, InstructionConfig, InstructionMode, InstructionsConfig, McpAuthenticationConfig, McpConfig, McpExposure, McpServerPlan, McpTransport, MetadataConfig, ModelConfig, OAuthTokenEndpointAuthMethod, - ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, ResolvedAdapterTargetDescriptor, - RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, TelemetryConfig, TelemetryPlan, - TelemetryProvider, TelemetryProviderConfig, ToolDefinitionConfig, ToolsConfig, WorkflowConfig, - WorkflowEntrypointConfig, WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, + RecordConfig, ResolutionStrategy, ResolveContext, ResolvedAdapterDescriptor, + ResolvedAdapterTargetDescriptor, RunPlan, RuntimeCapabilities, RuntimeConfig, SkillConfig, + TelemetryConfig, TelemetryPlan, TelemetryProvider, TelemetryProviderConfig, + ToolDefinitionConfig, ToolsConfig, WorkflowConfig, WorkflowEntrypointConfig, + WorkflowTargetSpec, inspect_adapter_config, load_adapter_descriptor, load_adapter_target_descriptor, resolve_diagnostic_plan_from_config, resolve_diagnostic_plan_from_config_with_adapter_directories, resolve_run_plan_from_config, resolve_run_plan_from_config_with_adapter_directories, @@ -49,8 +52,8 @@ pub use runtime::{ RunRequest, RunResult, RunStatus, RunUsage, RuntimeContext, RuntimeHandle, RuntimeTelemetryContext, ServiceHandle, ServiceOwnership, ServiceReference, TelemetryRef, attach_environment, attach_service, invoke_openai_stream, invoke_runtime, prepare_environment, - prepare_service, release_environment, release_service, run_plan, start_runtime, - start_runtime_in, start_runtime_with_service, stop_runtime, + prepare_service, release_environment, release_service, run_plan, run_record_folder, + start_runtime, start_runtime_in, start_runtime_with_service, stop_runtime, }; pub use schema::{ SchemaName, generate_all_schemas, generate_schema, generate_schema_json, write_schema_snapshots, diff --git a/crates/fabric-core/src/record/invocation.rs b/crates/fabric-core/src/record/invocation.rs new file mode 100644 index 000000000..2f41f53f6 --- /dev/null +++ b/crates/fabric-core/src/record/invocation.rs @@ -0,0 +1,684 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One invocation's record: the start line before the request, the end line +//! after the invocation. +//! +//! Record I/O errors do not replace the invocation result. A failed create, +//! write or sync stops the record for that invocation and adds a +//! `record_write_failed` event to the result; nothing here returns a record +//! error to the caller. + +use std::collections::BTreeMap; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime}; + +use serde_json::Value; + +use super::lines::{ + ContextValue, EndArtifact, EndError, EndLine, EndUsage, LineBody, ModelName, NotRecorded, + RecordLine, RecordSchema, RecordVersion, StartLine, cut_text, recorded_code, +}; +use super::writer::{RECORD_FILE_NAME, RecordWriter}; +use crate::config::RunPlan; +use crate::error::{FabricError, Result}; +use crate::runtime::{ + ArtifactManifest, ArtifactRef, ErrorInfo, ErrorStage, INVALID_AGENT_RUN_RESULT, + InvocationHandle, RunRequest, RunResult, RunStatus, RunUsage, RuntimeHandle, + event_with_metadata, unique_artifact_name, +}; +use crate::time::rfc3339_millis; + +/// Name of the result artifact that points at the record. +const RECORD_ARTIFACT_NAME: &str = "record"; +/// Kind of the result artifact that points at the record. +const RECORD_ARTIFACT_KIND: &str = "run_record"; +/// Event added to the result when the record could not be written. +const RECORD_WRITE_FAILED: &str = "record_write_failed"; +/// `RunRequest.context` keys whose string or integer values the start line +/// keeps: task identity and the Harbor trial link. +const RECORDED_CONTEXT_KEYS: [&str; 7] = [ + "task_id", + "task_revision", + "attempt", + "step", + "harbor_session_id", + "harbor_context_id", + "harbor_trial_name", +]; +/// End-line error code for a failed Fabric file write: `adapter-invocation.json` +/// before the request, or a result artifact after the adapter returned. +const ARTIFACT_WRITE_FAILED: &str = "artifact_write_failed"; +/// End-line error code for any other Fabric error after the record started. +const FABRIC_ERROR: &str = "fabric_error"; + +/// What the start line needs from core's invocation path. +pub(crate) struct RecordStart<'a> { + pub plan: &'a RunPlan, + pub runtime: &'a RuntimeHandle, + pub invocation: &'a InvocationHandle, + pub request: &'a RunRequest, + /// The invocation folder, `/.fabric//`. + pub fabric_home: &'a Path, + /// The host timeout that applies to this invocation. + pub timeout: Duration, +} + +/// The record of one running invocation. +#[derive(Debug)] +pub(crate) struct InvocationRecorder { + runtime_id: String, + invocation_id: String, + request_id: String, + path: PathBuf, + started: Instant, + writer: Option, + wrote_start: bool, + failure: Option, +} + +impl InvocationRecorder { + /// Create the record and write its start line. + pub(crate) fn start(start: RecordStart<'_>) -> Self { + let path = start.fabric_home.join(RECORD_FILE_NAME); + let mut recorder = Self { + runtime_id: start.invocation.runtime_id.clone(), + invocation_id: start.invocation.invocation_id.clone(), + request_id: start.invocation.request_id.clone(), + started: Instant::now(), + writer: None, + wrote_start: false, + failure: None, + path, + }; + match RecordWriter::create(&recorder.path) { + Ok(writer) => { + recorder.writer = Some(writer); + recorder.append(LineBody::Start(start_line(&start))); + recorder.wrote_start = recorder.failure.is_none(); + } + Err(error) => recorder.failure = Some(error), + } + recorder + } + + /// Write and sync the end line for `outcome`, then return it. A result + /// gains the `record` artifact when the start line was written, and a + /// `record_write_failed` event when any write or the sync failed. + pub(crate) fn finish(mut self, outcome: Result) -> Result { + let elapsed_ms = u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX); + let end = match &outcome { + Ok(result) => result_end_line(result, elapsed_ms), + Err(error) => error_end_line(error, elapsed_ms), + }; + self.append(LineBody::End(end)); + if let Some(writer) = &self.writer + && let Err(error) = writer.sync() + { + self.failure = Some(error); + } + let mut result = outcome?; + if self.wrote_start { + let name = unique_artifact_name(&result.artifacts, RECORD_ARTIFACT_NAME); + result.artifacts.artifacts.push(ArtifactRef { + name, + kind: RECORD_ARTIFACT_KIND.to_string(), + path: self.path.clone(), + media_type: Some("application/jsonl".to_string()), + metadata: BTreeMap::new(), + }); + } + if let Some(error) = &self.failure { + result.events.push(event_with_metadata( + RECORD_WRITE_FAILED, + format!("run record write failed; the record stops here: {error}"), + BTreeMap::from([ + ( + "runtime_id".to_string(), + Value::from(self.runtime_id.as_str()), + ), + ( + "invocation_id".to_string(), + Value::from(self.invocation_id.as_str()), + ), + ]), + )); + } + Ok(result) + } + + /// Append one line. After the first failure the record stops. + fn append(&mut self, body: LineBody) { + let Some(writer) = self.writer.as_mut() else { + return; + }; + let line = RecordLine { + schema: RecordSchema::RunRecord, + version: RecordVersion::V1Alpha1, + runtime_id: self.runtime_id.clone(), + invocation_id: self.invocation_id.clone(), + request_id: self.request_id.clone(), + time: rfc3339_millis(SystemTime::now()), + body, + }; + if let Err(error) = writer.append(&line) { + self.failure = Some(error); + self.writer = None; + } + } +} + +fn start_line(start: &RecordStart<'_>) -> StartLine { + StartLine { + agent_name: cut_text(&start.runtime.agent_name), + harness: cut_text(&start.runtime.harness), + adapter_id: start.runtime.adapter_id.as_deref().map(cut_text), + models: start + .plan + .config + .models + .iter() + .map(|(role, model)| { + ( + role.clone(), + ModelName { + provider: cut_text(&model.provider), + model: cut_text(&model.model), + }, + ) + }) + .collect(), + fabric_version: crate::version().to_string(), + timeout_ms: u64::try_from(start.timeout.as_millis()).unwrap_or(u64::MAX), + context: start + .request + .context + .iter() + .map(|(key, value)| (key.clone(), context_value(key, value))) + .collect(), + override_keys: override_keys(start.request.overrides.as_ref()), + } +} + +/// The sorted top-level key names of `overrides`; empty unless it is an object. +fn override_keys(overrides: Option<&Value>) -> Vec { + let Some(Value::Object(overrides)) = overrides else { + return Vec::new(); + }; + let mut keys: Vec = overrides.keys().map(|key| cut_text(key)).collect(); + keys.sort(); + keys +} + +/// An allowlisted key keeps a string (cut) or integer value; any other key, +/// or an allowlisted key with another value type, is recorded by name only. +fn context_value(key: &str, value: &Value) -> ContextValue { + if !RECORDED_CONTEXT_KEYS.contains(&key) { + return ContextValue::NotRecorded(NotRecorded::VALUE); + } + match value { + Value::String(text) => ContextValue::Text(cut_text(text)), + Value::Number(number) => number + .as_i64() + .map_or(ContextValue::NotRecorded(NotRecorded::VALUE), |integer| { + ContextValue::Integer(integer) + }), + _ => ContextValue::NotRecorded(NotRecorded::VALUE), + } +} + +fn result_end_line(result: &RunResult, elapsed_ms: u64) -> EndLine { + EndLine { + status: result.status, + elapsed_ms, + error: result.error.as_ref().map(result_error), + usage: result.usage.as_ref().map(end_usage), + artifacts: end_artifacts(&result.artifacts), + } +} + +fn error_end_line(error: &FabricError, elapsed_ms: u64) -> EndLine { + let (code, stage) = match error { + FabricError::AdapterLifecycleOperation { code, .. } => (code.as_str(), ErrorStage::Invoke), + // After the record starts, the check of the adapter's result + // extensions raises these: an extension the descriptor does not + // declare, or one that breaks its declared schema. Both are the same + // fault as a rejected result. + FabricError::AdapterCompatibility { .. } | FabricError::InvalidAdapterExtension { .. } => { + (INVALID_AGENT_RUN_RESULT, ErrorStage::Invoke) + } + FabricError::Write { .. } => (ARTIFACT_WRITE_FAILED, ErrorStage::Artifact), + _ => (FABRIC_ERROR, ErrorStage::Invoke), + }; + EndLine { + status: RunStatus::Failed, + elapsed_ms, + error: Some(EndError { + code: recorded_code(code), + stage, + retryable: false, + }), + usage: None, + artifacts: Vec::new(), + } +} + +fn result_error(error: &ErrorInfo) -> EndError { + EndError { + code: recorded_code(&error.code), + stage: error.stage, + retryable: error.retryable, + } +} + +fn end_usage(usage: &RunUsage) -> EndUsage { + EndUsage { + input_tokens: usage.input_tokens, + cached_input_tokens: usage.cached_input_tokens, + input_tokens_include_cache: usage.input_tokens_include_cache, + output_tokens: usage.output_tokens, + total_tokens: usage.total_tokens, + cost_usd: usage.cost_usd, + } +} + +fn end_artifacts(manifest: &ArtifactManifest) -> Vec { + manifest + .artifacts + .iter() + .map(|artifact| EndArtifact { + name: cut_text(&artifact.name), + kind: cut_text(&artifact.kind), + media_type: artifact.media_type.as_deref().map(cut_text), + path: manifest + .root + .as_deref() + .and_then(|root| artifact.path.strip_prefix(root).ok()) + .map(|relative| relative.to_string_lossy().into_owned()), + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{FabricConfig, ResolveContext, resolve_run_plan_from_config}; + use crate::record::test_hooks::{self, Fault}; + + #[test] + fn rejected_result_extensions_read_invalid_agent_run_result() { + let undeclared = FabricError::AdapterCompatibility { + adapter_id: "demo".to_string(), + field: "result.extensions".to_string(), + reason: "not declared".to_string(), + }; + let schema_invalid = FabricError::InvalidAdapterExtension { + adapter_id: "demo".to_string(), + descriptor_source: crate::config::DescriptorSource::Bundled, + descriptor_path: std::path::PathBuf::from("adapter.json"), + extension_path: "result.extensions.demo".to_string(), + reason: "breaks the declared schema".to_string(), + }; + for error in [undeclared, schema_invalid] { + let line = error_end_line(&error, 1); + assert_eq!( + line.error.expect("end line error").code, + INVALID_AGENT_RUN_RESULT, + "{error}" + ); + } + } + + fn record_test_fixture() -> (RunPlan, RuntimeHandle, InvocationHandle) { + let config: FabricConfig = serde_json::from_value(serde_json::json!({ + "schema_version": "fabric.agent/v1alpha1", + "metadata": {"name": "demo"}, + "harness": {"adapter_id": "nvidia.fabric.hermes", "resolution": "preinstalled"}, + "runtime": {}, + "models": {"default": {"provider": "nvidia", "model": "nemotron"}}, + })) + .expect("config"); + let plan = resolve_run_plan_from_config(config, ResolveContext::new("/tmp/fabric-base")) + .expect("plan"); + let runtime: RuntimeHandle = serde_json::from_value(serde_json::json!({ + "runtime_id": "rt", "runtime_binding": "binding", "agent_name": "demo", + "harness": "nvidia.fabric.hermes", "adapter_kind": "python", + "adapter_id": "nvidia.fabric.hermes", + "environment": { + "environment_id": "env", "provider": "local", + "control_location": "external_control", "ownership": "caller_owned", + }, + })) + .expect("runtime handle"); + let invocation = InvocationHandle { + invocation_id: "inv".to_string(), + request_id: "req".to_string(), + runtime_id: "rt".to_string(), + }; + (plan, runtime, invocation) + } + + #[test] + fn start_line_keeps_override_and_unknown_context_key_names_only() { + let (plan, runtime, invocation) = record_test_fixture(); + let mut request = RunRequest::text("input"); + request.context = BTreeMap::from([ + ("task_id".to_string(), Value::from("t".repeat(300))), + ("task_revision".to_string(), Value::from(1.5)), + ( + "headers".to_string(), + serde_json::json!({"authorization": "Bearer abc"}), + ), + ]); + request.overrides = Some(serde_json::json!({"zeta": 1, "acme": {"api_key": "sk-123"}})); + let start = RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: Path::new("/unused"), + timeout: Duration::from_secs(5), + }; + + let line = serde_json::to_value(RecordLine { + schema: RecordSchema::RunRecord, + version: RecordVersion::V1Alpha1, + runtime_id: "rt".to_string(), + invocation_id: "inv".to_string(), + request_id: "req".to_string(), + time: "2026-10-14T18:01:10.205Z".to_string(), + body: LineBody::Start(start_line(&start)), + }) + .expect("start line"); + + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunRecordLine) + .expect("record line schema"); + let validator = jsonschema::validator_for(&schema).expect("valid schema"); + assert!(validator.is_valid(&line), "{line}"); + assert_eq!(line["override_keys"], serde_json::json!(["acme", "zeta"])); + assert_eq!( + line["context"]["headers"], + serde_json::json!({"recorded": false}) + ); + assert_eq!( + line["context"]["task_revision"], + serde_json::json!({"recorded": false}) + ); + assert_eq!( + line["context"]["task_id"] + .as_str() + .expect("task id") + .chars() + .count(), + 256 + ); + assert_eq!(line["timeout_ms"], 5_000); + assert_eq!( + line["models"], + serde_json::json!({"default": {"provider": "nvidia", "model": "nemotron"}}) + ); + let text = line.to_string(); + assert!( + !text.contains("sk-123") && !text.contains("Bearer"), + "{text}" + ); + } + + #[test] + fn start_line_keeps_long_role_and_context_key_names_apart() { + let (mut plan, runtime, invocation) = record_test_fixture(); + let shared = "k".repeat(300); + let model = plan.config.models["default"].clone(); + for suffix in ["a", "b"] { + plan.config + .models + .insert(format!("{shared}{suffix}"), model.clone()); + } + let mut request = RunRequest::text("input"); + request.context = BTreeMap::from([ + (format!("{shared}a"), Value::from(1)), + (format!("{shared}b"), Value::from(2)), + ]); + let start = RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: Path::new("/unused"), + timeout: Duration::from_secs(5), + }; + + let line = start_line(&start); + + assert_eq!(line.models.len(), 3); + assert_eq!(line.context.len(), 2); + for suffix in ["a", "b"] { + assert!(line.models.contains_key(&format!("{shared}{suffix}"))); + assert!(line.context.contains_key(&format!("{shared}{suffix}"))); + } + } + + fn temp_folder(name: &str) -> PathBuf { + let folder = + std::env::temp_dir().join(format!("fabric-recorder-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&folder); + std::fs::create_dir_all(&folder).expect("folder"); + folder + } + + fn succeeded_result(invocation: &InvocationHandle) -> RunResult { + serde_json::from_value(serde_json::json!({ + "agent_name": "a", "harness": "h", "adapter_kind": "python", + "runtime_id": invocation.runtime_id, "invocation_id": invocation.invocation_id, + "request_id": invocation.request_id, "status": "succeeded", "output": "done", + })) + .expect("result") + } + + #[test] + fn finish_names_the_record_artifact_apart_from_an_adapter_artifact_named_record() { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder("unique-name"); + let request = RunRequest::text("input"); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + let mut result = succeeded_result(&invocation); + result.artifacts.artifacts.push(ArtifactRef { + name: RECORD_ARTIFACT_NAME.to_string(), + kind: "report".to_string(), + path: folder.join("report.json"), + media_type: None, + metadata: BTreeMap::new(), + }); + + let finished = recorder.finish(Ok(result)).expect("result"); + + let names: Vec<&str> = finished + .artifacts + .artifacts + .iter() + .map(|artifact| artifact.name.as_str()) + .collect(); + assert_eq!(names, ["record", "record_2"]); + assert_eq!(finished.artifacts.artifacts[1].kind, RECORD_ARTIFACT_KIND); + let _ = std::fs::remove_dir_all(folder); + } + + #[test] + fn finish_after_a_failed_create_adds_the_event_and_no_record_artifact() { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder("create-failure"); + std::fs::write(folder.join(RECORD_FILE_NAME), "planted").expect("planted file"); + let request = RunRequest::text("input"); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + let result = succeeded_result(&invocation); + + let finished = recorder.finish(Ok(result.clone())).expect("result"); + + assert_eq!(finished.artifacts, result.artifacts); + assert_eq!(finished.events.len(), 1); + assert_eq!(finished.events[0].kind, RECORD_WRITE_FAILED); + assert_eq!(finished.status, RunStatus::Succeeded); + assert_eq!( + std::fs::read_to_string(folder.join(RECORD_FILE_NAME)).expect("read"), + "planted" + ); + let _ = std::fs::remove_dir_all(folder); + } + + fn failed_result_with_usage(invocation: &InvocationHandle) -> RunResult { + serde_json::from_value(serde_json::json!({ + "agent_name": "a", "harness": "h", "adapter_kind": "python", + "runtime_id": invocation.runtime_id, "invocation_id": invocation.invocation_id, + "request_id": invocation.request_id, "status": "failed", "output": "partial", + "error": { + "stage": "invoke", "code": "adapter_failure", + "message": "adapter gave up", "retryable": true, + }, + "usage": {"input_tokens": 3, "output_tokens": 5, "cost_usd": 0.25}, + })) + .expect("result") + } + + /// Start and finish one record with `fault` injected, and return the + /// finished outcome with the record file's lines. + fn finish_with_fault( + name: &str, + fault: Fault, + outcome: Result, + ) -> (Result, Vec) { + let (plan, runtime, invocation) = record_test_fixture(); + let folder = temp_folder(name); + let request = RunRequest::text("input"); + test_hooks::FAULT.with(|slot| slot.set(Some(fault))); + let recorder = InvocationRecorder::start(RecordStart { + plan: &plan, + runtime: &runtime, + invocation: &invocation, + request: &request, + fabric_home: &folder, + timeout: Duration::from_secs(5), + }); + let finished = recorder.finish(outcome); + test_hooks::FAULT.with(|slot| slot.set(None)); + let lines = std::fs::read_to_string(folder.join(RECORD_FILE_NAME)) + .expect("read record") + .lines() + .map(|line| serde_json::from_str(line).expect("record line")) + .collect(); + let _ = std::fs::remove_dir_all(folder); + (finished, lines) + } + + /// `finished` equals `original` apart from the artifacts and events that + /// recording adds, and carries one `record_write_failed` event. + fn assert_result_kept_with_failure_event(finished: &RunResult, original: &RunResult) { + let mut without_record = finished.clone(); + without_record.artifacts = original.artifacts.clone(); + without_record.events = original.events.clone(); + assert_eq!(&without_record, original); + let failures: Vec<_> = finished + .events + .iter() + .filter(|event| event.kind == RECORD_WRITE_FAILED) + .collect(); + assert_eq!(failures.len(), 1, "{:?}", finished.events); + assert_eq!( + failures[0].metadata["runtime_id"], + original.runtime_id.as_str() + ); + assert_eq!( + failures[0].metadata["invocation_id"], + original.invocation_id.as_str() + ); + } + + fn has_record_artifact(result: &RunResult) -> bool { + result + .artifacts + .artifacts + .iter() + .any(|artifact| artifact.kind == RECORD_ARTIFACT_KIND) + } + + #[test] + fn finish_after_a_failed_start_line_keeps_the_result_and_writes_nothing() { + let (_, _, invocation) = record_test_fixture(); + let original = failed_result_with_usage(&invocation); + + let (finished, lines) = + finish_with_fault("start-append", Fault::StartAppend, Ok(original.clone())); + + let finished = finished.expect("result"); + assert_result_kept_with_failure_event(&finished, &original); + assert!(!has_record_artifact(&finished)); + assert!(lines.is_empty(), "{lines:?}"); + } + + #[test] + fn finish_after_a_failed_end_line_keeps_the_result_and_the_start_line() { + let (_, _, invocation) = record_test_fixture(); + let original = failed_result_with_usage(&invocation); + + let (finished, lines) = + finish_with_fault("end-append", Fault::EndAppend, Ok(original.clone())); + + let finished = finished.expect("result"); + assert_result_kept_with_failure_event(&finished, &original); + assert!(has_record_artifact(&finished)); + let kinds: Vec<&Value> = lines.iter().map(|line| &line["kind"]).collect(); + assert_eq!(kinds, ["start"]); + } + + #[test] + fn finish_after_a_failed_sync_keeps_the_result_and_both_lines() { + let (_, _, invocation) = record_test_fixture(); + let original = failed_result_with_usage(&invocation); + + let (finished, lines) = finish_with_fault("sync", Fault::Sync, Ok(original.clone())); + + let finished = finished.expect("result"); + assert_result_kept_with_failure_event(&finished, &original); + assert!(has_record_artifact(&finished)); + let kinds: Vec<&Value> = lines.iter().map(|line| &line["kind"]).collect(); + assert_eq!(kinds, ["start", "end"]); + assert_eq!(lines[1]["usage"]["input_tokens"], 3); + } + + #[test] + fn finish_returns_the_invocation_error_when_recording_also_fails() { + for fault in [Fault::StartAppend, Fault::EndAppend, Fault::Sync] { + let error = FabricError::AdapterLifecycleOperation { + operation: "invoke", + runtime_id: "rt".to_string(), + code: "host_timeout".to_string(), + message: "adapter host did not answer".to_string(), + diagnostics: String::new(), + }; + let expected = error.to_string(); + + let (finished, _) = finish_with_fault("error", fault, Err(error)); + + let error = finished.expect_err("invocation error"); + assert_eq!(error.to_string(), expected, "{fault:?}"); + assert!( + matches!(&error, FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout"), + "{fault:?}: {error}" + ); + } + } +} diff --git a/crates/fabric-core/src/record/lines.rs b/crates/fabric-core/src/record/lines.rs new file mode 100644 index 000000000..964a79eda --- /dev/null +++ b/crates/fabric-core/src/record/lines.rs @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Line types of `run-record.jsonl`, schema `nemo.fabric.run_record` version +//! `v1alpha1`. +//! +//! Every field is typed. A line holds identifiers, names, numbers, closed codes +//! and the values of a few allowlisted request context keys. It never holds a +//! value outside that allowlist: no prompt, output, error message or metadata. + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::Serialize; + +use crate::runtime::{ErrorStage, RunStatus}; + +/// Longest value a line keeps from config or the request, in characters. +/// Model role and context key names are kept whole, so map keys stay distinct. +const MAX_VALUE_CHARS: usize = 256; +/// Longest adapter error code a line keeps. +const MAX_CODE_CHARS: usize = 128; +/// Error code written in place of a code outside `^[a-z0-9_.:-]{1,128}$`. +const UNRECOGNIZED_CODE: &str = "unrecognized"; + +/// One line of a run record. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +#[schemars(title = "RunRecordLine")] +pub(crate) struct RecordLine { + /// Always `nemo.fabric.run_record`. + pub schema: RecordSchema, + /// Line format version. `v1alpha1` may change until a record reader ships. + pub version: RecordVersion, + /// Runtime id. + pub runtime_id: String, + /// Invocation id. + pub invocation_id: String, + /// Request id. + pub request_id: String, + /// UTC wall-clock time the line was written, RFC 3339 with milliseconds. + #[schemars(extend("format" = "date-time"))] + pub time: String, + /// Line kind and its fields. + #[serde(flatten)] + pub body: LineBody, +} + +/// The record's schema name. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) enum RecordSchema { + /// `nemo.fabric.run_record`. + #[serde(rename = "nemo.fabric.run_record")] + RunRecord, +} + +/// The record's line format version. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) enum RecordVersion { + /// `v1alpha1`. + #[serde(rename = "v1alpha1")] + V1Alpha1, +} + +/// A line's kind, with the fields that kind carries. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub(crate) enum LineBody { + /// Written once, before core sends the request to the adapter host. + Start(StartLine), + /// Written once, after the invocation returned a result or failed. + End(EndLine), +} + +/// Fields of the start line. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub(crate) struct StartLine { + /// Config name (`metadata.name`). + pub agent_name: String, + /// Harness identifier. + pub harness: String, + /// Adapter implementation id. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub adapter_id: Option, + /// Configured models by role: provider and model name only. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub models: BTreeMap, + /// NeMo Fabric version that wrote the record. + pub fabric_version: String, + /// Host invocation timeout that applied, in milliseconds. + pub timeout_ms: u64, + /// `RunRequest.context`. Allowlisted identity and Harbor link keys keep a + /// string or integer value; every other key is listed by name only. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub context: BTreeMap, + /// Top-level key names of `RunRequest.overrides`, sorted. Values are never + /// recorded. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub override_keys: Vec, +} + +/// One configured model. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct ModelName { + /// Model provider name. + pub provider: String, + /// Provider model identifier. + pub model: String, +} + +/// A request context value as the start line keeps it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +#[serde(untagged)] +pub(crate) enum ContextValue { + /// An allowlisted key's string value, cut to 256 characters. + Text(String), + /// An allowlisted key's integer value. + Integer(i64), + /// Any other key or value: the key name only. + NotRecorded(NotRecorded), +} + +/// Marks a key whose value the record leaves out. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct NotRecorded { + /// Always `false`. + #[schemars(extend("const" = false))] + pub recorded: bool, +} + +impl NotRecorded { + pub(crate) const VALUE: Self = Self { recorded: false }; +} + +/// Fields of the end line. +#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)] +pub(crate) struct EndLine { + /// Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`. + pub status: RunStatus, + /// Monotonic time from the start line to the end line, in milliseconds. + pub elapsed_ms: u64, + /// Error code, stage and retryable flag, without the message. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Numeric usage counters, without adapter metadata. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, + /// Result artifacts, with paths relative to the artifact root. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub artifacts: Vec, +} + +/// The end line's error. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct EndError { + /// Error code matching `^[a-z0-9_.:-]{1,128}$`, else `unrecognized`. + #[schemars(regex(pattern = r"^[a-z0-9_.:-]{1,128}$"))] + pub code: String, + /// Lifecycle stage where the failure surfaced. + pub stage: ErrorStage, + /// Whether a consumer-level retry is considered safe. + pub retryable: bool, +} + +/// The end line's usage: the numeric `RunUsage` fields. +#[derive(Debug, Clone, Default, PartialEq, Serialize, JsonSchema)] +pub(crate) struct EndUsage { + /// Input tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens: Option, + /// Cached input tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cached_input_tokens: Option, + /// Whether `input_tokens` includes `cached_input_tokens`; absent means unknown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_tokens_include_cache: Option, + /// Output tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_tokens: Option, + /// Total tokens reported by the provider. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_tokens: Option, + /// Reported cost in US dollars. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 0.0))] + pub cost_usd: Option, +} + +/// One result artifact on the end line. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, JsonSchema)] +pub(crate) struct EndArtifact { + /// Logical artifact name. + pub name: String, + /// Artifact kind. + pub kind: String, + /// Media type. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub media_type: Option, + /// Path relative to the artifact root; absent for a path outside it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, +} + +/// Cut `text` to [`MAX_VALUE_CHARS`] characters: 255 plus "…". +pub(crate) fn cut_text(text: &str) -> String { + if text.chars().count() <= MAX_VALUE_CHARS { + return text.to_string(); + } + let mut cut: String = text.chars().take(MAX_VALUE_CHARS - 1).collect(); + cut.push('…'); + cut +} + +/// An error code as the end line keeps it: unchanged when it matches +/// `^[a-z0-9_.:-]{1,128}$`, else [`UNRECOGNIZED_CODE`], because a code outside +/// that pattern may be a message. +pub(crate) fn recorded_code(code: &str) -> String { + let valid = (1..=MAX_CODE_CHARS).contains(&code.len()) + && code.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"_.:-".contains(&byte) + }); + if valid { + code.to_string() + } else { + UNRECOGNIZED_CODE.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cut_text_over_the_limit_keeps_255_chars_and_an_ellipsis() { + let cut = cut_text(&"é".repeat(1000)); + + assert_eq!(cut.chars().count(), MAX_VALUE_CHARS); + assert!(cut.ends_with('…')); + assert_eq!(cut_text("short"), "short"); + assert_eq!(cut_text(&"a".repeat(256)), "a".repeat(256)); + } + + #[test] + fn recorded_code_keeps_only_codes_in_the_pattern() { + for code in [ + "timeout", + "host_timeout", + "provider.rate_limit", + "http:429", + "a-b", + ] { + assert_eq!(recorded_code(code), code); + } + for code in [ + "", + "Timeout", + "rate limit", + "error: key sk-123", + "über", + &"a".repeat(129), + ] { + assert_eq!(recorded_code(code), UNRECOGNIZED_CODE, "{code}"); + } + assert_eq!(recorded_code(&"a".repeat(128)), "a".repeat(128)); + } +} diff --git a/crates/fabric-core/src/record/mod.rs b/crates/fabric-core/src/record/mod.rs new file mode 100644 index 000000000..4b4b2deee --- /dev/null +++ b/crates/fabric-core/src/record/mod.rs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The opt-in per-invocation run record, `run-record.jsonl`. +//! +//! When `runtime.record.enabled` is set, core appends one start line before it +//! sends the request to the adapter host and one end line after the invocation +//! returns or fails. Recording is synchronous and best-effort: record I/O +//! errors do not replace the invocation result. A failed write stops the +//! record and becomes a `record_write_failed` event on the result, and slow +//! storage delays the invocation. +//! +//! Everything here is crate-private. The line format is public only through +//! the `run-record-line` JSON Schema, version `v1alpha1`. + +mod invocation; +mod lines; +mod writer; + +pub(crate) use invocation::{InvocationRecorder, RecordStart}; +pub(crate) use lines::RecordLine; +#[cfg(test)] +pub(crate) use writer::test_hooks; diff --git a/crates/fabric-core/src/record/writer.rs b/crates/fabric-core/src/record/writer.rs new file mode 100644 index 000000000..ea95049cf --- /dev/null +++ b/crates/fabric-core/src/record/writer.rs @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The run record file: created once, owner-only, appended one line per write. +//! +//! Each line is one `write_all` on an unbuffered file, so a SIGKILL of Fabric +//! or the adapter host keeps every line whose write returned. Only the end +//! line is synced, once, so the record survives a crash of the machine only +//! up to its last synced line. + +use std::fs::{File, OpenOptions}; +use std::io::{self, Write}; +use std::path::Path; + +use super::lines::RecordLine; + +/// File name of the record inside the invocation folder. +pub(crate) const RECORD_FILE_NAME: &str = "run-record.jsonl"; + +/// Appends lines to one invocation's record. +#[derive(Debug)] +pub(crate) struct RecordWriter { + file: File, +} + +impl RecordWriter { + /// Create the record at `path`. `create_new` refuses an existing file or + /// symbolic link at `path`, and on Unix the file is mode 0600. + pub(crate) fn create(path: &Path) -> io::Result { + #[cfg(test)] + std::thread::sleep(test_hooks::CREATE_DELAY.with(std::cell::Cell::get)); + let mut options = OpenOptions::new(); + options.append(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + Ok(Self { + file: options.open(path)?, + }) + } + + /// Append `line` and its newline with one write. + pub(crate) fn append(&mut self, line: &RecordLine) -> io::Result<()> { + #[cfg(test)] + test_hooks::fail_append(&line.body)?; + let mut bytes = serde_json::to_vec(line).map_err(io::Error::other)?; + bytes.push(b'\n'); + self.file.write_all(&bytes) + } + + /// Flush the file's data and metadata to stable storage. + pub(crate) fn sync(&self) -> io::Result<()> { + #[cfg(test)] + test_hooks::fail_if(test_hooks::Fault::Sync)?; + self.file.sync_all() + } +} + +/// Per-thread hooks that make record I/O slow or fail in tests. +#[cfg(test)] +pub(crate) mod test_hooks { + use std::cell::Cell; + use std::io; + use std::time::Duration; + + use super::super::lines::LineBody; + + /// A record operation that a test makes fail. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum Fault { + StartAppend, + EndAppend, + Sync, + } + + thread_local! { + /// Delay added before each record file is created on this thread. + pub(crate) static CREATE_DELAY: Cell = const { Cell::new(Duration::ZERO) }; + /// Operation that fails on this thread. + pub(crate) static FAULT: Cell> = const { Cell::new(None) }; + } + + pub(super) fn fail_append(body: &LineBody) -> io::Result<()> { + match body { + LineBody::Start(_) => fail_if(Fault::StartAppend), + LineBody::End(_) => fail_if(Fault::EndAppend), + } + } + + pub(super) fn fail_if(fault: Fault) -> io::Result<()> { + if FAULT.with(Cell::get) == Some(fault) { + return Err(io::Error::other(format!("injected {fault:?} failure"))); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_folder(name: &str) -> std::path::PathBuf { + let folder = std::env::temp_dir().join(format!( + "fabric-record-writer-{name}-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&folder); + std::fs::create_dir_all(&folder).expect("create temp folder"); + folder + } + + #[test] + fn record_writer_refuses_an_existing_file_or_a_symlink() { + let folder = temp_folder("excl"); + let path = folder.join(RECORD_FILE_NAME); + std::fs::write(&path, "old").expect("seed file"); + + let error = RecordWriter::create(&path).expect_err("existing file"); + + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!(std::fs::read_to_string(&path).expect("read"), "old"); + #[cfg(unix)] + { + let linked = folder.join("linked"); + std::fs::create_dir_all(&linked).expect("linked folder"); + let target = folder.join("target"); + std::os::unix::fs::symlink(&target, linked.join(RECORD_FILE_NAME)).expect("symlink"); + assert!(RecordWriter::create(&linked.join(RECORD_FILE_NAME)).is_err()); + assert!(!target.exists()); + } + let _ = std::fs::remove_dir_all(folder); + } + + #[cfg(unix)] + #[test] + fn record_writer_new_file_is_owner_only() { + use std::os::unix::fs::PermissionsExt; + let folder = temp_folder("mode"); + let path = folder.join(RECORD_FILE_NAME); + + RecordWriter::create(&path).expect("create"); + let mode = std::fs::metadata(&path).expect("stat").permissions().mode(); + + assert_eq!(mode & 0o777, 0o600); + let _ = std::fs::remove_dir_all(folder); + } +} diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 3c3463596..6d473abc8 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -32,12 +32,15 @@ use crate::config::{ }; use crate::environment::resolve_environment_provider; use crate::error::{FabricError, Result}; +use crate::record::{InvocationRecorder, RecordStart}; mod openshell; use openshell::OpenShellRuntimeAdapter; static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; +/// Lifecycle error code for an adapter result that core rejects. +pub(crate) const INVALID_AGENT_RUN_RESULT: &str = "invalid_agent_run_result"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; const LOCAL_HOST_START_TIMEOUT: Duration = Duration::from_secs(90); // Protocol liveness backstop. Adapters remain responsible for normal request @@ -1898,11 +1901,39 @@ impl LocalHostInvocation { } fn run_local_host_invocation_with_timeout( + plan: &RunPlan, + runtime: &RuntimeHandle, + request: RunRequest, + invocation_kind: LocalHostInvocation, + invoke_timeout: Duration, +) -> Result { + let mut recorder = None; + let outcome = local_host_invocation_result( + plan, + runtime, + request, + invocation_kind, + invoke_timeout, + &mut recorder, + ); + match recorder { + // The end line and its sync run here, after the host lock is released. + Some(recorder) => recorder.finish(outcome), + None => outcome, + } +} + +/// Run one invocation on the local host. When the plan turns the run record +/// on, `recorder` is set before core sets the adapter deadline and writes +/// `adapter-invocation.json`, so every invocation that core tried to send has +/// a record. +fn local_host_invocation_result( plan: &RunPlan, runtime: &RuntimeHandle, mut request: RunRequest, invocation_kind: LocalHostInvocation, invoke_timeout: Duration, + recorder: &mut Option, ) -> Result { let operation = invocation_kind.operation(); if request.request_id.is_empty() { @@ -1944,6 +1975,18 @@ fn run_local_host_invocation_with_timeout( &host_guard.environment_env, &adapter_invocation.runtime_context.environment.env, )?; + // Record I/O runs before the adapter deadline is set, so slow record + // storage delays the request instead of shortening the adapter's budget. + if plan.config.runtime.record_enabled() && artifacts.root.is_some() { + *recorder = Some(InvocationRecorder::start(RecordStart { + plan, + runtime, + invocation: &invocation, + request: &request, + fabric_home: &fabric_home, + timeout: invoke_timeout, + })); + } adapter_invocation.runtime_context.deadline_millis = adapter_deadline_millis(invoke_timeout); let (lifecycle_request, adapter_payload) = match invocation_kind { @@ -2023,7 +2066,7 @@ fn run_local_host_invocation_with_timeout( lifecycle_error( operation, &runtime.runtime_id, - "invalid_agent_run_result", + INVALID_AGENT_RUN_RESULT, format!("adapter returned an invalid AgentRunResult: {error}"), &stderr, ) @@ -2032,7 +2075,7 @@ fn run_local_host_invocation_with_timeout( lifecycle_error( operation, &runtime.runtime_id, - "invalid_agent_run_result", + INVALID_AGENT_RUN_RESULT, format!("adapter returned an invalid AgentRunResult: {error}"), &stderr, ) @@ -2042,7 +2085,7 @@ fn run_local_host_invocation_with_timeout( return Err(lifecycle_error( operation, &runtime.runtime_id, - "invalid_agent_run_result", + INVALID_AGENT_RUN_RESULT, "adapter returned artifacts without a configured runtime artifact root", &stderr, )); @@ -3475,7 +3518,7 @@ fn relay_artifact_media_type(kind: &str, format: Option<&str>) -> Option<&'stati } } -fn unique_artifact_name(manifest: &ArtifactManifest, base: &str) -> String { +pub(crate) fn unique_artifact_name(manifest: &ArtifactManifest, base: &str) -> String { if !manifest .artifacts .iter() @@ -3508,9 +3551,26 @@ fn process_command_args(plan: &RunPlan, settings: &ProcessAdapterSettings) -> Ve args } -fn artifact_manifest(plan: &RunPlan) -> Result { - let root = plan - .config +/// The folder that holds a plan's run records, `.fabric` under its artifact +/// root, when `runtime.record.enabled` is set. `None` otherwise. Plan +/// validation guarantees that such a plan has an artifact root, the `local` +/// environment provider and a local adapter host. +pub fn run_record_folder(plan: &RunPlan) -> Option { + if !plan.config.runtime.record_enabled() { + return None; + } + artifact_root(plan).map(|root| fabric_folder(&root)) +} + +/// Core's own folder under an artifact root. +fn fabric_folder(root: &Path) -> PathBuf { + root.join(".fabric") +} + +/// The runtime's artifact root resolved against the plan's base folder, else +/// the environment's artifact root. +fn artifact_root(plan: &RunPlan) -> Option { + plan.config .runtime .artifacts .as_ref() @@ -3519,7 +3579,11 @@ fn artifact_manifest(plan: &RunPlan) -> Result { plan.environment_plan .as_ref() .and_then(|environment| environment.artifacts.clone()) - }); + }) +} + +fn artifact_manifest(plan: &RunPlan) -> Result { + let root = artifact_root(plan); if let Some(root) = &root { std::fs::create_dir_all(root).map_err(|source| FabricError::Write { path: root.clone(), @@ -3541,8 +3605,7 @@ fn prepare_fabric_home( .root .clone() .unwrap_or_else(|| std::env::temp_dir().join("nemo-fabric")); - let fabric_home = root - .join(".fabric") + let fabric_home = fabric_folder(&root) .join(&runtime.runtime_id) .join(&invocation.invocation_id); std::fs::create_dir_all(&fabric_home).map_err(|source| FabricError::Write { @@ -3840,7 +3903,7 @@ fn telemetry_ref( }) } -fn event_with_metadata( +pub(crate) fn event_with_metadata( kind: impl Into, message: impl Into, metadata: BTreeMap, @@ -4129,12 +4192,16 @@ for line in sys.stdin: "context_env": invocation["runtime_context"]["environment"].get("env", {}), "deadline_millis": invocation["runtime_context"].get("deadline_millis"), } - if MODE in {"adapter_reported_failure", "blank_error_code"}: + if MODE in {"adapter_reported_failure", "blank_error_code", "free_text_error_code"}: + codes = { + "blank_error_code": " \t", + "free_text_error_code": "Rate limit for key sk-live-123 exceeded", + } result = { "status": "failed", "output": output, "error": { - "code": " \t" if MODE == "blank_error_code" else "fake_adapter_failure", + "code": codes.get(MODE, "fake_adapter_failure"), "message": "adapter rejected the invocation", "retryable": True, "extensions": {"source": "fake-host"}, @@ -5721,6 +5788,293 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + + fn record_plan(mode: &str) -> (PathBuf, RunPlan) { + let (root, mut plan) = local_host_plan(mode); + plan.config.runtime.record = Some(crate::config::RecordConfig { enabled: true }); + (root, plan) + } + + #[test] + fn run_record_folder_is_set_only_when_the_record_is_on() { + let (root, mut plan) = record_plan("success"); + let artifact_root = resolve_path( + &plan.base_dir, + plan.config.runtime.artifacts.as_ref().expect("artifacts"), + ); + assert_eq!( + run_record_folder(&plan), + Some(artifact_root.join(".fabric")) + ); + + plan.config.runtime.record = None; + assert_eq!(run_record_folder(&plan), None, "record off"); + let _ = fs::remove_dir_all(root); + } + + fn record_files(root: &Path) -> Vec { + let mut records = Vec::new(); + let mut folders = vec![root.join("artifacts/.fabric")]; + while let Some(folder) = folders.pop() { + let Ok(entries) = fs::read_dir(&folder) else { + continue; + }; + for path in entries.flatten().map(|entry| entry.path()) { + if path.is_dir() { + folders.push(path); + } else if path.ends_with("run-record.jsonl") { + records.push(path); + } + } + } + records + } + + fn record_line_validator() -> jsonschema::Validator { + let schema = crate::schema::generate_schema(crate::schema::SchemaName::RunRecordLine) + .expect("record line schema"); + jsonschema::validator_for(&schema).expect("valid schema") + } + + /// The only record under `root`, with every line checked against the + /// published line schema. + fn only_record(root: &Path) -> (String, Vec) { + let records = record_files(root); + assert_eq!(records.len(), 1, "{records:?}"); + let text = fs::read_to_string(&records[0]).expect("read record"); + let validator = record_line_validator(); + let lines: Vec = text + .lines() + .map(|line| serde_json::from_str(line).expect("record line is JSON")) + .collect(); + for line in &lines { + assert!(validator.is_valid(line), "{line}"); + } + (text, lines) + } + + /// The one line of `kind`. Lookup by kind keeps these tests independent + /// of other line kinds written between the start and end lines. + fn line_of_kind<'a>(lines: &'a [Value], kind: &str) -> &'a Value { + let mut matching = lines.iter().filter(|line| line["kind"] == kind); + let line = matching.next().unwrap_or_else(|| panic!("no {kind} line")); + assert!(matching.next().is_none(), "more than one {kind} line"); + line + } + + #[test] + fn record_off_with_an_artifact_root_writes_no_record() { + let (root, plan) = local_host_plan("typed_result"); + assert!(plan.config.runtime.artifacts.is_some()); + + let result = run_plan(&plan, RunRequest::text("no record")).expect("run"); + + assert!(record_files(&root).is_empty()); + assert!( + result + .artifacts + .artifacts + .iter() + .all(|artifact| artifact.kind != "run_record") + ); + assert!( + result + .events + .iter() + .all(|event| event.kind != "record_write_failed") + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_on_writes_a_start_line_and_an_end_line() { + let (root, plan) = record_plan("typed_result"); + let mut request = RunRequest::text("record"); + request.context = BTreeMap::from([ + ("task_id".to_string(), serde_json::json!("repo-io-fix")), + ("attempt".to_string(), serde_json::json!(2)), + ( + "harbor_trial_name".to_string(), + serde_json::json!("trial-1"), + ), + ("step".to_string(), serde_json::json!({"nested": "value"})), + ("api_key".to_string(), serde_json::json!("sk-live-SECRET")), + ]); + + let result = run_plan(&plan, request).expect("run"); + let (text, lines) = only_record(&root); + + let (start, end) = (line_of_kind(&lines, "start"), line_of_kind(&lines, "end")); + assert_eq!(lines.first(), Some(start), "{text}"); + assert_eq!(lines.last(), Some(end), "{text}"); + for line in &lines { + assert_eq!(line["schema"], "nemo.fabric.run_record"); + assert_eq!(line["version"], "v1alpha1"); + assert_eq!(line["runtime_id"], result.runtime_id.as_str()); + assert_eq!(line["invocation_id"], result.invocation_id.as_str()); + assert_eq!(line["request_id"], result.request_id.as_str()); + } + assert_eq!(start["agent_name"], "local-host-test-agent"); + assert_eq!(start["adapter_id"], "acme.fabric.local-host"); + assert_eq!(start["timeout_ms"], 3_600_000); + assert_eq!( + start["context"], + serde_json::json!({ + "api_key": {"recorded": false}, + "attempt": 2, + "harbor_trial_name": "trial-1", + "step": {"recorded": false}, + "task_id": "repo-io-fix", + }) + ); + assert!(!text.contains("SECRET"), "{text}"); + assert_eq!(end["status"], "succeeded"); + assert!(end["elapsed_ms"].is_u64()); + assert_eq!( + end["usage"], + serde_json::json!({ + "input_tokens": 3, + "cached_input_tokens": 1, + "input_tokens_include_cache": true, + "output_tokens": 5, + "total_tokens": 8, + "cost_usd": 0.25, + }) + ); + let report = end["artifacts"] + .as_array() + .expect("artifacts") + .iter() + .find(|artifact| artifact["name"] == "report") + .expect("report artifact"); + assert_eq!( + report, + &serde_json::json!({ + "name": "report", + "kind": "report", + "media_type": "application/json", + "path": "reports/result.json", + }) + ); + for marker in ["fake-host", "\"fake\"", "trace-1"] { + assert!(!text.contains(marker), "{marker} in {text}"); + } + let record = result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.name == "record") + .expect("record artifact"); + assert_eq!(record.kind, "run_record"); + assert_eq!(record.media_type.as_deref(), Some("application/jsonl")); + assert_eq!(record.path, record_files(&root)[0]); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_end_line_keeps_the_error_code_and_drops_the_message() { + let (root, plan) = record_plan("adapter_reported_failure"); + + let result = run_plan(&plan, RunRequest::text("fail")).expect("failed result"); + let (text, lines) = only_record(&root); + + assert_eq!(result.status, RunStatus::Failed); + assert_eq!( + line_of_kind(&lines, "end")["error"], + serde_json::json!({"code": "fake_adapter_failure", "stage": "invoke", "retryable": true}) + ); + assert!(!text.contains("adapter rejected"), "{text}"); + let mut wrong_kind = line_of_kind(&lines, "end").clone(); + wrong_kind["kind"] = serde_json::json!("start"); + assert!(!record_line_validator().is_valid(&wrong_kind)); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_end_line_replaces_a_free_text_error_code() { + let (root, plan) = record_plan("free_text_error_code"); + + let result = run_plan(&plan, RunRequest::text("fail")).expect("failed result"); + let (text, lines) = only_record(&root); + + assert!(result.error.expect("error").code.contains("sk-live-123")); + assert_eq!(line_of_kind(&lines, "end")["error"]["code"], "unrecognized"); + assert!(!text.contains("sk-live-123"), "{text}"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_lifecycle_error_writes_a_failed_end_line() { + let (root, plan) = record_plan("invoke_timeout"); + let runtime = start_runtime(&plan).expect("start local host"); + + let error = run_local_host_adapter_with_timeout( + &plan, + &runtime, + RunRequest::text("hang"), + Duration::from_millis(100), + ) + .expect_err("timeout raises"); + let (_, lines) = only_record(&root); + + assert!(error.to_string().contains("host_timeout"), "{error}"); + let end = line_of_kind(&lines, "end"); + assert_eq!(line_of_kind(&lines, "start")["timeout_ms"], 100); + assert_eq!(end["status"], "failed"); + assert_eq!( + end["error"], + serde_json::json!({"code": "host_timeout", "stage": "invoke", "retryable": false}) + ); + let _ = stop_runtime(&plan, &runtime); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_preparation_does_not_shorten_the_adapter_deadline() { + let (root, mut plan) = record_plan("success"); + plan.config.runtime.timeout_seconds = Some(60.0); + let runtime = start_runtime(&plan).expect("start local host"); + let delay = Duration::from_millis(500); + crate::record::test_hooks::CREATE_DELAY.with(|slot| slot.set(delay)); + + let before = now_millis(); + let result = invoke_runtime(&plan, &runtime, RunRequest::text("slow record storage")); + let after = now_millis(); + crate::record::test_hooks::CREATE_DELAY.with(|slot| slot.set(Duration::ZERO)); + + let result = result.expect("invoke"); + let deadline = u128::from(result.output["deadline_millis"].as_u64().expect("deadline")); + // The adapter's 50-second budget starts after the record is created, + // so slow record storage does not eat into it. + assert!( + (before + delay.as_millis() + 50_000..=after + 50_000).contains(&deadline), + "deadline {deadline}, before {before}, after {after}" + ); + assert_eq!(record_files(&root).len(), 1); + stop_runtime(&plan, &runtime).expect("stop local host"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_warm_runtime_writes_one_record_per_invocation() { + let (root, plan) = record_plan("success"); + let runtime = start_runtime(&plan).expect("start local host"); + + let first = invoke_runtime(&plan, &runtime, RunRequest::text("one")).expect("first"); + let second = invoke_runtime(&plan, &runtime, RunRequest::text("two")).expect("second"); + stop_runtime(&plan, &runtime).expect("stop"); + + let records = record_files(&root); + assert_eq!(records.len(), 2); + for result in [&first, &second] { + assert!(records.iter().any(|path| path.ends_with(format!( + ".fabric/{}/{}/run-record.jsonl", + result.runtime_id, result.invocation_id + )))); + } + let _ = fs::remove_dir_all(root); + } + #[test] fn default_environment_prepares_through_local_provider() { let (root, mut plan) = local_host_plan("success"); diff --git a/crates/fabric-core/src/schema.rs b/crates/fabric-core/src/schema.rs index ec329cf49..240b22a8b 100644 --- a/crates/fabric-core/src/schema.rs +++ b/crates/fabric-core/src/schema.rs @@ -13,6 +13,7 @@ use crate::config::{ AdapterDescriptor, AdapterTargetDescriptor, AgentConfig, FabricConfig, RunPlan, }; use crate::error::{FabricError, Result}; +use crate::record::RecordLine; use crate::runtime::{ AdapterInvocation, ArtifactManifest, EnvironmentHandle, EnvironmentReference, ErrorInfo, FabricEvent, InvocationHandle, OpenAiStreamInvocation, OpenAiStreamRecord, RunRequest, @@ -67,11 +68,13 @@ pub enum SchemaName { ErrorInfo, /// NeMo Fabric lifecycle event schema. FabricEvent, + /// One line of an opt-in per-invocation run record, version `v1alpha1`. + RunRecordLine, } impl SchemaName { /// All public schemas in stable output order. - pub const ALL: [Self; 22] = [ + pub const ALL: [Self; 23] = [ Self::Agent, Self::AgentConfig, Self::AgentRunRequest, @@ -94,6 +97,7 @@ impl SchemaName { Self::ArtifactManifest, Self::ErrorInfo, Self::FabricEvent, + Self::RunRecordLine, ]; /// Stable file stem for this schema. @@ -121,6 +125,7 @@ impl SchemaName { Self::ArtifactManifest => "artifact-manifest", Self::ErrorInfo => "error-info", Self::FabricEvent => "fabric-event", + Self::RunRecordLine => "run-record-line", } } @@ -175,6 +180,7 @@ impl SchemaName { "artifact-manifest" | "artifact_manifest" => Ok(Self::ArtifactManifest), "error-info" | "error_info" => Ok(Self::ErrorInfo), "fabric-event" | "fabric_event" => Ok(Self::FabricEvent), + "run-record-line" | "run_record_line" => Ok(Self::RunRecordLine), _ => Err(FabricError::UnknownSchema { schema: value.to_string(), available: Self::ALL @@ -211,6 +217,7 @@ pub fn generate_schema(schema: SchemaName) -> Result { SchemaName::ArtifactManifest => to_value(schema_for!(ArtifactManifest)), SchemaName::ErrorInfo => to_value(schema_for!(ErrorInfo)), SchemaName::FabricEvent => to_value(schema_for!(FabricEvent)), + SchemaName::RunRecordLine => to_value(schema_for!(RecordLine)), } } diff --git a/crates/fabric-core/src/time.rs b/crates/fabric-core/src/time.rs new file mode 100644 index 000000000..06ace9edf --- /dev/null +++ b/crates/fabric-core/src/time.rs @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! RFC 3339 UTC timestamps with millisecond precision. +//! +//! Core has no date crate, so this converts Unix milliseconds to a civil date +//! with Howard Hinnant's `civil_from_days` algorithm. + +use std::time::{SystemTime, UNIX_EPOCH}; + +const MILLIS_PER_DAY: i64 = 86_400_000; + +/// Format `time` as RFC 3339 UTC with three fraction digits, such as +/// `2026-10-14T18:01:10.205Z`. +pub(crate) fn rfc3339_millis(time: SystemTime) -> String { + format_unix_millis(unix_millis(time)) +} + +/// Milliseconds since the Unix epoch; negative before it. +fn unix_millis(time: SystemTime) -> i64 { + match time.duration_since(UNIX_EPOCH) { + Ok(after) => i64::try_from(after.as_millis()).unwrap_or(i64::MAX), + Err(before) => -i64::try_from(before.duration().as_millis()).unwrap_or(i64::MAX), + } +} + +fn format_unix_millis(unix_millis: i64) -> String { + let days = unix_millis.div_euclid(MILLIS_PER_DAY); + let millis_of_day = unix_millis.rem_euclid(MILLIS_PER_DAY); + let (year, month, day) = civil_from_days(days); + let seconds_of_day = millis_of_day / 1000; + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{:03}Z", + seconds_of_day / 3600, + seconds_of_day / 60 % 60, + seconds_of_day % 60, + millis_of_day % 1000, + ) +} + +fn civil_from_days(days: i64) -> (i64, i64, i64) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + (year, month, day) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn rfc3339_millis_formats_fixed_vectors() { + assert_eq!(format_unix_millis(0), "1970-01-01T00:00:00.000Z"); + assert_eq!( + format_unix_millis(1_792_000_870_205), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!( + format_unix_millis(951_782_400_000), + "2000-02-29T00:00:00.000Z" + ); + assert_eq!( + format_unix_millis(4_107_542_399_999), + "2100-02-28T23:59:59.999Z" + ); + assert_eq!(format_unix_millis(-1), "1969-12-31T23:59:59.999Z"); + assert_eq!( + rfc3339_millis(UNIX_EPOCH + Duration::from_millis(1_792_000_870_205)), + "2026-10-14T18:01:10.205Z" + ); + assert_eq!( + rfc3339_millis(UNIX_EPOCH - Duration::from_millis(1)), + "1969-12-31T23:59:59.999Z" + ); + } +} diff --git a/docs/experimentation/cli.mdx b/docs/experimentation/cli.mdx index c75e88dc1..e0dd0734e 100644 --- a/docs/experimentation/cli.mdx +++ b/docs/experimentation/cli.mdx @@ -149,6 +149,27 @@ still requires an Anthropic Messages-compatible model, while a Codex preset requires a Responses-compatible model. The `plan` and `doctor` commands accept the same overrides. +Write a run record and label the run with four switches: + +```bash +nemo-fabric run --preset hermes --input "Say hello" \ + --record --artifacts ./runs --name hermes-baseline --task say-hello +``` + +`--record` sets `runtime.record.enabled`, and `--artifacts` sets +`runtime.artifacts`, resolved against the current folder. The invocation then +writes `./runs/.fabric///run-record.jsonl` beside its +`stdout.txt`. `--name` sets the config name (`metadata.name`), and `--task` sets +`task_id` in the request context; the record's start line keeps both, and core +forwards `task_id` to the adapter in the request context, as it forwards every +context key. The `plan` and `doctor` commands +accept `--record`, `--artifacts`, and `--name`, so they show the config that +`run` executes; `--task` changes only the request, so it is a `run` switch. +When the run writes a record, the CLI prints +`nemo-fabric: run record under ` to standard error before the run +starts. `--record` needs an artifact root: the presets set none, so without +`--artifacts` planning fails on `runtime.record.enabled`. + ## Experiment With Examples List the maintained workflows and inspect an example's available variants: diff --git a/docs/integrations/consumer/harbor.mdx b/docs/integrations/consumer/harbor.mdx index fd053963a..f20bf8004 100644 --- a/docs/integrations/consumer/harbor.mdx +++ b/docs/integrations/consumer/harbor.mdx @@ -34,3 +34,41 @@ The existing `FabricAgent` bridge follows this flow for `fabric_environment_env` `environment_env_names` is Harbor's existing transport field. `environment.env_names` is the core NeMo Fabric forwarding field described in [Set Environment Values Or Forward Them By Name](../../sdk/python.mdx#set-environment-values-or-forward-them-by-name). This addition does not migrate the Harbor bridge to the core field: Harbor's retained run specification contains names, but its runner still puts resolved values in `environment.env`, where values are serialized into the NeMo Fabric run plan. Keep using Harbor's managed environment and redaction rules for credentials. Core `env_names` does not replace Harbor's secret delivery or redaction. + +## Trial Identity In Run Records + +Set `fabric_record=True` on `FabricAgent` to write a NeMo Fabric run record for +each agent run in a trial (one per step on a multi-step trial) under the +bridge's artifact root. It is off by default, and without +it the bridge sends the same request as before. With it on, the bridge reads +the trial's `lock.json` and `config.json`, which Harbor writes to the trial +folder before the agent starts, and adds these keys to the request context: + +| Key | Source | +| --- | --- | +| `task_id` | The task name in `lock.json`: `org/name` for a registry task, and the task folder's name for a git or local task | +| `task_revision` | Harbor's content digest of the task (`sha256:...`) in `lock.json`, for every task kind | +| `harbor_trial_name` | The trial's `trial_name` in `config.json` | +| `step` | On a multi-step trial only (Harbor creates the trial's `steps` folder), the bridge's run count in the trial: 1 for the first step, 2 for the second, and so on | + +The record's start line keeps them with `harbor_session_id` and +`harbor_context_id`. NeMo Fabric forwards these keys to the adapter, as it +forwards every request context key; no first-party adapter reads them. Harbor +runs each attempt as its own trial and gives the trial no attempt index, so the +bridge sets no `attempt`. Group a task's attempts by `task_id` and +`task_revision`. A missing or unreadable `lock.json` or `config.json`, or one +that does not hold a JSON object, adds none of its keys, and the run goes on. +Otherwise the bridge copies `task.name`, `task.digest`, and `trial_name` each on +its own when it is a nonempty string, and does not check the file against +Harbor's lock or config model. + +On the host, each record is at +`/agent/fabric-artifacts//.fabric///run-record.jsonl`, +where the config name is `harbor-` followed by the last part of the adapter +ID, such as `harbor-hermes`. On a multi-step trial, Harbor moves or copies the +agent folder's contents into `/steps//agent/` after each step, so +each step's records are there. + +`fabric_record=True` needs a NeMo Fabric version with `runtime.record` in the +trial environment. With the first-party adapters, an older `fabric_package` +rejects the config. diff --git a/docs/reference/api/python-library-reference/index.md b/docs/reference/api/python-library-reference/index.md index 4f65cae06..4190fea98 100644 --- a/docs/reference/api/python-library-reference/index.md +++ b/docs/reference/api/python-library-reference/index.md @@ -42,6 +42,7 @@ SPDX-License-Identifier: Apache-2.0 --> - [`models.McpServerConfig`](./nemo_fabric.models.md#class-mcpserverconfig): MCP server configuration. - [`models.MetadataConfig`](./nemo_fabric.models.md#class-metadataconfig): Human-readable agent identity. - [`models.ModelConfig`](./nemo_fabric.models.md#class-modelconfig): Configuration for one model role. +- [`models.RecordConfig`](./nemo_fabric.models.md#class-recordconfig): Opt-in per-invocation run record. - [`models.RelayAtifConfig`](./nemo_fabric.models.md#class-relayatifconfig): NeMo Relay ATIF export configuration. - [`models.RelayAtofConfig`](./nemo_fabric.models.md#class-relayatofconfig): NeMo Relay ATOF export configuration. - [`models.RelayAtofFileSinkConfig`](./nemo_fabric.models.md#class-relayatoffilesinkconfig): NeMo Relay ATOF file sink configuration. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.models.md b/docs/reference/api/python-library-reference/nemo_fabric.models.md index db3a42208..703cff7b4 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.models.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.models.md @@ -472,6 +472,78 @@ Returns the set of fields that have been explicitly set on this model instance. +**Returns:** + A set of strings representing the fields that have been set, i.e. that were not filled from defaults. + + + +--- + + +### classmethod `from_mapping` + +```python +def from_mapping(value: Mapping[str, Any]) -> Self +``` + +Validate a mapping using this Pydantic model. + +--- + + +### method `to_mapping` + +```python +def to_mapping() -> dict[str, Any] +``` + +Return a detached JSON-compatible mapping for Rust/core calls. + + +--- + + +## class `RecordConfig` + +Opt-in per-invocation run record. + +When ``enabled`` is true, NeMo Fabric writes ``/.fabric///run-record.jsonl`` for each invocation. It requires an artifact root, the ``local`` environment provider, and a ``process`` or ``python`` adapter. + + + +### Fields + +The model defines the following fields: + +| Field | Type | Required | Default | Constraints | Description | +| --- | --- | --- | --- | --- | --- | +| `enabled` | `StrictBool` | Yes | — | `Strict(strict=True)` | — | + +--- + +### property extra_fields + +Return fields preserved by the extension point for this model. + +--- + +### property model_extra + +Get extra fields set during validation. + + + +**Returns:** + A dictionary of extra fields, or `None` if `config.extra` is not set to `"allow"`. + +--- + +### property model_fields_set + +Returns the set of fields that have been explicitly set on this model instance. + + + **Returns:** A set of strings representing the fields that have been set, i.e. that were not filled from defaults. @@ -520,6 +592,7 @@ The model defines the following fields: | `artifacts` | `str \| Path \| None` | No | `None` | — | — | | `timeout_seconds` | `float \| None` | No | `None` | `Gt(gt=0), _PydanticGeneralMetadata(allow_inf_nan=False)` | — | | `max_turns` | `int \| None` | No | `None` | `Gt(gt=0), Le(le=4294967295)` | — | +| `record` | `RecordConfig \| None` | No | `None` | — | — | --- diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index 52a3b6f57..71771291a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 125 +position: 131 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 26a286a0c..a6a947127 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 126 +position: 132 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index 2c89a0e39..630b1a01b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 127 +position: 133 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx index 40e387ace..65c0f3eaf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitykind.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Kind" sidebar-title: "CapabilityKind" description: "Capability kind." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx index e121e6b73..010851195 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-capabilitytarget.mdx @@ -2,7 +2,7 @@ title: "Enum Capability Target" sidebar-title: "CapabilityTarget" description: "Component responsible for executing a configured capability." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx index 74dc754f8..b1e6d19ac 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatifstorageconfig.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atif Storage Config" sidebar-title: "RelayAtifStorageConfig" description: "Relay ATIF remote storage configuration." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx index bbf8ff9e5..2e4357780 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofmode.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Mode" sidebar-title: "RelayAtofMode" description: "Relay ATOF file mode." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx index d53f00f8d..d699b5db0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofsinkconfig.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Sink Config" sidebar-title: "RelayAtofSinkConfig" description: "Relay ATOF sink configuration." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx index 892413a9e..9ee176668 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamfieldnamepolicy.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Field Name Policy" sidebar-title: "RelayAtofStreamFieldNamePolicy" description: "Relay ATOF stream field-name policy." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx index ea48753dd..a90cec5f3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayatofstreamtransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Atof Stream Transport" sidebar-title: "RelayAtofStreamTransport" description: "Relay ATOF stream transport." -position: 80 +position: 81 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx index beb3da6eb..ba9321f6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetryendpointtype.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Endpoint Type" sidebar-title: "RelayOpenTelemetryEndpointType" description: "Relay OpenTelemetry span projection." -position: 81 +position: 82 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx index da13057aa..9b78e9a7e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayopentelemetrymarkprojection.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Open Telemetry Mark Projection" sidebar-title: "RelayOpenTelemetryMarkProjection" description: "Relay OpenTelemetry mark projection policy." -position: 82 +position: 83 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx index 8db6e0872..a73f2ad1b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayotlptransport.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Otlp Transport" sidebar-title: "RelayOtlpTransport" description: "Relay OTLP transport." -position: 83 +position: 84 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx index 95ec26616..506a2c506 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-relayunsupportedbehavior.mdx @@ -2,7 +2,7 @@ title: "Enum Relay Unsupported Behavior" sidebar-title: "RelayUnsupportedBehavior" description: "Relay unsupported/unknown config handling." -position: 84 +position: 85 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx index f130d1764..7e6f001b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-resolutionstrategy.mdx @@ -2,7 +2,7 @@ title: "Enum Resolution Strategy" sidebar-title: "ResolutionStrategy" description: "Adapter install or availability strategy." -position: 54 +position: 55 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx index 8ce5c9908..d1bfc189f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/enum-telemetryprovider.mdx @@ -2,7 +2,7 @@ title: "Enum Telemetry Provider" sidebar-title: "TelemetryProvider" description: "Telemetry runtime provider." -position: 64 +position: 65 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx index fc493bae7..de34aedc1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-descriptor.mdx @@ -2,7 +2,7 @@ title: "Function load_adapter_descriptor" sidebar-title: "load_adapter_descriptor" description: "Load an adapter descriptor from JSON package metadata." -position: 71 +position: 72 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx index 142fdba30..bd5cd97d5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-load-adapter-target-descriptor.mdx @@ -2,7 +2,7 @@ title: "Function load_adapter_target_descriptor" sidebar-title: "load_adapter_target_descriptor" description: "Load an Adapter Target Descriptor from declarative JSON metadata." -position: 72 +position: 73 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx index 6bc8a973b..a6d607703 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/fn-resolve-run-plan-from-config.mdx @@ -2,7 +2,7 @@ title: "Function resolve_run_plan_from_config" sidebar-title: "resolve_run_plan_from_config" description: "Resolve a typed NVIDIA NeMo Fabric config into a runnable plan." -position: 73 +position: 74 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index fa7255361..98bec48af 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 128 +position: 134 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -53,6 +53,7 @@ NeMo Fabric config models and loading helpers. - [McpServerPlan](struct-mcpserverplan.mdx): Resolved MCP server exposure. - [MetadataConfig](struct-metadataconfig.mdx): Human-readable metadata. - [ModelConfig](struct-modelconfig.mdx): Model configuration. +- [RecordConfig](struct-recordconfig.mdx): Per-invocation run record settings. - [RelayAtifConfig](struct-relayatifconfig.mdx): Relay ATIF export configuration. - [RelayAtofConfig](struct-relayatofconfig.mdx): Relay ATOF export configuration. - [RelayComponentConfig](struct-relaycomponentconfig.mdx): Generic NeMo Relay plugin component configuration. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx new file mode 100644 index 000000000..d33e42889 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-recordconfig.mdx @@ -0,0 +1,108 @@ +--- +title: "Struct Record Config" +sidebar-title: "RecordConfig" +description: "Per-invocation run record settings." +position: 54 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
bool,\n}"}} />
+ +Per-invocation run record settings. + +When enabled, core appends a start line and an end line for each local invocation to `/.fabric///run-record.jsonl`. + +## Fields + +### `enabled: bool` + +Whether to write a run record for each invocation. Requires an artifact root, the `local` environment provider, and a `process` or `python` adapter. + +## Trait Implementations + +### `impl Clone for RecordConfig` + +
Clone for RecordConfig"}} />
+ +#### `clone` + +
clone(&self) -> RecordConfig"}} />
+ +#### `clone_from` + +
clone_from(&mut self, source: &Self)"}} />
+ +### `impl Debug for RecordConfig` + +
Debug for RecordConfig"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Default for RecordConfig` + +
Default for RecordConfig"}} />
+ +#### `default` + +
default() -> RecordConfig"}} />
+ +### `impl<'de> Deserialize<'de> for RecordConfig` + +
Deserialize<'de> for RecordConfig"}} />
+ +#### `deserialize` + +
deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where\n    __D: Deserializer<'de>,"}} />
+ +### `impl JsonSchema for RecordConfig` + +
RecordConfig"}} />
+ +#### `schema_name` + +
Cow<'static, str>"}} />
+ +#### `schema_id` + +
Cow<'static, str>"}} />
+ +#### `json_schema` + +
+ +#### `inline_schema` + +
bool"}} />
+ +### `impl PartialEq for RecordConfig` + +
PartialEq for RecordConfig"}} />
+ +#### `eq` + +
eq(&self, other: &RecordConfig) -> bool"}} />
+ +#### `ne` + +
ne(&self, other: &Rhs) -> bool"}} />
+ +### `impl Serialize for RecordConfig` + +
Serialize for RecordConfig"}} />
+ +#### `serialize` + +
serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where\n    __S: Serializer,"}} />
+ +### `impl Eq for RecordConfig` + +
Eq for RecordConfig"}} />
+ +### `impl StructuralPartialEq for RecordConfig` + +
StructuralPartialEq for RecordConfig"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx index 4a2989ac6..030a4323a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatifconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Atif Config" sidebar-title: "RelayAtifConfig" description: "Relay ATIF export configuration." -position: 38 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx index 4ff5d7de9..d595a140a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayatofconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Atof Config" sidebar-title: "RelayAtofConfig" description: "Relay ATOF export configuration." -position: 39 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx index 56259d7a5..4c490558c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relaycomponentconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Component Config" sidebar-title: "RelayComponentConfig" description: "Generic NeMo Relay plugin component configuration." -position: 40 +position: 41 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx index 570ef8d8e..ffc52a18c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Config" sidebar-title: "RelayConfig" description: "NeMo Relay integration configuration." -position: 41 +position: 42 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx index 712c72795..8d9e4a386 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayconfigpolicy.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Config Policy" sidebar-title: "RelayConfigPolicy" description: "Relay validation policy." -position: 42 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx index a01db8716..4db7528da 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayobservabilityconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Observability Config" sidebar-title: "RelayObservabilityConfig" description: "NeMo Relay observability component configuration." -position: 43 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx index 4bade1462..922156d9f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Open Telemetry Config" sidebar-title: "RelayOpenTelemetryConfig" description: "Relay OpenTelemetry export section." -position: 44 +position: 45 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx index da8138f4e..6bb243f6c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-relayopentelemetryendpointconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Relay Open Telemetry Endpoint Config" sidebar-title: "RelayOpenTelemetryEndpointConfig" description: "One typed Relay OpenTelemetry OTLP destination." -position: 45 +position: 46 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx index 8e70ade88..bf1537e85 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Resolve Context" sidebar-title: "ResolveContext" description: "Source context used when resolving an in-memory NeMo Fabric config." -position: 55 +position: 56 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx index 7c1e65663..34ad4c681 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadapterdescriptor.mdx @@ -2,7 +2,7 @@ title: "Struct Resolved Adapter Descriptor" sidebar-title: "ResolvedAdapterDescriptor" description: "Adapter descriptor selected for a run plan." -position: 56 +position: 57 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx index 8adac3efc..b3be2d837 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-resolvedadaptertargetdescriptor.mdx @@ -2,7 +2,7 @@ title: "Struct Resolved Adapter Target Descriptor" sidebar-title: "ResolvedAdapterTargetDescriptor" description: "Adapter target descriptor selected for a run plan." -position: 57 +position: 58 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx index e3c43e4e1..b8f75e62d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runplan.mdx @@ -2,7 +2,7 @@ title: "Struct RunPlan" sidebar-title: "RunPlan" description: "Resolved NeMo Fabric run plan." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx index 0d972cb87..5c2282536 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimecapabilities.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Capabilities" sidebar-title: "RuntimeCapabilities" description: "Lifecycle behavior implemented by a resolved runtime path." -position: 59 +position: 60 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx index 2c89a11b2..d9b6e6015 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-runtimeconfig.mdx @@ -2,14 +2,14 @@ title: "Struct Runtime Config" sidebar-title: "RuntimeConfig" description: "Invocation runtime contract." -position: 60 +position: 61 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
+
String,\n    pub output_schema: String,\n    pub artifacts: Option<PathBuf>,\n    pub timeout_seconds: Option<f64>,\n    pub max_turns: Option<u32>,\n    pub record: Option<RecordConfig>,\n    pub extensions: BTreeMap<String, Value>,\n}"}} />
Invocation runtime contract. @@ -35,6 +35,10 @@ Maximum duration of one invocation in seconds. Maximum number of harness turns within one invocation. +### `record: Option` + +Opt-in per-invocation run record. Core-only; never sent to adapters. + ### `extensions: BTreeMap` Additive normalized runtime fields. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx index 47644f228..1f4e2fe76 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-skillconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Skill Config" sidebar-title: "SkillConfig" description: "Skill capability configuration." -position: 61 +position: 62 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx index 2e1b4af6b..61e3aa7d0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Config" sidebar-title: "TelemetryConfig" description: "Telemetry configuration." -position: 62 +position: 63 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx index f12176331..326bcdbb5 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryplan.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Plan" sidebar-title: "TelemetryPlan" description: "Resolved telemetry plan." -position: 63 +position: 64 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx index f683b0886..bef871340 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-telemetryproviderconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Provider Config" sidebar-title: "TelemetryProviderConfig" description: "Provider-specific telemetry configuration." -position: 65 +position: 66 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx index b5e0b460f..771156fff 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-tooldefinitionconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Tool Definition Config" sidebar-title: "ToolDefinitionConfig" description: "One named normalized tool or tool-group definition." -position: 66 +position: 67 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx index 64b2d6eec..f9f3714b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Tools Config" sidebar-title: "ToolsConfig" description: "Harness-neutral tool capability configuration." -position: 67 +position: 68 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx index d2b51fe8b..ffe368f07 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-toolsplan.mdx @@ -2,7 +2,7 @@ title: "Struct Tools Plan" sidebar-title: "ToolsPlan" description: "Normalized tool policy for a run." -position: 58 +position: 59 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx index 21beec0df..ab0796f56 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Workflow Config" sidebar-title: "WorkflowConfig" description: "Registered workflow target and immutable construction settings." -position: 68 +position: 69 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx index a002d8fc3..03fecb115 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowentrypointconfig.mdx @@ -2,7 +2,7 @@ title: "Struct Workflow Entrypoint Config" sidebar-title: "WorkflowEntrypointConfig" description: "Adapter-owned workflow entry point supplied by an Adapter Target Descriptor." -position: 69 +position: 70 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx index e66041abb..a0e9695a1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/struct-workflowtargetspec.mdx @@ -2,7 +2,7 @@ title: "Struct Workflow Target Spec" sidebar-title: "WorkflowTargetSpec" description: "Workflow-specific Adapter Target Descriptor fields." -position: 70 +position: 71 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx index d8c0693d4..bbae2fe19 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/enum-doctorstatus.mdx @@ -2,7 +2,7 @@ title: "Enum Doctor Status" sidebar-title: "DoctorStatus" description: "Diagnostic status." -position: 76 +position: 77 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx index bb8aa23e8..b97461157 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/fn-doctor-plan.mdx @@ -2,7 +2,7 @@ title: "Function doctor_plan" sidebar-title: "doctor_plan" description: "Inspect a resolved run plan without mutating the environment." -position: 77 +position: 78 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index ea299ed94..916915ab1 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 129 +position: 135 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx index c64d5792e..9ae58d2b6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorcheck.mdx @@ -2,7 +2,7 @@ title: "Struct Doctor Check" sidebar-title: "DoctorCheck" description: "Diagnostic check result." -position: 74 +position: 75 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx index c7d65e76f..c015cc391 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/struct-doctorreport.mdx @@ -2,7 +2,7 @@ title: "Struct Doctor Report" sidebar-title: "DoctorReport" description: "Diagnostic report for a resolved run plan." -position: 75 +position: 76 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 75900423a..6462f4179 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -2,14 +2,14 @@ title: "Enum Fabric Error" sidebar-title: "FabricError" description: "Errors raised by NeMo Fabric config loading and validation." -position: 78 +position: 79 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -497,6 +497,42 @@ Environment provider. Adapter kind. +### `UnsupportedEnvironmentRelease` + +
+ +An environment provider does not implement release. + +#### Fields + +##### `provider: String` + +Environment provider. + +### `EnvironmentProviderOperation` + +
+ +An environment provider process could not complete an operation. + +#### Fields + +##### `provider: String` + +Environment provider. + +##### `operation: String` + +Lifecycle operation. + +##### `code: String` + +Stable provider or host error code. + +##### `message: String` + +Human-readable message that excludes credentials and raw diagnostics. + ### `EnvironmentHandleRequired`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index 0bf1ac774..ecc3b0451 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 134 +position: 136 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx index 0a91b0129..6f11a50d8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/type-result.mdx @@ -2,7 +2,7 @@ title: "Type Alias Result" sidebar-title: "Result" description: "Core NeMo Fabric result type." -position: 79 +position: 80 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 8af97f312..72ff2934f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 133 +position: 139 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 08ae7ee42..bd31141d2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -66,6 +66,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use config::MetadataConfig;` - `pub use config::ModelConfig;` - `pub use config::OAuthTokenEndpointAuthMethod;` +- `pub use config::RecordConfig;` - `pub use config::ResolutionStrategy;` - `pub use config::ResolveContext;` - `pub use config::ResolvedAdapterDescriptor;` @@ -132,6 +133,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use runtime::release_environment;` - `pub use runtime::release_service;` - `pub use runtime::run_plan;` +- `pub use runtime::run_record_folder;` - `pub use runtime::start_runtime;` - `pub use runtime::start_runtime_in;` - `pub use runtime::start_runtime_with_service;` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx index eb802320b..cdaccd776 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-errorstage.mdx @@ -2,7 +2,7 @@ title: "Enum Error Stage" sidebar-title: "ErrorStage" description: "NeMo Fabric lifecycle stage associated with an error." -position: 23 +position: 24 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx index cca22a793..83505d3b2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaichatcompletionchunkobject.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Chat Completion Chunk Object" sidebar-title: "OpenAiChatCompletionChunkObject" description: "Exact OpenAI object discriminator accepted by the v1 chunk profile." -position: 24 +position: 25 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx index f2d130b85..5e337a0e0 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamhost.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Host" sidebar-title: "OpenAiStreamHost" description: "Supported native-streaming listener host." -position: 25 +position: 26 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx index 1c3f839ae..eb3f8fe51 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprofile.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Profile" sidebar-title: "OpenAiStreamProfile" description: "Supported OpenAI-compatible chunk profile." -position: 26 +position: 27 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx index 86fddd94c..ee69f77c3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamprotocolversion.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Protocol Version" sidebar-title: "OpenAiStreamProtocolVersion" description: "Supported southbound native-streaming protocol version." -position: 27 +position: 28 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx index a3a933cdf..d02adc8bf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-openaistreamrecord.mdx @@ -2,7 +2,7 @@ title: "Enum OpenAI Stream Record" sidebar-title: "OpenAiStreamRecord" description: "One correlated NDJSON record on the adapter-native stream channel." -position: 28 +position: 29 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx index 97954ce13..f7acd827c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-runstatus.mdx @@ -2,7 +2,7 @@ title: "Enum RunStatus" sidebar-title: "RunStatus" description: "Runtime completion status." -position: 29 +position: 30 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx index c5050a301..310cf8983 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/enum-serviceownership.mdx @@ -2,7 +2,7 @@ title: "Enum Service Ownership" sidebar-title: "ServiceOwnership" description: "Ownership of a service exposed through NeMo Fabric." -position: 30 +position: 31 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx index d8ff25825..494089eb6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-attach-service.mdx @@ -2,7 +2,7 @@ title: "Function attach_service" sidebar-title: "attach_service" description: "Attach to a caller-owned long-lived service for a resolved run plan." -position: 31 +position: 33 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx index b81268897..1fa28e020 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-openai-stream.mdx @@ -2,7 +2,7 @@ title: "Function invoke_openai_stream" sidebar-title: "invoke_openai_stream" description: "Invoke a started harness runtime and pass through native OpenAI chat-completion chunks." -position: 32 +position: 34 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx index e1471de73..28b005c5d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-invoke-runtime.mdx @@ -2,7 +2,7 @@ title: "Function invoke_runtime" sidebar-title: "invoke_runtime" description: "Invoke a started harness runtime." -position: 33 +position: 35 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx index a59f0067c..49e08a5bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-prepare-service.mdx @@ -2,7 +2,7 @@ title: "Function prepare_service" sidebar-title: "prepare_service" description: "Prepare a Fabric-owned long-lived service for a resolved run plan." -position: 35 +position: 37 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx index b356d2db0..72ca1b83e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-release-service.mdx @@ -2,7 +2,7 @@ title: "Function release_service" sidebar-title: "release_service" description: "Release a Fabric-owned service or detach from a caller-owned service." -position: 36 +position: 39 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 55a5a47cb..331c08b3c 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -2,7 +2,7 @@ title: "Function run_plan" sidebar-title: "run_plan" description: "Invoke a NeMo Fabric run plan." -position: 37 +position: 40 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-record-folder.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-record-folder.mdx new file mode 100644 index 000000000..8d824b096 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-record-folder.mdx @@ -0,0 +1,14 @@ +--- +title: "Function run_record_folder" +sidebar-title: "run_record_folder" +description: "The folder that holds a plan's run records, `.fabric` under its artifact root, when `runtime.record.enabled` is set. `None` otherwise. Plan validation guarantees that such a plan has an artifact root, the `local` environment provider and a local adapter host." +position: 41 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +
RunPlan) -> Option<PathBuf>"}} />
+ +The folder that holds a plan's run records, `.fabric` under its artifact root, when `runtime.record.enabled` is set. `None` otherwise. Plan validation guarantees that such a plan has an artifact root, the `local` environment provider and a local adapter host. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-in.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-in.mdx index d31455b12..ccec560e4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-in.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-in.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_in" sidebar-title: "start_runtime_in" description: "Start a runtime in an explicitly prepared environment." -position: 42 +position: 43 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx index 547b47760..96b3675dc 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime-with-service.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime_with_service" sidebar-title: "start_runtime_with_service" description: "Start a runtime connected to an active service." -position: 39 +position: 44 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx index 5b10ad19a..0d7562938 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-start-runtime.mdx @@ -2,7 +2,7 @@ title: "Function start_runtime" sidebar-title: "start_runtime" description: "Start or connect to a harness runtime in its configured local environment." -position: 41 +position: 42 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx index 71febb037..77db99a02 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-stop-runtime.mdx @@ -2,7 +2,7 @@ title: "Function stop_runtime" sidebar-title: "stop_runtime" description: "Stop or detach from a harness runtime." -position: 40 +position: 45 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index 5fa3f62d9..a53561ac7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 135 +position: 137 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -59,6 +59,7 @@ Runtime invocation helpers. - [release_environment](fn-release-environment.mdx): Release or detach an environment through its provider. - [release_service](fn-release-service.mdx): Release a Fabric-owned service or detach from a caller-owned service. - [run_plan](fn-run-plan.mdx): Invoke a NeMo Fabric run plan. +- [run_record_folder](fn-run-record-folder.mdx): The folder that holds a plan's run records, `.fabric` under its artifact root, when `runtime.record.enabled` is set. `None` otherwise. Plan validation guarantees that such a plan has an artifact root, the `local` environment provider and a local adapter host. - [start_runtime](fn-start-runtime.mdx): Start or connect to a harness runtime in its configured local environment. - [start_runtime_in](fn-start-runtime-in.mdx): Start a runtime in an explicitly prepared environment. - [start_runtime_with_service](fn-start-runtime-with-service.mdx): Start a runtime connected to an active service. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx index 55d47378d..67102fa82 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-environmenthandle.mdx @@ -29,11 +29,11 @@ Where NeMo Fabric control code runs. ### `workspace: Option` -Workspace visible to the harness runtime. +Workspace visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `artifacts: Option` -Artifact root visible to the harness runtime. +Artifact root visible to the harness runtime. For non-local providers this is an environment-local path and must not be resolved on the host. ### `env: BTreeMap` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx index 9468739a0..376449e6a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-errorinfo.mdx @@ -2,7 +2,7 @@ title: "Struct Error Info" sidebar-title: "ErrorInfo" description: "Normalized error metadata." -position: 5 +position: 6 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx index 5330e632a..5ec70a8ad 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-fabricevent.mdx @@ -2,7 +2,7 @@ title: "Struct Fabric Event" sidebar-title: "FabricEvent" description: "NeMo Fabric lifecycle or progress event." -position: 6 +position: 7 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx index 3f3ac0d42..1df9629e6 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-invocationhandle.mdx @@ -2,7 +2,7 @@ title: "Struct Invocation Handle" sidebar-title: "InvocationHandle" description: "One request sent to a runtime." -position: 7 +position: 8 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx index 026b2de72..96e965402 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunk.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk" sidebar-title: "OpenAiChatCompletionChunk" description: "OpenAI Chat Completions chunk accepted by the native streaming profile." -position: 8 +position: 9 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx index 94b2e70ee..f3dbf5424 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkchoice.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Choice" sidebar-title: "OpenAiChatCompletionChunkChoice" description: "One choice within an OpenAI Chat Completions streaming chunk." -position: 9 +position: 10 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx index 845afed45..53b0a6972 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaichatcompletionchunkdelta.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Chat Completion Chunk Delta" sidebar-title: "OpenAiChatCompletionChunkDelta" description: "Incremental assistant message fields carried by one OpenAI choice." -position: 10 +position: 11 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx index 49b4c9c49..b5a3b7fcb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreaminvocation.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Invocation" sidebar-title: "OpenAiStreamInvocation" description: "One adapter-native OpenAI streaming invocation." -position: 11 +position: 12 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx index 8ad3827d5..a178c8113 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamsink.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Sink" sidebar-title: "OpenAiStreamSink" description: "Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric." -position: 12 +position: 13 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx index 2ed1e5c62..4a7f2e2f9 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-openaistreamtransport.mdx @@ -2,7 +2,7 @@ title: "Struct OpenAI Stream Transport" sidebar-title: "OpenAiStreamTransport" description: "SDK-owned loopback transport for one native OpenAI streaming invocation." -position: 13 +position: 14 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx index 8dcc6950f..d0200756d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runrequest.mdx @@ -2,7 +2,7 @@ title: "Struct RunRequest" sidebar-title: "RunRequest" description: "A request passed to a NeMo Fabric-managed harness runtime." -position: 14 +position: 15 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx index 1cb5bd052..f74b804b8 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runresult.mdx @@ -2,7 +2,7 @@ title: "Struct RunResult" sidebar-title: "RunResult" description: "Result from a NeMo Fabric-managed harness invocation." -position: 15 +position: 16 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx index 28af565b2..625527702 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimecontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Context" sidebar-title: "RuntimeContext" description: "Context generated for one invocation of a started runtime." -position: 17 +position: 18 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx index b0392902d..0671b565e 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Handle" sidebar-title: "RuntimeHandle" description: "Active or resumable harness runtime." -position: 18 +position: 19 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx index 1525ed89d..890abc95b 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runtimetelemetrycontext.mdx @@ -2,7 +2,7 @@ title: "Struct Runtime Telemetry Context" sidebar-title: "RuntimeTelemetryContext" description: "Runtime telemetry config passed to adapters." -position: 19 +position: 20 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx index 6f16e387b..864be9b27 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-runusage.mdx @@ -2,7 +2,7 @@ title: "Struct RunUsage" sidebar-title: "RunUsage" description: "Normalized invocation usage exposed to NeMo Fabric consumers." -position: 16 +position: 17 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx index 8c36ff5de..dead906b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicehandle.mdx @@ -2,7 +2,7 @@ title: "Struct Service Handle" sidebar-title: "ServiceHandle" description: "Public identity and binding for one prepared or attached service." -position: 20 +position: 21 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx index e28440353..4a249074a 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-servicereference.mdx @@ -2,7 +2,7 @@ title: "Struct Service Reference" sidebar-title: "ServiceReference" description: "Caller-supplied reference to an already-running service." -position: 21 +position: 22 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx index cd66e4025..94338baeb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/struct-telemetryref.mdx @@ -2,7 +2,7 @@ title: "Struct Telemetry Ref" sidebar-title: "TelemetryRef" description: "Reference to telemetry emitted by Relay or another configured telemetry path." -position: 22 +position: 23 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx index 0e4166bba..282ef841d 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/enum-schemaname.mdx @@ -33,6 +33,7 @@ pub enum SchemaName { ArtifactManifest, ErrorInfo, FabricEvent, + RunRecordLine, } ``` @@ -172,6 +173,12 @@ Structured error metadata schema. NeMo Fabric lifecycle event schema. +### `RunRecordLine` + +
+ +One line of an opt-in per-invocation run record, version `v1alpha1`. + ## Implementations ### `impl SchemaName` @@ -180,7 +187,7 @@ NeMo Fabric lifecycle event schema. #### `ALL` -
22]"}} />
+
23]"}} />
All public schemas in stable output order. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 45c3ca38e..9d6706929 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 136 +position: 138 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index adafb55f8..b2ef27be4 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -265,7 +265,7 @@ Relay subfields: | `models..top_p`, `.max_tokens` | No | No | Yes | Yes | Yes; passed through LiteLLM | | `instructions.system` | `replace`, `append` | `replace`; maps to Codex base instructions | `replace` | `replace` | `replace` | | `runtime.input_schema`, `.output_schema` | Core | Core | Core | Core | Core | -| `runtime.artifacts`, `.timeout_seconds` | Core | Core | Core | Core | Core | +| `runtime.artifacts`, `.timeout_seconds`, `.record` | Core | Core | Core | Core | Core | | `runtime.max_turns` | Yes | No | Yes; maps to LangGraph supersteps | Yes; maps to Hermes iterations | Yes | | `environment.provider`, `.control_location`, `.ownership` | Core | Core | Core | Core | Core | | `environment.workspace`, `.artifacts`, `.env`, `.env_names` | Core | Core | Core | Core | Core | @@ -996,6 +996,87 @@ If NeMo Fabric cannot resolve config, start a runtime, or obtain a normalized re the SDK raises a `FabricError` subclass instead of returning a partial `RunResult`. +## Run Records + +A run record is an append-only JSON Lines file that NeMo Fabric writes for each +invocation, so a run that is killed still leaves its identity, start time, and +timeout behind. Recording is off by default. Setting `runtime.artifacts` alone +does not turn it on. Turn it on with `runtime.record`: + +```python +from nemo_fabric import RecordConfig, RuntimeConfig + +runtime = RuntimeConfig(artifacts="runs", record=RecordConfig(enabled=True)) +``` + +`enabled` is required inside `record`. The record needs an artifact root +(`runtime.artifacts` or `environment.artifacts`), the `local` environment +provider, and a `process` or `python` adapter. Planning fails with +`runtime.record.enabled` when any of these is missing. + +Each invocation that NeMo Fabric tries to send to the adapter host writes +`/.fabric///run-record.jsonl`, and the +result lists it as the `record` artifact with kind `run_record`: + +```python +record = next(a for a in result.artifacts.artifacts if a.kind == "run_record") +``` + +The file holds a start line and an end line. Every line carries `schema` +(`nemo.fabric.run_record`), `version` (`v1alpha1`), `kind`, `runtime_id`, +`invocation_id`, `request_id`, and `time`. + +- The `start` line is written before the request is sent. It holds the agent + name, harness, adapter ID, each configured model's provider and model name, + the NeMo Fabric version, and the timeout in milliseconds. From + `RunRequest.context`, it keeps the string or integer values of `task_id`, + `task_revision`, `attempt`, `step`, `harbor_session_id`, `harbor_context_id`, + and `harbor_trial_name`, cut to 256 characters. Every other context key + appears by name only as `{"recorded": false}`. `override_keys` lists the + top-level `RunRequest.overrides` key names, sorted, without their values. +- The `end` line is written after the invocation returns or fails. It holds + `status`, `elapsed_ms`, the error's `code`, `stage`, and `retryable` flag, + the numeric `RunUsage` counters, and each result artifact's name, kind, + media type, and path relative to the artifact root. An error code outside + `^[a-z0-9_.:-]{1,128}$` is written as `unrecognized`. When NeMo Fabric + raises instead of returning a result, the end line reads `failed`. An + adapter host error keeps its code, such as `host_timeout`, `host_crashed`, + `host_io`, or `invalid_agent_run_result`. A result extension that the + adapter's descriptor does not declare, or that breaks its declared schema, + also reads `invalid_agent_run_result`. + A failed write of `adapter-invocation.json` or a result artifact reads + `artifact_write_failed`, and any other error reads `fabric_error`. + +The end line describes the invocation. A one-shot `Fabric.run` stops the +runtime after the invocation; when that stop fails, the result reads `failed` +with stage `stop`, and the record still holds the invocation's own end line. + +`run-record.jsonl` never holds prompts, model output, error messages, tool +payloads, adapter metadata, or context values outside the keys above. The same +folder also holds `adapter-invocation.json` and `stdout.txt`, which NeMo Fabric +writes for every invocation whether or not recording is on; those files are +not covered by this rule. + +Each line is written with one append, so a killed process keeps every line +whose write returned. NeMo Fabric syncs the file once, after the end line. A +record without an end line belongs to an invocation that is still running or +that never finished; treat its outcome as unknown, not as failed. + +Recording is synchronous and best-effort. If a write or the sync fails, the +record stops. Record I/O errors do not replace the invocation result: a +returned result keeps its status, output, error, and usage and gains a +`record_write_failed` event, and a raised error stays the same error. A record +whose end line failed to write therefore has no end line, and that event says +why. NeMo Fabric creates the record and writes the start line before it sets +the adapter's deadline, so slow storage delays the request without shortening +that deadline. The end line and the sync run before the result returns, so +slow storage delays the return too. Recording sets no time limit on either +delay. The file is created with mode `0600` on Linux and macOS. Creation +refuses an existing file or symbolic link at `run-record.jsonl`. + +The line format is published as `schemas/sdk/run-record-line.schema.json` at +version `v1alpha1`, which may change until a reader for run records ships. + ## Events Each `RunResult` includes the normalized events collected for that invocation. diff --git a/schemas/SCHEMA.md b/schemas/SCHEMA.md index bfac432e9..c139d5bb6 100644 --- a/schemas/SCHEMA.md +++ b/schemas/SCHEMA.md @@ -128,6 +128,9 @@ descriptor schema. - `sdk/artifact-manifest`: normalized artifact references. - `sdk/error-info`: structured runtime or adapter error metadata. - `sdk/fabric-event`: NeMo Fabric lifecycle or progress event. +- `sdk/run-record-line`: one line of the opt-in per-invocation run record, + `run-record.jsonl`. The schema is `v1alpha1` and may change until a reader + for run records ships. ### Deferred Core Objects diff --git a/schemas/sdk/agent.schema.json b/schemas/sdk/agent.schema.json index 720929df3..7d6c5e994 100644 --- a/schemas/sdk/agent.schema.json +++ b/schemas/sdk/agent.schema.json @@ -558,6 +558,20 @@ } ] }, + "RecordConfig": { + "additionalProperties": false, + "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "properties": { + "enabled": { + "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -1329,6 +1343,17 @@ "description": "Output schema label.", "type": "string" }, + "record": { + "anyOf": [ + { + "$ref": "#/$defs/RecordConfig" + }, + { + "type": "null" + } + ], + "description": "Opt-in per-invocation run record. Core-only; never sent to adapters." + }, "timeout_seconds": { "description": "Maximum duration of one invocation in seconds.", "exclusiveMinimum": 0.0, diff --git a/schemas/sdk/run-plan.schema.json b/schemas/sdk/run-plan.schema.json index b33341bf2..5d3211b4e 100644 --- a/schemas/sdk/run-plan.schema.json +++ b/schemas/sdk/run-plan.schema.json @@ -1926,6 +1926,20 @@ } ] }, + "RecordConfig": { + "additionalProperties": false, + "description": "Per-invocation run record settings.\n\nWhen enabled, core appends a start line and an end line for each local\ninvocation to `/.fabric///run-record.jsonl`.", + "properties": { + "enabled": { + "description": "Whether to write a run record for each invocation. Requires an\nartifact root, the `local` environment provider, and a `process` or\n`python` adapter.", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, "RelayAtifConfig": { "additionalProperties": true, "description": "Relay ATIF export configuration.", @@ -2772,6 +2786,17 @@ "description": "Output schema label.", "type": "string" }, + "record": { + "anyOf": [ + { + "$ref": "#/$defs/RecordConfig" + }, + { + "type": "null" + } + ], + "description": "Opt-in per-invocation run record. Core-only; never sent to adapters." + }, "timeout_seconds": { "description": "Maximum duration of one invocation in seconds.", "exclusiveMinimum": 0.0, diff --git a/schemas/sdk/run-record-line.schema.json b/schemas/sdk/run-record-line.schema.json new file mode 100644 index 000000000..7b0f8642e --- /dev/null +++ b/schemas/sdk/run-record-line.schema.json @@ -0,0 +1,429 @@ +{ + "$defs": { + "ContextValue": { + "anyOf": [ + { + "description": "An allowlisted key's string value, cut to 256 characters.", + "type": "string" + }, + { + "description": "An allowlisted key's integer value.", + "format": "int64", + "type": "integer" + }, + { + "$ref": "#/$defs/NotRecorded", + "description": "Any other key or value: the key name only." + } + ], + "description": "A request context value as the start line keeps it." + }, + "EndArtifact": { + "description": "One result artifact on the end line.", + "properties": { + "kind": { + "description": "Artifact kind.", + "type": "string" + }, + "media_type": { + "description": "Media type.", + "type": [ + "string", + "null" + ] + }, + "name": { + "description": "Logical artifact name.", + "type": "string" + }, + "path": { + "description": "Path relative to the artifact root; absent for a path outside it.", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "name", + "kind" + ], + "type": "object" + }, + "EndError": { + "description": "The end line's error.", + "properties": { + "code": { + "description": "Error code matching `^[a-z0-9_.:-]{1,128}$`, else `unrecognized`.", + "pattern": "^[a-z0-9_.:-]{1,128}$", + "type": "string" + }, + "retryable": { + "description": "Whether a consumer-level retry is considered safe.", + "type": "boolean" + }, + "stage": { + "$ref": "#/$defs/ErrorStage", + "description": "Lifecycle stage where the failure surfaced." + } + }, + "required": [ + "code", + "stage", + "retryable" + ], + "type": "object" + }, + "EndLine": { + "description": "Fields of the end line.", + "properties": { + "artifacts": { + "description": "Result artifacts, with paths relative to the artifact root.", + "items": { + "$ref": "#/$defs/EndArtifact" + }, + "type": "array" + }, + "elapsed_ms": { + "description": "Monotonic time from the start line to the end line, in milliseconds.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/EndError" + }, + { + "type": "null" + } + ], + "description": "Error code, stage and retryable flag, without the message." + }, + "status": { + "$ref": "#/$defs/RunStatus", + "description": "Final status. A Fabric lifecycle error, such as a host timeout, reads `failed`." + }, + "usage": { + "anyOf": [ + { + "$ref": "#/$defs/EndUsage" + }, + { + "type": "null" + } + ], + "description": "Numeric usage counters, without adapter metadata." + } + }, + "required": [ + "status", + "elapsed_ms" + ], + "type": "object" + }, + "EndUsage": { + "description": "The end line's usage: the numeric `RunUsage` fields.", + "properties": { + "cached_input_tokens": { + "description": "Cached input tokens.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "cost_usd": { + "description": "Reported cost in US dollars.", + "format": "double", + "minimum": 0.0, + "type": [ + "number", + "null" + ] + }, + "input_tokens": { + "description": "Input tokens.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "input_tokens_include_cache": { + "description": "Whether `input_tokens` includes `cached_input_tokens`; absent means unknown.", + "type": [ + "boolean", + "null" + ] + }, + "output_tokens": { + "description": "Output tokens.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "total_tokens": { + "description": "Total tokens reported by the provider.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "ErrorStage": { + "description": "NeMo Fabric lifecycle stage associated with an error.", + "oneOf": [ + { + "const": "config", + "description": "Configuration resolution failed.", + "type": "string" + }, + { + "const": "plan", + "description": "Effective config planning failed.", + "type": "string" + }, + { + "const": "prepare", + "description": "Environment preparation failed.", + "type": "string" + }, + { + "const": "start", + "description": "Runtime start/connect failed.", + "type": "string" + }, + { + "const": "invoke", + "description": "Runtime invocation failed.", + "type": "string" + }, + { + "const": "stop", + "description": "Runtime stop/detach failed.", + "type": "string" + }, + { + "const": "release", + "description": "Environment release failed.", + "type": "string" + }, + { + "const": "artifact", + "description": "Artifact collection or writing failed.", + "type": "string" + } + ] + }, + "ModelName": { + "description": "One configured model.", + "properties": { + "model": { + "description": "Provider model identifier.", + "type": "string" + }, + "provider": { + "description": "Model provider name.", + "type": "string" + } + }, + "required": [ + "provider", + "model" + ], + "type": "object" + }, + "NotRecorded": { + "description": "Marks a key whose value the record leaves out.", + "properties": { + "recorded": { + "const": false, + "description": "Always `false`.", + "type": "boolean" + } + }, + "required": [ + "recorded" + ], + "type": "object" + }, + "RecordSchema": { + "description": "The record's schema name.", + "oneOf": [ + { + "const": "nemo.fabric.run_record", + "description": "`nemo.fabric.run_record`.", + "type": "string" + } + ] + }, + "RecordVersion": { + "description": "The record's line format version.", + "oneOf": [ + { + "const": "v1alpha1", + "description": "`v1alpha1`.", + "type": "string" + } + ] + }, + "RunStatus": { + "description": "Runtime completion status.", + "oneOf": [ + { + "const": "succeeded", + "description": "The harness completed successfully.", + "type": "string" + }, + { + "const": "failed", + "description": "The harness completed with a non-zero status.", + "type": "string" + }, + { + "const": "cancelled", + "description": "The invocation or runtime was cancelled.", + "type": "string" + } + ] + }, + "StartLine": { + "description": "Fields of the start line.", + "properties": { + "adapter_id": { + "description": "Adapter implementation id.", + "type": [ + "string", + "null" + ] + }, + "agent_name": { + "description": "Config name (`metadata.name`).", + "type": "string" + }, + "context": { + "additionalProperties": { + "$ref": "#/$defs/ContextValue" + }, + "description": "`RunRequest.context`. Allowlisted identity and Harbor link keys keep a\nstring or integer value; every other key is listed by name only.", + "type": "object" + }, + "fabric_version": { + "description": "NeMo Fabric version that wrote the record.", + "type": "string" + }, + "harness": { + "description": "Harness identifier.", + "type": "string" + }, + "models": { + "additionalProperties": { + "$ref": "#/$defs/ModelName" + }, + "description": "Configured models by role: provider and model name only.", + "type": "object" + }, + "override_keys": { + "description": "Top-level key names of `RunRequest.overrides`, sorted. Values are never\nrecorded.", + "items": { + "type": "string" + }, + "type": "array" + }, + "timeout_ms": { + "description": "Host invocation timeout that applied, in milliseconds.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "agent_name", + "harness", + "fabric_version", + "timeout_ms" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "One line of a run record.", + "oneOf": [ + { + "$ref": "#/$defs/StartLine", + "description": "Written once, before core sends the request to the adapter host.", + "properties": { + "kind": { + "const": "start", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "$ref": "#/$defs/EndLine", + "description": "Written once, after the invocation returned a result or failed.", + "properties": { + "kind": { + "const": "end", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ], + "properties": { + "invocation_id": { + "description": "Invocation id.", + "type": "string" + }, + "request_id": { + "description": "Request id.", + "type": "string" + }, + "runtime_id": { + "description": "Runtime id.", + "type": "string" + }, + "schema": { + "$ref": "#/$defs/RecordSchema", + "description": "Always `nemo.fabric.run_record`." + }, + "time": { + "description": "UTC wall-clock time the line was written, RFC 3339 with milliseconds.", + "format": "date-time", + "type": "string" + }, + "version": { + "$ref": "#/$defs/RecordVersion", + "description": "Line format version. `v1alpha1` may change until a record reader ships." + } + }, + "required": [ + "schema", + "version", + "runtime_id", + "invocation_id", + "request_id", + "time" + ], + "title": "RunRecordLine", + "type": "object" +} \ No newline at end of file diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py index 2cbf7dd17..c10134ced 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/__init__.py @@ -24,6 +24,7 @@ from nemo_fabric.models import McpServerConfig from nemo_fabric.models import MetadataConfig from nemo_fabric.models import ModelConfig +from nemo_fabric.models import RecordConfig from nemo_fabric.models import RelayAtifConfig from nemo_fabric.models import RelayAtofConfig from nemo_fabric.models import RelayAtofFileSinkConfig @@ -98,6 +99,7 @@ "MetadataConfig", "ModelConfig", "OpenAIInvokeStream", + "RecordConfig", "RelayAtifConfig", "RelayAtofConfig", "RelayAtofFileSinkConfig", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md index 14f33030b..637cd462d 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/README.md @@ -55,6 +55,44 @@ The bridge does not inspect harness IDs, native error-code lists, or diagnostic text. Other adapter-specific timeout codes are not automatically reclassified. +## Trial Identity In Run Records + +Set `fabric_record=True` on `FabricAgent` to write a NeMo Fabric run record for +each agent run in a trial (one per step on a multi-step trial) under the +bridge's artifact root. It is off by default, and without +it the bridge sends the same request as before. With it on, the bridge reads +the trial's `lock.json` and `config.json`, which Harbor writes to the trial +folder before the agent starts, and adds these keys to the request context: + +| Key | Source | +| --- | --- | +| `task_id` | The task name in `lock.json`: `org/name` for a registry task, and the task folder's name for a git or local task | +| `task_revision` | Harbor's content digest of the task (`sha256:...`) in `lock.json`, for every task kind | +| `harbor_trial_name` | The trial's `trial_name` in `config.json` | +| `step` | On a multi-step trial only (Harbor creates the trial's `steps` folder), the bridge's run count in the trial: 1 for the first step, 2 for the second, and so on | + +The record's start line keeps them with `harbor_session_id` and +`harbor_context_id`. NeMo Fabric forwards these keys to the adapter, as it +forwards every request context key; no first-party adapter reads them. Harbor +runs each attempt as its own trial and gives the trial no attempt index, so the +bridge sets no `attempt`. Group a task's attempts by `task_id` and +`task_revision`. A missing or unreadable `lock.json` or `config.json`, or one +that does not hold a JSON object, adds none of its keys, and the run goes on. +Otherwise the bridge copies `task.name`, `task.digest`, and `trial_name` each on +its own when it is a nonempty string, and does not check the file against +Harbor's lock or config model. + +On the host, each record is at +`/agent/fabric-artifacts//.fabric///run-record.jsonl`, +where the config name is `harbor-` followed by the last part of the adapter +ID, such as `harbor-hermes`. On a multi-step trial, Harbor moves or copies the +agent folder's contents into `/steps//agent/` after each step, so +each step's records are there. + +`fabric_record=True` needs a NeMo Fabric version with `runtime.record` in the +trial environment. With the first-party adapters, an older `fabric_package` +rejects the config. + ## Finalized Telemetry Artifacts The task runner validates telemetry after the one-shot NeMo Fabric runtime shuts down. For Relay trajectories finalized during shutdown, it reads the runtime-owned plugin configuration and collects local ATIF files from that runtime's configured output directory. It validates the trajectory before promoting it to Harbor's `agent/trajectory.json`; ambiguous or malformed artifacts remain telemetry failures rather than changing the verifier reward. Remote-only output and nested filename templates are not collected by this fallback. Adapter invocation results and multi-turn behavior are unchanged. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/fabric_agent.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/fabric_agent.py index 2c7e25a95..3b5c741f0 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/fabric_agent.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/integrations/harbor/fabric_agent.py @@ -33,6 +33,7 @@ from nemo_fabric import InstructionsConfig from nemo_fabric import MetadataConfig from nemo_fabric import ModelConfig +from nemo_fabric import RecordConfig from nemo_fabric import RelayAtifConfig from nemo_fabric import RelayAtofConfig from nemo_fabric import RelayAtofFileSinkConfig @@ -152,6 +153,10 @@ class FabricAgentOptions(AgentOptions): fabric_runtime_timeout_seconds: float | None = Field( default=None, gt=0, description="Fabric runtime timeout in seconds." ) + fabric_record: bool = Field( + default=False, + description="Write a Fabric run record with the trial's task identity.", + ) fabric_environment_env: dict[str, str] | None = Field( default=None, description="Task-local Fabric environment variables; credentials require host-variable references without inline defaults.", @@ -299,6 +304,7 @@ def __init__( fabric_system_instruction: str | None = None, fabric_max_turns: int | None = None, fabric_runtime_timeout_seconds: float | None = None, + fabric_record: bool = False, fabric_environment_env: dict[str, str] | None = None, fabric_blocked_tools: list[str] | None = None, fabric_enabled_tools: list[str] | None = None, @@ -333,6 +339,7 @@ def __init__( fabric_system_instruction=fabric_system_instruction, fabric_max_turns=fabric_max_turns, fabric_runtime_timeout_seconds=fabric_runtime_timeout_seconds, + fabric_record=fabric_record, fabric_environment_env=fabric_environment_env, fabric_blocked_tools=fabric_blocked_tools, fabric_enabled_tools=fabric_enabled_tools, @@ -359,6 +366,9 @@ def __init__( self.fabric_system_instruction = options.fabric_system_instruction self.fabric_max_turns = options.fabric_max_turns self.fabric_runtime_timeout_seconds = options.fabric_runtime_timeout_seconds + self.fabric_record = options.fabric_record + # Harbor calls run() once per step of a multi-step task. + self._invocation_count = 0 self.fabric_environment_env = resolve_env_vars( options.fabric_environment_env or {} ) @@ -482,6 +492,7 @@ async def run( context: AgentContext, ) -> None: self._result_path = None + self._invocation_count += 1 token = uuid.uuid4().hex spec = self._build_spec(instruction) host_spec_path = self.logs_dir / f"fabric-run-{token}.json" @@ -533,13 +544,21 @@ async def run( ensure_success("NeMo Fabric run failed", result) def _build_request(self, instruction: str) -> RunRequest: - context = {"source": "harbor"} + context: dict[str, Any] = {"source": "harbor"} session_id = getattr(self, "session_id", None) context_id = getattr(self, "context_id", None) if session_id is not None: context["harbor_session_id"] = session_id if context_id is not None: context["harbor_context_id"] = str(context_id) + if self.fabric_record: + # The trial folder is the parent of the agent's logs folder. + trial_dir = self.logs_dir.parent + context.update(harbor_trial_identity(trial_dir)) + # A multi-step trial creates /steps// before each + # step's agent runs; a single-step trial never creates steps/. + if self._invocation_count and (trial_dir / "steps").is_dir(): + context["step"] = self._invocation_count return RunRequest(input=instruction, context=context) def _build_spec(self, instruction: str) -> FabricRunPayload: @@ -567,6 +586,7 @@ def _build_config(self) -> FabricConfig: system_instruction=self.fabric_system_instruction, max_turns=self.fabric_max_turns, timeout_seconds=self.fabric_runtime_timeout_seconds, + record=self.fabric_record, environment_env=self.fabric_environment_env, blocked_tools=self.fabric_blocked_tools, enabled_tools=self.fabric_enabled_tools, @@ -648,6 +668,40 @@ def _runner_env(self) -> dict[str, str]: return env +def harbor_trial_identity(trial_dir: Path) -> dict[str, str]: + """Return the task identity of a Harbor trial, with unset fields left out. + + Harbor writes ``lock.json`` and ``config.json`` to the trial folder before + the agent runs. The task ID is the lock's task name: ``org/name`` for a + registry task, and the task folder's name for a git or local task, never + a local path. The task revision is the lock's content digest of the task + (``sha256:...``), which Harbor computes for every task kind. A missing or + unreadable file, or one that does not hold a JSON object, adds none of its + keys. Otherwise each field that is a nonempty string is copied on its own; + the files are not checked against Harbor's lock or config model. + """ + + identity: dict[str, str] = {} + task = _read_json_object(trial_dir / "lock.json").get("task") + if isinstance(task, dict): + for key, field in (("task_id", "name"), ("task_revision", "digest")): + value = task.get(field) + if isinstance(value, str) and value: + identity[key] = value + trial_name = _read_json_object(trial_dir / "config.json").get("trial_name") + if isinstance(trial_name, str) and trial_name: + identity["harbor_trial_name"] = trial_name + return identity + + +def _read_json_object(path: Path) -> dict[str, Any]: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + return document if isinstance(document, dict) else {} + + def build_harbor_config( *, adapter_id: str, @@ -658,6 +712,7 @@ def build_harbor_config( system_instruction: str | None = None, max_turns: int | None = None, timeout_seconds: float | None = None, + record: bool = False, environment_env: dict[str, str] | None = None, blocked_tools: list[str] | None = None, enabled_tools: list[str] | None = None, @@ -692,6 +747,7 @@ def build_harbor_config( artifacts=artifact_root, timeout_seconds=timeout_seconds, max_turns=max_turns, + record=RecordConfig(enabled=True) if record else None, ), environment=EnvironmentConfig( provider="local", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py index 831ad0717..d81aed5df 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/models.py @@ -25,6 +25,7 @@ from pydantic import ConfigDict from pydantic import Field from pydantic import SerializerFunctionWrapHandler +from pydantic import StrictBool from pydantic import field_validator from pydantic import model_serializer from pydantic import model_validator @@ -187,6 +188,20 @@ class InstructionsConfig(FabricBaseModel): system: InstructionConfig | None = None +class RecordConfig(FabricBaseModel): + """Opt-in per-invocation run record. + + When ``enabled`` is true, NeMo Fabric writes + ``/.fabric///run-record.jsonl`` + for each invocation. It requires an artifact root, the ``local`` + environment provider, and a ``process`` or ``python`` adapter. + """ + + model_config = ConfigDict(extra="forbid") + + enabled: StrictBool + + class RuntimeConfig(FabricBaseModel): """Invocation runtime contract.""" @@ -199,6 +214,7 @@ class RuntimeConfig(FabricBaseModel): allow_inf_nan=False, ) max_turns: int | None = Field(default=None, gt=0, le=(1 << 32) - 1) + record: RecordConfig | None = None class EnvironmentConfig(FabricBaseModel): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py index 7ef06a715..28236de0b 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/types.py @@ -372,11 +372,19 @@ class _RuntimeConfig(_ConfigMapping): artifacts: Optional artifact-root path. timeout_seconds: Optional invocation deadline in seconds. max_turns: Optional harness turn limit. + record: Optional run record settings, ``{"enabled": bool}``. extra_fields: Preserved extension fields not recognized by this SDK. """ _fields = frozenset( - {"input_schema", "output_schema", "artifacts", "timeout_seconds", "max_turns"} + { + "input_schema", + "output_schema", + "artifacts", + "timeout_seconds", + "max_turns", + "record", + } ) def __init__( @@ -387,6 +395,7 @@ def __init__( artifacts: str | Path | None = None, timeout_seconds: float | None = None, max_turns: int | None = None, + record: Mapping[str, Any] | None = None, extra_fields: Mapping[str, Any] | None = None, ) -> None: values: dict[str, Any] = {} @@ -419,6 +428,13 @@ def __init__( "runtime max_turns must be between 1 and 4294967295" ) values["max_turns"] = max_turns + if record is not None: + record = _mapping(record, "runtime record") + if set(record) != {"enabled"} or not isinstance(record["enabled"], bool): + raise FabricConfigError( + "runtime record must be an object with only a boolean enabled field" + ) + values["record"] = {"enabled": record["enabled"]} super().__init__(values, extra_fields=extra_fields) @classmethod @@ -432,6 +448,7 @@ def from_mapping(cls, value: Mapping[str, Any]) -> "_RuntimeConfig": artifacts=data.get("artifacts"), timeout_seconds=data.get("timeout_seconds"), max_turns=data.get("max_turns"), + record=data.get("record"), extra_fields={ key: item for key, item in data.items() if key not in cls._fields }, diff --git a/skills/nemo-fabric-integrate/references/results-and-errors.md b/skills/nemo-fabric-integrate/references/results-and-errors.md index f2660d5e3..be25dfea7 100644 --- a/skills/nemo-fabric-integrate/references/results-and-errors.md +++ b/skills/nemo-fabric-integrate/references/results-and-errors.md @@ -46,6 +46,31 @@ if result.usage is not None: Treat missing counters and cost as unknown, not zero. `cached_input_tokens` reports cached prompt tokens; `input_tokens_include_cache` declares whether `input_tokens` includes them (`True`), excludes them (`False`), or has unknown semantics (absent). Do not add cache to an inclusive input count. Adapters can put cache-write, reasoning, peak request, and per-model detail in the descriptor-declared `usage.extensions["nemo_fabric.detailed_token_usage.v1"]` object. NeMo Fabric preserves that object at `result.usage["metadata"]["nemo_fabric.detailed_token_usage.v1"]`. Treat detailed input and output independently: any detailed accounting field for a dimension makes that dimension authoritative, with no fallback to its top-level counters. The views can diverge. Calculate an inclusive total only when the selected detailed counters provide enough information; otherwise leave the total unknown. Keep estimated costs separate in provider metadata; `cost_usd` is reported cost, not an estimate. +## Run Records + +Set `RuntimeConfig(artifacts=..., record=RecordConfig(enabled=True))` to have +NeMo Fabric write `run-record.jsonl` for each invocation under `/.fabric///`. Recording is off by default and +needs an artifact root, the `local` environment provider, and a `process` or +`python` adapter. Once the start line is written, the result lists the file as +the artifact with kind `run_record`; if creating the record fails, the result +has a `record_write_failed` event and no such artifact. The file has a `start` +line, written before the request, and an `end` line with status and elapsed +time, plus the error code, numeric usage, and artifact names when present. It +holds no prompts, output, or error messages. The start line does keep the +values of seven allowlisted `RunRequest.context` keys (task identity and the +Harbor trial link): strings cut to 256 characters and integers that fit a +signed 64-bit value (any other value is recorded by key name only), so don't +put prompt text in those keys. A record without an end line has an unknown +outcome; do not count it as failed. Recording is synchronous and best-effort: +when the invocation returns a result, a failed record write adds a +`record_write_failed` event to it, and its status, output, error, and usage +stay the same; when the invocation raises, the error stays the same error. Slow +storage delays the invocation. The format is the `v1alpha1` [run record line +schema](https://github.com/NVIDIA/NeM +o-Fabric/blob/main/schemas/sdk/run-record-line.schema.json) and may change +until a reader ships. + ## Correlation IDs `runtime_id` identifies the runtime lifecycle, `invocation_id` identifies one diff --git a/tests/integrations/test_harbor_runner.py b/tests/integrations/test_harbor_runner.py index c2d10221f..b8a4c7b7b 100644 --- a/tests/integrations/test_harbor_runner.py +++ b/tests/integrations/test_harbor_runner.py @@ -738,6 +738,7 @@ def test_harbor_023_options_schema_and_preflight(): "fabric_model_base_url", "fabric_package", "fabric_python", + "fabric_record", "fabric_runtime_timeout_seconds", "fabric_system_instruction", "fabric_telemetry", diff --git a/tests/python/test_harbor_integration.py b/tests/python/test_harbor_integration.py index a61e43567..20c09b43f 100644 --- a/tests/python/test_harbor_integration.py +++ b/tests/python/test_harbor_integration.py @@ -20,6 +20,7 @@ try: from harbor.models.agent.context import AgentContext + from harbor.models.job.lock import TaskLock from harbor.models.task.config import MCPServerConfig from nemo_fabric.integrations.harbor import FabricAgent except ImportError: @@ -445,6 +446,131 @@ def test_harbor_propagates_runtime_identity(tmp_path: Path): assert agent.SUPPORTS_ATIF is False +_LOCAL_DIGEST = "sha256:" + "a" * 64 +_GIT_DIGEST = "sha256:" + "b" * 64 +_PACKAGE_DIGEST = "sha256:" + "c" * 64 + + +def _write_trial_files(trial_dir: Path, task_lock: dict[str, str] | None) -> None: + """Write the trial's ``config.json`` and, unless ``task_lock`` is None, + the ``lock.json`` that Harbor writes before the agent runs. The task + entry goes through Harbor's ``TaskLock`` model, so a renamed or newly + validated field fails here.""" + + trial_dir.mkdir(parents=True, exist_ok=True) + (trial_dir / "config.json").write_text( + json.dumps({"task": {"path": "tasks/fix-io"}, "trial_name": "hello-world__AbC12"}), + encoding="utf-8", + ) + if task_lock is not None: + (trial_dir / "lock.json").write_text( + json.dumps( + {"schema_version": 2, "task": TaskLock(**task_lock).model_dump(mode="json")} + ), + encoding="utf-8", + ) + + +@pytest.mark.parametrize( + ("task_lock", "expected"), + [ + ( + {"name": "fix-io", "type": "local", "digest": _LOCAL_DIGEST}, + {"task_id": "fix-io", "task_revision": _LOCAL_DIGEST}, + ), + ( + { + "name": "fix-io", + "type": "git", + "digest": _GIT_DIGEST, + "git_commit_id": "abc123", + }, + {"task_id": "fix-io", "task_revision": _GIT_DIGEST}, + ), + ( + {"name": "harbor/hello-world", "type": "package", "digest": _PACKAGE_DIGEST}, + {"task_id": "harbor/hello-world", "task_revision": _PACKAGE_DIGEST}, + ), + (None, {}), + ], + ids=["local", "git", "package", "no-lock"], +) +def test_harbor_record_reads_task_identity_from_the_trial_lock( + tmp_path: Path, task_lock: dict[str, str] | None, expected: dict[str, str] +): + _write_trial_files(tmp_path, task_lock) + agent = FabricAgent( + logs_dir=tmp_path / "agent", + fabric_adapter_id="nvidia.fabric.hermes", + fabric_record=True, + ) + + context = agent._build_request("fix it").context + + assert context == { + "source": "harbor", + **expected, + "harbor_trial_name": "hello-world__AbC12", + } + assert agent._build_config().to_mapping()["runtime"]["record"] == {"enabled": True} + + +def test_harbor_record_stays_off_without_fabric_record(tmp_path: Path): + _write_trial_files(tmp_path, {"name": "fix-io", "type": "local", "digest": _LOCAL_DIGEST}) + agent = FabricAgent( + logs_dir=tmp_path / "agent", + fabric_adapter_id="nvidia.fabric.hermes", + ) + + assert agent._build_request("fix it").context == {"source": "harbor"} + assert "record" not in agent._build_config().to_mapping()["runtime"] + + +def test_harbor_record_ignores_trial_files_that_are_not_json_objects(tmp_path: Path): + (tmp_path / "lock.json").write_text("not json", encoding="utf-8") + (tmp_path / "config.json").write_text("[]", encoding="utf-8") + agent = FabricAgent( + logs_dir=tmp_path / "agent", + fabric_adapter_id="nvidia.fabric.hermes", + fabric_record=True, + ) + + assert agent._build_request("fix it").context == {"source": "harbor"} + + +async def _run_contexts(trial_dir: Path, runs: int) -> list[dict[str, Any]]: + agent = FabricAgent( + logs_dir=trial_dir / "agent", + fabric_adapter_id="nvidia.fabric.hermes", + fabric_record=True, + ) + environment = FakeHarborEnvironment() + await agent.setup(environment) # type: ignore[arg-type] + for run in range(runs): + await agent.run(f"step {run}", environment, AgentContext()) # type: ignore[arg-type] + return [ + json.loads(environment.files[target])["request"]["context"] + for _, target in environment.uploads + if target.startswith("/tmp/fabric-run-") + ] + + +async def test_harbor_record_numbers_each_step_of_a_multi_step_trial(tmp_path: Path): + # Harbor creates /steps//agent before each step's agent runs. + (tmp_path / "steps" / "one" / "agent").mkdir(parents=True) + + contexts = await _run_contexts(tmp_path, runs=2) + + assert [context["step"] for context in contexts] == [1, 2] + + +async def test_harbor_record_single_step_trial_records_no_step(tmp_path: Path): + contexts = await _run_contexts(tmp_path, runs=1) + + assert len(contexts) == 1 + assert "step" not in contexts[0] + + async def test_harbor_structured_package_install_is_shell_safe(tmp_path: Path): with ( Path(__file__).resolve().parents[2] / "sdk/python/nemo-fabric/pyproject.toml" diff --git a/tests/python/test_sdk_consumer.py b/tests/python/test_sdk_consumer.py index bac8db3e3..a9e5e5727 100644 --- a/tests/python/test_sdk_consumer.py +++ b/tests/python/test_sdk_consumer.py @@ -10,7 +10,7 @@ from unittest.mock import MagicMock import pytest -from nemo_fabric import Fabric, FabricConfigError, RunPlan +from nemo_fabric import Fabric, FabricConfigError, RecordConfig, RunPlan from _utils.configs import hermes_shim_config @@ -148,3 +148,16 @@ async def test_installed_descriptors_reject_invalid_config_before_native_start( base_dir=hermes_shim_agent_dir, ) mock_start.assert_not_called() + + +def test_record_without_an_artifact_root_fails_native_planning(hermes_shim_agent_dir): + config = hermes_shim_config() + config.runtime.artifacts = None + config.environment.artifacts = None + config.runtime.record = RecordConfig(enabled=True) + + with pytest.raises(FabricConfigError, match="runtime.record.enabled"): + Fabric().plan(config, base_dir=hermes_shim_agent_dir) + + config.runtime.artifacts = "./artifacts" + assert Fabric().plan(config, base_dir=hermes_shim_agent_dir).config.runtime.record == {"enabled": True} diff --git a/tests/python/test_sdk_contract.py b/tests/python/test_sdk_contract.py index b71bea32d..d342d283e 100644 --- a/tests/python/test_sdk_contract.py +++ b/tests/python/test_sdk_contract.py @@ -36,6 +36,7 @@ from nemo_fabric import McpConfig from nemo_fabric import McpServerConfig from nemo_fabric import MetadataConfig +from nemo_fabric import RecordConfig from nemo_fabric import RelayAtifConfig from nemo_fabric import RelayAtofConfig from nemo_fabric import RelayAtofFileSinkConfig @@ -698,6 +699,28 @@ def test_instruction_content_must_be_non_empty(content: str): _FabricConfigSnapshot.from_mapping(raw) +def test_runtime_record_round_trips_and_rejects_other_shapes(): + config = FabricConfig( + metadata=MetadataConfig(name="demo"), + harness=HarnessConfig(adapter_id="test.fabric.shim"), + runtime=RuntimeConfig(artifacts="runs", record=RecordConfig(enabled=True)), + ) + + assert config.to_mapping()["runtime"]["record"] == {"enabled": True} + assert "record" not in RuntimeConfig().to_mapping() + for invalid in ({}, {"enabled": "yes"}, {"enabled": True, "durable": True}): + with pytest.raises(ValidationError): + RecordConfig.model_validate(invalid) + + raw = _plan()["config"] + raw["runtime"] = {"record": {"enabled": True}} + assert _FabricConfigSnapshot.from_mapping(raw).runtime.record == {"enabled": True} + for invalid in ({}, {"enabled": 1}, {"enabled": True, "durable": True}): + raw["runtime"]["record"] = invalid + with pytest.raises(FabricConfigError, match="boolean enabled field"): + _FabricConfigSnapshot.from_mapping(raw) + + def test_max_turns_matches_rust_u32_range(): maximum = (1 << 32) - 1