Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
5e4c29d
feat: write an opt-in run record for each invocation
mnajafian-nv Oct 16, 2026
6655b13
test: cover run record write failures
mnajafian-nv Oct 16, 2026
f43809e
fix: start the run record before the adapter deadline
mnajafian-nv Oct 16, 2026
100394a
fix: keep run record model role and context key names whole
mnajafian-nv Oct 16, 2026
264e34b
docs: describe run recording as synchronous and best-effort
mnajafian-nv Oct 16, 2026
ca9dd3b
feat: record task identity from the CLI and the Harbor bridge
mnajafian-nv Oct 16, 2026
bdbfc0e
test: check Harbor trial locks through Harbor's own model
mnajafian-nv Oct 16, 2026
f5b6c39
docs: say Harbor gives a trial no attempt index
mnajafian-nv Oct 16, 2026
0bfcb56
test: check the CLI run record line goes to standard error
mnajafian-nv Oct 16, 2026
ddf1c56
test: assert Harbor step numbers in run order
mnajafian-nv Oct 16, 2026
e61ea03
docs: say which trial file problems drop Harbor task identity keys
mnajafian-nv Oct 16, 2026
ebffc5a
docs: name the trial file fields the Harbor bridge copies
mnajafian-nv Oct 16, 2026
6b10b68
docs: limit the record write failure event to returned results
mnajafian-nv Oct 16, 2026
ac9f02c
docs: name the result fields a record write failure leaves unchanged
mnajafian-nv Oct 16, 2026
f83521b
fix: record a schema-invalid result extension as invalid_agent_run_re…
mnajafian-nv Oct 16, 2026
5acc3b4
Merge branch 'main' into feat/run-record-task-identity-v2
mnajafian-nv Oct 16, 2026
3b6e98e
Merge upstream main into feat/opt-in-run-record
mnajafian-nv Oct 16, 2026
c233cf4
test: reject the run record on the OpenShell provider
mnajafian-nv Oct 16, 2026
caa72f5
docs: regenerate references after merging main
mnajafian-nv Oct 16, 2026
d059199
Merge feat/opt-in-run-record into feat/run-record-task-identity-v2
mnajafian-nv Oct 16, 2026
41cca94
docs: regenerate references after merging main
mnajafian-nv Oct 16, 2026
e61e58b
Merge upstream main into feat/opt-in-run-record
mnajafian-nv Oct 18, 2026
74cda83
Merge feat/opt-in-run-record into feat/run-record-task-identity-v2
mnajafian-nv Oct 18, 2026
7242433
Merge upstream main into feat/run-record-task-identity-v2
mnajafian-nv Oct 19, 2026
ffb184c
docs: note which context values the run record keeps
mnajafian-nv Oct 20, 2026
094883e
docs: qualify the run record artifact and context values
mnajafian-nv Oct 20, 2026
6ec7ce9
docs: note optional run record end fields and the integer range
mnajafian-nv Oct 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 128 additions & 12 deletions crates/fabric-cli/src/app.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use clap::error::ErrorKind;
use clap::{ArgGroup, Args, Parser, Subcommand};
use nemo_fabric_core::{
ResolveContext, RunRequest, RunStatus, doctor_plan, resolve_diagnostic_plan_from_config,
resolve_run_plan_from_config, run_plan,
resolve_run_plan_from_config, run_plan, run_record_folder,
};

use crate::examples;
Expand Down Expand Up @@ -47,6 +47,13 @@ enum Command {
/// Inline request text.
#[arg(long, default_value = "")]
input: String,
/// Task ID recorded on the run (sets `task_id` in the request context).
#[arg(
long,
value_name = "ID",
value_parser = clap::builder::NonEmptyStringValueParser::new()
)]
task: Option<String>,
},
/// Print the NVIDIA NeMo Fabric core version.
Version,
Expand Down Expand Up @@ -87,6 +94,45 @@ enum ExampleCommand {
},
}

/// Config switches that turn on and label the run record.
#[derive(Debug, Default, Args)]
struct RecordSwitches {
/// Write a run record for each invocation (sets `runtime.record.enabled`).
/// Needs `--artifacts` unless the example sets an artifact root.
#[arg(long)]
record: bool,
/// Artifact root, resolved against the current folder (sets `runtime.artifacts`).
#[arg(long, value_name = "DIR")]
artifacts: Option<std::path::PathBuf>,
/// Config name recorded on the run (sets `metadata.name`).
#[arg(
long,
value_name = "NAME",
value_parser = clap::builder::NonEmptyStringValueParser::new()
)]
name: Option<String>,
}

impl RecordSwitches {
/// Apply the switches to the selected config.
fn apply(&self, config: &mut nemo_fabric_core::FabricConfig) -> std::io::Result<()> {
if self.record {
config
.runtime
.record
.get_or_insert_with(nemo_fabric_core::RecordConfig::default)
.enabled = true;
}
if let Some(artifacts) = &self.artifacts {
config.runtime.artifacts = Some(std::path::absolute(artifacts)?);
}
if let Some(name) = &self.name {
config.metadata.name.clone_from(name);
}
Ok(())
}
}

#[derive(Debug, Args)]
#[command(group(
ArgGroup::new("source")
Expand Down Expand Up @@ -125,6 +171,8 @@ struct Selector {
value_parser = parse_temperature
)]
temperature: Option<f64>,
#[command(flatten)]
record: RecordSwitches,
}

fn parse_temperature(value: &str) -> Result<f64, String> {
Expand All @@ -150,6 +198,13 @@ impl SelectedSource {
}
}

fn config_mut(&mut self) -> &mut nemo_fabric_core::FabricConfig {
match self {
Self::Preset(selected) => &mut selected.config,
Self::Example(selected) => &mut selected.config,
}
}

fn base_dir(&self) -> &std::path::Path {
match self {
Self::Preset(selected) => selected.base_dir(),
Expand Down Expand Up @@ -245,13 +300,26 @@ fn run(cli: Cli) -> Result<(), Box<dyn std::error::Error>> {
)?;
println!("{}", serde_json::to_string_pretty(&doctor_plan(&plan))?);
}
Command::Run { selector, input } => {
Command::Run {
selector,
input,
task,
} => {
let selected = select_source(&selector)?;
let plan = resolve_run_plan_from_config(
selected.config().clone(),
ResolveContext::new(selected.base_dir()),
)?;
let result = match run_plan(&plan, RunRequest::text(input)) {
let mut request = RunRequest::text(input);
if let Some(task) = task {
request
.context
.insert("task_id".to_string(), serde_json::Value::from(task));
}
if let Some(folder) = run_record_folder(&plan) {
eprintln!("nemo-fabric: run record under {}", folder.display());
}
let result = match run_plan(&plan, request) {
Ok(result) => result,
Err(error) => {
println!(
Expand Down Expand Up @@ -284,17 +352,17 @@ fn require_successful_run(status: RunStatus) -> Result<(), String> {
}

fn select_source(selector: &Selector) -> Result<SelectedSource, Box<dyn std::error::Error>> {
if let Some(name) = &selector.preset {
let mut selected = if let Some(name) = &selector.preset {
let mut selected = find_preset(name)?.stage()?;
apply_model_overrides(&mut selected.config, selector)?;
return Ok(SelectedSource::Preset(selected));
}
if let Some(name) = &selector.example {
return Ok(SelectedSource::Example(
find_example(name)?.select(selector.variant.as_deref())?,
));
}
unreachable!("Clap requires exactly one source selector")
SelectedSource::Preset(selected)
} else if let Some(name) = &selector.example {
SelectedSource::Example(find_example(name)?.select(selector.variant.as_deref())?)
} else {
unreachable!("Clap requires exactly one source selector")
};
selector.record.apply(selected.config_mut())?;
Ok(selected)
}

fn apply_model_overrides(
Expand Down Expand Up @@ -381,6 +449,7 @@ mod tests {
variant: None,
model: Some("meta/llama-3.3-70b-instruct".to_string()),
temperature: Some(0.2),
record: RecordSwitches::default(),
};
let selected = select_source(&selector).expect("select preset");
let model = selected.config().models.get("default").expect("model");
Expand All @@ -389,6 +458,7 @@ mod tests {
assert_eq!(model.model, "meta/llama-3.3-70b-instruct");
assert_eq!(model.temperature, Some(0.2));
assert_eq!(model.api_key_env.as_deref(), Some("NVIDIA_API_KEY"));
assert!(selected.config().runtime.record.is_none());
}

#[test]
Expand All @@ -411,6 +481,7 @@ mod tests {
variant: None,
model: Some("other-model".to_string()),
temperature: None,
record: RecordSwitches::default(),
};
assert_eq!(
select_source(&selector)
Expand Down Expand Up @@ -477,4 +548,49 @@ mod tests {
fn plan_requires_a_preset() {
assert!(Cli::try_parse_from(["nemo-fabric", "plan"]).is_err());
}

#[test]
fn record_switches_parse_on_plan_doctor_and_run_and_set_the_config() {
let selectors: Vec<Selector> = ["plan", "doctor", "run"]
.into_iter()
.map(|command| {
let cli = Cli::try_parse_from([
"nemo-fabric",
command,
"--preset",
"scripted",
"--record",
"--artifacts",
"runs",
"--name",
"swe-scripted",
])
.expect("parse record switches");
match cli.command {
Command::Plan(selector) | Command::Doctor(selector) => selector,
Command::Run { selector, .. } => selector,
_ => panic!("selector command expected"),
}
})
.collect();
for selector in &selectors {
assert!(selector.record.record);
assert_eq!(selector.record.name.as_deref(), Some("swe-scripted"));
}

let selected = select_source(&selectors[0]).expect("select");

let config = selected.config();
assert!(
config
.runtime
.record
.as_ref()
.is_some_and(|record| record.enabled)
);
let artifacts = config.runtime.artifacts.as_ref().expect("artifacts");
assert!(artifacts.is_absolute());
assert!(artifacts.ends_with("runs"));
assert_eq!(config.metadata.name, "swe-scripted");
}
}
1 change: 1 addition & 0 deletions crates/fabric-cli/src/presets.rs
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,7 @@ fn config(
artifacts: None,
timeout_seconds: None,
max_turns: None,
record: None,
extensions: BTreeMap::new(),
},
environment: Some(EnvironmentConfig {
Expand Down
1 change: 1 addition & 0 deletions crates/fabric-cli/templates/rust/main.rs.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ fn build_config() -> FabricConfig {
artifacts: None,
timeout_seconds: {{TIMEOUT_SECONDS}},
max_turns: {{MAX_TURNS}},
record: None,
extensions: BTreeMap::new(),
},
environment: Some(EnvironmentConfig {
Expand Down
61 changes: 61 additions & 0 deletions crates/fabric-cli/tests/run_record.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! CLI regression coverage for the run record switches.

use std::process::Command;

#[test]
fn run_with_record_writes_task_and_name_and_keeps_stdout_json() {
let root = std::env::temp_dir().join(format!("fabric-cli-record-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);

let output = Command::new(env!("CARGO_BIN_EXE_nemo-fabric"))
.args([
"run", "--preset", "scripted", "--input", "hello", "--record",
])
.arg("--artifacts")
.arg(&root)
.args(["--name", "swe-scripted", "--task", "repo-io-fix"])
.output()
.expect("run CLI");
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(output.status.success(), "scripted run failed: {stderr}");

// The folder line comes first on standard error, and standard output is
// only the JSON result.
let folder = root.join(".fabric");
assert_eq!(
stderr.lines().next(),
Some(format!("nemo-fabric: run record under {}", folder.display()).as_str()),
"{stderr}"
);
let result: serde_json::Value =
serde_json::from_slice(&output.stdout).expect("standard output is one JSON document");
assert_eq!(result["status"], "succeeded");

let records: Vec<_> = std::fs::read_dir(&folder)
.expect("runtime folders")
.flatten()
.flat_map(|runtime| {
std::fs::read_dir(runtime.path())
.expect("invocations")
.flatten()
})
.map(|invocation| invocation.path().join("run-record.jsonl"))
.filter(|path| path.exists())
.collect();
assert_eq!(records.len(), 1, "{records:?}");
let text = std::fs::read_to_string(&records[0]).expect("record");
let start: serde_json::Value =
serde_json::from_str(text.lines().next().expect("start line")).expect("json");
assert_eq!(start["context"]["task_id"], "repo-io-fix");
assert_eq!(start["agent_name"], "swe-scripted");
assert!(
text.lines()
.last()
.expect("end line")
.contains("\"kind\":\"end\"")
);
let _ = std::fs::remove_dir_all(root);
}
Loading
Loading