From 0710033aed185575748b6258eaa59994905372ac Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 07:31:22 -0400 Subject: [PATCH 001/112] test(50-02): add failing 999.141 home-isolation and scan tests - tests/common/mod.rs: shared helpers (HermeticHome under /tmp, FakeBin, wait_for, kill_and_reap, LaunchedRunReaper, entries_under); the hermetic_devflow builder is a stub that sets nothing yet - tests/test_home_isolation.rs: R-13 test with a plain-Command negative control, and the permanent source scan with planted controls; the spawn and env-mutation counters are stubs - RED: both tests fail on their planned assertions (HOME not set; planted raw spawn counted 0) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/tests/common/mod.rs | 200 ++++++++ .../devflow-cli/tests/test_home_isolation.rs | 427 ++++++++++++++++++ 2 files changed, 627 insertions(+) create mode 100644 crates/devflow-cli/tests/common/mod.rs create mode 100644 crates/devflow-cli/tests/test_home_isolation.rs diff --git a/crates/devflow-cli/tests/common/mod.rs b/crates/devflow-cli/tests/common/mod.rs new file mode 100644 index 00000000..2e5748fd --- /dev/null +++ b/crates/devflow-cli/tests/common/mod.rs @@ -0,0 +1,200 @@ +//! Helpers shared by the `devflow` CLI integration tests (999.141, D-07). +//! +//! Each test target includes this file with `mod common;`. It holds helpers +//! only and no `#[test]`: every target that declares `mod common;` compiles +//! it, so a test here would be added to each of those targets' counts. +//! +//! The rule it exists for: an e2e child that runs the `devflow` binary gets a +//! tempdir `HOME`, `XDG_CACHE_HOME`, `XDG_CONFIG_HOME` and `DEVFLOW_CACHE_DIR` +//! and has the repo-local `GIT_*` variables removed, so its registry entries +//! (and, from 50-11, its sockets) never land in the operator's real +//! `~/.cache/devflow`. Spawn `devflow` through [`hermetic_devflow`] only; +//! `test_home_isolation.rs` scans every suite that declares `mod common;` for +//! a raw spawn. Isolation is set on each child `Command`, never by mutating +//! the test process's own environment. + +#![allow( + dead_code, + reason = "each test target uses a different subset of the shared helpers" +)] + +use devflow_core::phase_id::PhaseId; +use std::fs; +use std::os::unix::fs::{PermissionsExt, symlink}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; +use std::time::{Duration, Instant}; + +/// The compiled `devflow` binary under test. The only place in `tests/` that +/// names Cargo's binary path variable once migration completes. +pub fn devflow_bin() -> &'static str { + env!("CARGO_BIN_EXE_devflow") +} + +/// A throwaway `HOME` for one or more `devflow` children, removed on drop. +/// +/// Created directly under `/tmp` with a short prefix, never under a possibly +/// long `$TMPDIR`, so a socket path below it stays under the 107-byte +/// `sun_path` limit (R-5). +pub struct HermeticHome { + dir: tempfile::TempDir, +} + +impl HermeticHome { + /// Create a fresh, empty home under `/tmp`. + pub fn new() -> Self { + let dir = tempfile::Builder::new() + .prefix("dfh") + .tempdir_in("/tmp") + .expect("create hermetic HOME under /tmp"); + Self { dir } + } + + /// The home directory itself (the child's `HOME`). + pub fn home(&self) -> &Path { + self.dir.path() + } + + /// The child's DevFlow cache dir (`home/.cache/devflow`), where its + /// registry `roots/` entries land. + pub fn cache_dir(&self) -> PathBuf { + self.home().join(".cache").join("devflow") + } + + /// The child's `XDG_CONFIG_HOME` (`home/.config`). + pub fn config_dir(&self) -> PathBuf { + self.home().join(".config") + } +} + +impl Default for HermeticHome { + fn default() -> Self { + Self::new() + } +} + +/// A `devflow` command whose `HOME`, `XDG_CACHE_HOME`, `XDG_CONFIG_HOME` and +/// `DEVFLOW_CACHE_DIR` all point inside `home`, with every repo-local and +/// repo-redirecting `GIT_*` variable removed. `home` must outlive the child +/// and any monitor it launches. +pub fn hermetic_devflow(home: &HermeticHome) -> Command { + let _ = home; + Command::new(devflow_bin()) +} + +/// A directory of stub programs to put first on a child's `PATH`. Removed on +/// drop. +pub struct FakeBin { + _dir: tempfile::TempDir, + pub path: PathBuf, +} + +/// A fake `claude` whose body is `agent_body`, plus `git` and `sh` links. +pub fn fake_bin_dir(agent_body: &str) -> FakeBin { + fake_bin_dir_for("claude", agent_body) +} + +/// A fake agent named `program` (for example `claude` or `codex`) whose body +/// is `agent_body`, plus `git` and `sh` links. +pub fn fake_bin_dir_for(program: &str, agent_body: &str) -> FakeBin { + let dir = tempfile::tempdir().unwrap(); + let agent = dir.path().join(program); + fs::write(&agent, format!("#!/bin/sh\n{agent_body}\n")).unwrap(); + let mut permissions = fs::metadata(&agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).unwrap(); + for linked in ["git", "sh"] { + symlink(format!("/usr/bin/{linked}"), dir.path().join(linked)).unwrap(); + } + FakeBin { + path: dir.path().to_path_buf(), + _dir: dir, + } +} + +/// Poll `predicate` every 25ms until it is true, panicking with `what` once +/// `timeout` elapses. +pub fn wait_for(mut predicate: impl FnMut() -> bool, timeout: Duration, what: &str) { + let started = Instant::now(); + while !predicate() { + assert!( + started.elapsed() < timeout, + "timed out after {timeout:?} waiting for {what}" + ); + std::thread::sleep(Duration::from_millis(25)); + } +} + +/// Kill `child` if it is still running, then wait on it. +pub fn kill_and_reap(child: &mut Child) { + if child.try_wait().expect("poll child").is_none() { + let _ = child.kill(); + } + let _ = child.wait(); +} + +/// Terminates, on drop, the monitor recorded in the phase state and the +/// agent named in the agent-pid file, unless its pid is in `spare`. Bind it +/// before (or right after) the spawn that launches a run, and keep the +/// run's [`HermeticHome`] alive until this has dropped. +pub struct LaunchedRunReaper { + root: PathBuf, + phase: PhaseId, + spare: Vec, +} + +impl LaunchedRunReaper { + pub fn new(root: &Path, phase: PhaseId, spare: Vec) -> Self { + Self { + root: root.to_path_buf(), + phase, + spare, + } + } +} + +impl Drop for LaunchedRunReaper { + fn drop(&mut self) { + let monitor = devflow_core::workflow::load_state(&self.root, self.phase) + .ok() + .and_then(|state| state.monitor_pid); + let agent = fs::read_to_string(devflow_core::agent_result::agent_pid_path( + &self.root, self.phase, + )) + .ok() + .and_then(|pid| pid.trim().parse::().ok()); + for pid in [monitor, agent].into_iter().flatten() { + if !self.spare.contains(&pid) && devflow_core::agent::agent_running(pid) { + let _ = devflow_core::agent::terminate_and_verify( + pid, + devflow_core::agent::TERMINATE_VERIFY_WAIT, + devflow_core::agent::TERMINATE_VERIFY_POLL, + ); + } + } + } +} + +/// Every file below `dir`, recursively, sorted. Empty when `dir` is missing. +pub fn entries_under(dir: &Path) -> Vec { + let mut files = Vec::new(); + collect_files(dir, &mut files); + files.sort(); + files +} + +fn collect_files(dir: &Path, files: &mut Vec) { + let Ok(entries) = fs::read_dir(dir) else { + return; + }; + for entry in entries { + let entry = entry.expect("read directory entry"); + let path = entry.path(); + // `file_type` does not follow symlinks, so a link loop cannot recurse. + if entry.file_type().expect("read entry type").is_dir() { + collect_files(&path, files); + } else { + files.push(path); + } + } +} diff --git a/crates/devflow-cli/tests/test_home_isolation.rs b/crates/devflow-cli/tests/test_home_isolation.rs new file mode 100644 index 00000000..ab2b2a03 --- /dev/null +++ b/crates/devflow-cli/tests/test_home_isolation.rs @@ -0,0 +1,427 @@ +//! R-13 guard for 999.141 (D-07): an e2e `devflow` child keeps its writes +//! inside its own tempdir `HOME`. +//! +//! `a_child_writing_outside_its_tempdir_home_fails` runs a real `devflow +//! start` through `common::hermetic_devflow` and checks that its registry +//! entry lands under the hermetic home. Its negative control runs the same +//! start as a plain `Command` whose `HOME` names a fake "real" home, and shows +//! that home is populated, so the first check is able to fail. +//! +//! `suites_using_common_never_spawn_devflow_raw_or_mutate_the_environment` is +//! a permanent source scan: every `tests/*.rs` that declares `mod common;` must +//! spawn `devflow` only through the builder and must never mutate the test +//! process's environment. A suite is covered as soon as it declares +//! `mod common;`, without editing this file. +//! +//! Every scanned token is spelled through `concat!` split inside the token, so +//! this file's own source holds a raw spawn token exactly once: at the +//! negative-control spawn. Nothing here reads the operator's real +//! `~/.cache/devflow`. + +mod common; + +use devflow_core::phase_id::PhaseId; +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Duration; + +/// Bounds the Define gate a launched supervise run parks at, so a reaper +/// failure cannot leave an immortal monitor. +const GATE_TIMEOUT_SECS: &str = "15"; + +/// How long a launched run may take to record its agent pid. +const AGENT_PID_TIMEOUT: Duration = Duration::from_secs(10); + +fn git(root: &Path, args: &[&str]) { + let output = devflow_core::test_support::git_command(root) + .args(args) + .output() + .expect("spawn git"); + assert!( + output.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// The same `develop`-only repo `start_lock_e2e.rs` starts a phase in. +fn init_repo(root: &Path, phase: PhaseId) { + git(root, &["init", "-q"]); + git(root, &["config", "user.email", "devflow@example.com"]); + git(root, &["config", "user.name", "DevFlow Tests"]); + git(root, &["config", "commit.gpgsign", "false"]); + git(root, &["config", "core.hooksPath", "/dev/null"]); + git(root, &["checkout", "-q", "-b", "develop"]); + fs::create_dir_all(root.join(".planning")).unwrap(); + fs::write(root.join("README.md"), "base\n").unwrap(); + fs::write( + root.join(".planning/ROADMAP.md"), + format!("# Roadmap\n\n### Phase {phase}: home isolation fixture\n"), + ) + .unwrap(); + git(root, &["add", "README.md", ".planning/ROADMAP.md"]); + git(root, &["commit", "-q", "-m", "base"]); +} + +/// Adds the `start` arguments and the fake-agent `PATH` to `command`. +fn add_start_args(command: &mut Command, root: &Path, phase: PhaseId, fake_bin: &common::FakeBin) { + let existing = std::env::var_os("PATH").unwrap_or_default(); + let path = format!("{}:{}", fake_bin.path.display(), existing.to_string_lossy()); + command + .args([ + "start", + "--phase", + &phase.to_string(), + "--agent", + "claude", + "--mode", + "supervise", + "--no-worktree", + "--legacy-claude-launch", + ]) + .arg(root) + .current_dir(root) + .env("PATH", path) + .env("DEVFLOW_GATE_TIMEOUT_SECS", GATE_TIMEOUT_SECS); +} + +/// The value `command` sets for `name`; panics when it inherits or removes it. +fn env_set_by(command: &Command, name: &str) -> PathBuf { + let (_, value) = command + .get_envs() + .find(|(key, _)| *key == OsStr::new(name)) + .unwrap_or_else(|| panic!("the hermetic command must set {name}; it inherits it")); + let value = + value.unwrap_or_else(|| panic!("the hermetic command must set {name}, not remove it")); + PathBuf::from(value) +} + +/// Runs `command` (a `start` built by the caller), reaps the run it launches, +/// and returns every file under `roots` afterwards. +fn run_start_and_list( + mut command: Command, + root: &Path, + phase: PhaseId, + roots: &Path, +) -> Vec { + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); + let output = command.output().expect("run devflow start"); + assert!( + output.status.success(), + "devflow start failed\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + // Wait for the agent pid so the reaper can end the agent as well as the + // monitor. + let agent_pid = devflow_core::agent_result::agent_pid_path(root, phase); + common::wait_for( + || agent_pid.exists(), + AGENT_PID_TIMEOUT, + "the launched agent's pid file", + ); + common::entries_under(roots) +} + +#[test] +fn a_child_writing_outside_its_tempdir_home_fails() { + let phase = PhaseId::new(89); + let fake_bin = common::fake_bin_dir("exec sleep 60"); + + // Hermetic arm: the builder must set HOME and every cache variable under + // the hermetic home, and the run's registry entry must land there. The + // environment is checked before the spawn, so a builder that sets nothing + // fails here instead of writing to the inherited HOME. + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + init_repo(repo.path(), phase); + let mut hermetic = common::hermetic_devflow(&home); + add_start_args(&mut hermetic, repo.path(), phase, &fake_bin); + assert_eq!( + env_set_by(&hermetic, "HOME"), + home.home(), + "HOME must be the hermetic home" + ); + for (name, expected) in [ + ("XDG_CACHE_HOME", home.home().join(".cache")), + ("XDG_CONFIG_HOME", home.config_dir()), + ("DEVFLOW_CACHE_DIR", home.cache_dir()), + ] { + let value = env_set_by(&hermetic, name); + assert!( + value == expected && value.starts_with(home.home()), + "{name} must be {} under the hermetic home; it is {}", + expected.display(), + value.display() + ); + } + let hermetic_roots = home.cache_dir().join("roots"); + let entries = run_start_and_list(hermetic, repo.path(), phase, &hermetic_roots); + assert!( + !entries.is_empty(), + "the hermetic run must register under {}; it holds nothing", + hermetic_roots.display() + ); + + // Negative control (R-13): the one deliberate non-hermetic devflow spawn + // in tests/. A child left on an inherited-style HOME writes there, so the + // assertion above can fail. R2 is a fake "real home", never the operator's. + let real_home = tempfile::Builder::new() + .prefix("dfr") + .tempdir_in("/tmp") + .expect("create fake real home under /tmp"); + let repo2 = tempfile::tempdir().unwrap(); + init_repo(repo2.path(), phase); + let mut plain = Command::new(common::devflow_bin()); + plain + .env("HOME", real_home.path()) + .env_remove("DEVFLOW_CACHE_DIR") + .env_remove("XDG_CACHE_HOME"); + // Git hermeticity is a separate concern from HOME; keep the control from + // retargeting a real repository when the suite runs under a hook. + for name in devflow_core::test_support::REPO_LOCAL_GIT_VARS + .iter() + .chain(devflow_core::test_support::ALSO_REDIRECTING_GIT_VARS) + { + plain.env_remove(name); + } + add_start_args(&mut plain, repo2.path(), phase, &fake_bin); + let control_roots = real_home.path().join(".cache/devflow/roots"); + let entries = run_start_and_list(plain, repo2.path(), phase, &control_roots); + assert!( + !entries.is_empty(), + "the plain-Command control must populate {}; it holds nothing, so the \ + hermetic assertion cannot fail", + control_roots.display() + ); +} + +/// `src` with every `//` comment (line, doc and trailing) removed. A `//` +/// inside a string, raw string or char literal is not a comment. +fn code_only(src: &str) -> String { + let chars: Vec = src.chars().collect(); + let mut out = String::with_capacity(src.len()); + let mut index = 0; + while index < chars.len() { + index = match chars[index] { + '/' if chars.get(index + 1) == Some(&'/') => skip_to_line_end(&chars, index), + '"' => copy_string(&chars, index, &mut out), + 'r' if raw_string_hashes(&chars, index).is_some() => { + copy_raw_string(&chars, index, &mut out) + } + '\'' => copy_char_or_lifetime(&chars, index, &mut out), + other => { + out.push(other); + index + 1 + } + }; + } + out +} + +fn skip_to_line_end(chars: &[char], mut index: usize) -> usize { + while index < chars.len() && chars[index] != '\n' { + index += 1; + } + index +} + +/// Copies a `"…"` literal starting at `start`, honouring `\` escapes. +fn copy_string(chars: &[char], start: usize, out: &mut String) -> usize { + out.push('"'); + let mut index = start + 1; + while index < chars.len() { + out.push(chars[index]); + match chars[index] { + '\\' if index + 1 < chars.len() => { + out.push(chars[index + 1]); + index += 2; + } + '"' => return index + 1, + _ => index += 1, + } + } + index +} + +fn is_ident_char(c: char) -> bool { + c.is_alphanumeric() || c == '_' +} + +/// The `#` count of a raw string (`r"`, `r#"`, `br"`…) starting at the `r` at +/// `index`, or `None` when that `r` is part of an identifier or not followed +/// by `#*"`. +fn raw_string_hashes(chars: &[char], index: usize) -> Option { + let before = index.checked_sub(1).map(|i| chars[i]); + let prefix_ok = match before { + None => true, + Some('b') => index < 2 || !is_ident_char(chars[index - 2]), + Some(c) => !is_ident_char(c), + }; + if !prefix_ok { + return None; + } + let hashes = chars[index + 1..].iter().take_while(|&&c| c == '#').count(); + (chars.get(index + 1 + hashes) == Some(&'"')).then_some(hashes) +} + +fn copy_raw_string(chars: &[char], start: usize, out: &mut String) -> usize { + let hashes = raw_string_hashes(chars, start).expect("caller checked the raw-string opener"); + let body = start + 2 + hashes; + out.extend(&chars[start..body]); + let mut index = body; + while index < chars.len() { + let closes = chars[index] == '"' + && chars[index + 1..] + .iter() + .take(hashes) + .filter(|&&c| c == '#') + .count() + == hashes; + if closes { + out.extend(&chars[index..=index + hashes]); + return index + hashes + 1; + } + out.push(chars[index]); + index += 1; + } + index +} + +/// Copies a char literal (`'x'`, `'\''`, `'\u{1F600}'`) or, for a lifetime +/// or label, only its quote. +fn copy_char_or_lifetime(chars: &[char], start: usize, out: &mut String) -> usize { + let end = if chars.get(start + 1) == Some(&'\\') { + chars + .get(start + 2..) + .unwrap_or_default() + .iter() + .position(|&c| c == '\'') + .map(|offset| start + 2 + offset) + } else if chars.get(start + 2) == Some(&'\'') { + Some(start + 2) + } else { + None + }; + match end { + Some(end) => { + out.extend(&chars[start..=end]); + end + 1 + } + None => { + out.push('\''); + start + 1 + } + } +} + +/// Raw devflow spawns in comment-stripped `code`: the two direct +/// `Command::new` forms and Cargo's binary path variable. +fn raw_devflow_spawns(code: &str) -> usize { + let _ = code; + 0 +} + +/// Process-environment mutations in comment-stripped `code`, counted as whole +/// identifiers so `a_set_variable` is not one. +fn env_mutations(code: &str) -> usize { + let _ = code; + 0 +} + +fn read_source(path: &Path) -> String { + fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display())) +} + +fn declares_mod_common(code: &str) -> bool { + code.lines().any(|line| line.trim() == "mod common;") +} + +/// Every `*.rs` directly under `tests/` whose code declares `mod common;`, +/// sorted, with its comment-stripped code. +fn suites_using_common(tests_dir: &Path) -> Vec<(PathBuf, String)> { + let mut paths: Vec = fs::read_dir(tests_dir) + .unwrap_or_else(|err| panic!("read {}: {err}", tests_dir.display())) + .map(|entry| entry.expect("read tests dir entry").path()) + .filter(|path| path.is_file() && path.extension() == Some(OsStr::new("rs"))) + .collect(); + paths.sort(); + paths + .into_iter() + .map(|path| { + let code = code_only(&read_source(&path)); + (path, code) + }) + .filter(|(_, code)| declares_mod_common(code)) + .collect() +} + +#[test] +fn suites_using_common_never_spawn_devflow_raw_or_mutate_the_environment() { + // (1) Planted controls: each scanner flags its planted positive and + // passes the look-alike. + let planted_spawn = concat!("let out = Command::new(", "devflow_bin()).output();"); + assert_eq!( + raw_devflow_spawns(&code_only(planted_spawn)), + 1, + "a planted raw spawn must be flagged" + ); + let commented_spawn = format!("// {planted_spawn}\n"); + assert_eq!( + raw_devflow_spawns(&code_only(&commented_spawn)), + 0, + "a commented spawn must pass" + ); + let planted_mutation = concat!("std::env::", "set_", "var(\"HOME\", \"/x\");"); + assert_eq!( + env_mutations(&code_only(planted_mutation)), + 1, + "a planted env mutation must be flagged" + ); + assert_eq!( + env_mutations(&code_only("let a_set_variable = 1;")), + 0, + "a_set_variable must pass" + ); + + // (2) Every suite that declares `mod common;`; this file always does. + let tests_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests"); + let suites = suites_using_common(&tests_dir); + assert!( + suites + .iter() + .any(|(path, _)| path.ends_with("test_home_isolation.rs")), + "no suite under {} declares `mod common;`, not even test_home_isolation.rs", + tests_dir.display() + ); + + // (3) Raw spawns: none, except exactly the one R-13 control here. + for (path, code) in &suites { + let found = raw_devflow_spawns(code); + let allowed = usize::from(path.ends_with("test_home_isolation.rs")); + assert_eq!( + found, + allowed, + "{} has {found} raw devflow spawn(s); it must have {allowed} \ + (spawn through common::hermetic_devflow; test_home_isolation.rs keeps exactly its R-13 control)", + path.display() + ); + } + + // (4) No process-environment mutation in common/mod.rs or any suite. + let common_path = tests_dir.join("common").join("mod.rs"); + let common_code = code_only(&read_source(&common_path)); + for (path, code) in std::iter::once((&common_path, &common_code)) + .chain(suites.iter().map(|(path, code)| (path, code))) + { + let found = env_mutations(code); + assert_eq!( + found, + 0, + "{} has {found} process-environment mutation(s); set the variable on the child Command", + path.display() + ); + } +} From d48c95ec8ace064c4ce82a785ab07c74f579f9a0 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 07:32:26 -0400 Subject: [PATCH 002/112] feat(50-02): implement hermetic devflow builder and source scan - hermetic_devflow sets HOME, XDG_CACHE_HOME, XDG_CONFIG_HOME and DEVFLOW_CACHE_DIR under a /tmp HermeticHome and removes every repo-local and redirecting GIT_* variable - code_only strips // comments outside string, raw-string and char literals; raw_devflow_spawns and env_mutations (whole identifiers) count the scanned tokens - GREEN: a_child_writing_outside_its_tempdir_home_fails and suites_using_common_never_spawn_devflow_raw_or_mutate_the_environment pass; three mutants (HOME override dropped, set_var planted in common, R-13 control removed) each fail on the intended assertion Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/tests/common/mod.rs | 15 ++++++++++-- .../devflow-cli/tests/test_home_isolation.rs | 24 +++++++++++++++---- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/crates/devflow-cli/tests/common/mod.rs b/crates/devflow-cli/tests/common/mod.rs index 2e5748fd..df7d9515 100644 --- a/crates/devflow-cli/tests/common/mod.rs +++ b/crates/devflow-cli/tests/common/mod.rs @@ -78,8 +78,19 @@ impl Default for HermeticHome { /// repo-redirecting `GIT_*` variable removed. `home` must outlive the child /// and any monitor it launches. pub fn hermetic_devflow(home: &HermeticHome) -> Command { - let _ = home; - Command::new(devflow_bin()) + let mut command = Command::new(devflow_bin()); + command + .env("HOME", home.home()) + .env("XDG_CACHE_HOME", home.home().join(".cache")) + .env("XDG_CONFIG_HOME", home.config_dir()) + .env("DEVFLOW_CACHE_DIR", home.cache_dir()); + for name in devflow_core::test_support::REPO_LOCAL_GIT_VARS + .iter() + .chain(devflow_core::test_support::ALSO_REDIRECTING_GIT_VARS) + { + command.env_remove(name); + } + command } /// A directory of stub programs to put first on a child's `PATH`. Removed on diff --git a/crates/devflow-cli/tests/test_home_isolation.rs b/crates/devflow-cli/tests/test_home_isolation.rs index ab2b2a03..c03e6a55 100644 --- a/crates/devflow-cli/tests/test_home_isolation.rs +++ b/crates/devflow-cli/tests/test_home_isolation.rs @@ -320,15 +320,31 @@ fn copy_char_or_lifetime(chars: &[char], start: usize, out: &mut String) -> usiz /// Raw devflow spawns in comment-stripped `code`: the two direct /// `Command::new` forms and Cargo's binary path variable. fn raw_devflow_spawns(code: &str) -> usize { - let _ = code; - 0 + [ + concat!("Command::new(", "devflow_bin())"), + concat!("Command::new(common::", "devflow_bin())"), + concat!("CARGO_BIN_EXE_", "devflow"), + ] + .iter() + .map(|token| code.matches(token).count()) + .sum() } /// Process-environment mutations in comment-stripped `code`, counted as whole /// identifiers so `a_set_variable` is not one. fn env_mutations(code: &str) -> usize { - let _ = code; - 0 + [concat!("set_", "var"), concat!("remove_", "var")] + .iter() + .map(|token| { + code.match_indices(token) + .filter(|&(at, _)| { + let before = code[..at].chars().next_back(); + let after = code[at + token.len()..].chars().next(); + !before.is_some_and(is_ident_char) && !after.is_some_and(is_ident_char) + }) + .count() + }) + .sum() } fn read_source(path: &Path) -> String { From 2d4fbf52d0e68a4c09df0797e80dcefee5355aef Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 07:35:09 -0400 Subject: [PATCH 003/112] test(50-02): run start_lock_e2e and stop_e2e through the hermetic builder - both suites declare mod common; and spawn devflow only through common::hermetic_devflow with a per-test HermeticHome bound before the spawn - start_lock_e2e drops its local devflow_bin, FakeBin, fake_bin_dir, wait_for, kill_and_reap and LaunchedRunReaper for the common ones; resume_command takes the hermetic home instead of a cache dir - the four start tests that launch a run now bind a LaunchedRunReaper - stop_e2e keeps its own init_repo, wait_for and write_live_lock; its kill_and_reap is replaced by the common one - test counts unchanged (12 and 22); under a sentinel HOME the three suites leave no devflow file, while the pre-migration start_lock_e2e leaves three registry entries there Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/tests/start_lock_e2e.rs | 209 ++++++++------------- crates/devflow-cli/tests/stop_e2e.rs | 101 +++++----- 2 files changed, 140 insertions(+), 170 deletions(-) diff --git a/crates/devflow-cli/tests/start_lock_e2e.rs b/crates/devflow-cli/tests/start_lock_e2e.rs index 87b50be0..082000bf 100644 --- a/crates/devflow-cli/tests/start_lock_e2e.rs +++ b/crates/devflow-cli/tests/start_lock_e2e.rs @@ -4,6 +4,12 @@ //! the compiled CLI as a separate process. That makes the refusal observable //! at the same boundary an operator uses, rather than coupling the test to //! `commands::start`'s implementation details. +//! +//! Every `devflow` child runs through `common::hermetic_devflow`, so its +//! registry entries land in a tempdir `HOME`, never the operator's real +//! `~/.cache/devflow` (999.141). + +mod common; use devflow_core::gates::{GateResponse, Gates}; use devflow_core::mode::Mode; @@ -11,14 +17,9 @@ use devflow_core::phase_id::PhaseId; use devflow_core::stage::Stage; use devflow_core::state::{AgentKind, State}; use std::fs; -use std::os::unix::fs::{PermissionsExt, symlink}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Output}; -use std::time::{Duration, Instant}; - -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} +use std::time::Duration; fn git(root: &Path, args: &[&str]) { let output = devflow_core::test_support::git_command(root) @@ -50,31 +51,16 @@ fn init_repo(root: &Path, phase: PhaseId) { git(root, &["commit", "-q", "-m", "base"]); } -struct FakeBin { - _dir: tempfile::TempDir, - path: PathBuf, -} - -fn fake_bin_dir(agent_body: &str) -> FakeBin { - let dir = tempfile::tempdir().unwrap(); - let claude = dir.path().join("claude"); - fs::write(&claude, format!("#!/bin/sh\n{agent_body}\n")).unwrap(); - let mut permissions = fs::metadata(&claude).unwrap().permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&claude, permissions).unwrap(); - for program in ["git", "sh"] { - symlink(format!("/usr/bin/{program}"), dir.path().join(program)).unwrap(); - } - FakeBin { - path: dir.path().to_path_buf(), - _dir: dir, - } -} - -fn child(root: &Path, phase: PhaseId, fake_bin: &FakeBin, mode: &str) -> Command { +fn child( + home: &common::HermeticHome, + root: &Path, + phase: PhaseId, + fake_bin: &common::FakeBin, + mode: &str, +) -> Command { let existing = std::env::var_os("PATH").unwrap_or_default(); let path = format!("{}:{}", fake_bin.path.display(), existing.to_string_lossy()); - let mut command = Command::new(devflow_bin()); + let mut command = common::hermetic_devflow(home); command .args([ "start", @@ -93,31 +79,14 @@ fn child(root: &Path, phase: PhaseId, fake_bin: &FakeBin, mode: &str) -> Command command } -fn wait_for(mut predicate: impl FnMut() -> bool, timeout: Duration, what: &str) { - let started = Instant::now(); - while !predicate() { - assert!( - started.elapsed() < timeout, - "timed out after {timeout:?} waiting for {what}" - ); - std::thread::sleep(Duration::from_millis(25)); - } -} - -fn kill_and_reap(child: &mut Child) { - if child.try_wait().expect("poll start child").is_none() { - let _ = child.kill(); - } - let _ = child.wait(); -} - #[test] fn start_refuses_while_another_process_holds_the_phase_lock() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); let phase = PhaseId::new(90); init_repo(root, phase); - let fake_bin = fake_bin_dir("exit 0"); + let fake_bin = common::fake_bin_dir("exit 0"); let holder = devflow_core::lock::acquire(root, phase).expect("test holds phase lock"); let holder_pid = std::process::id(); let worktrees_before = devflow_core::test_support::git_command(root) @@ -126,7 +95,7 @@ fn start_refuses_while_another_process_holds_the_phase_lock() { .expect("list baseline worktrees"); assert!(worktrees_before.status.success()); - let output = child(root, phase, &fake_bin, "supervise") + let output = child(&home, root, phase, &fake_bin, "supervise") .output() .expect("run devflow start"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -172,13 +141,15 @@ fn start_refuses_while_another_process_holds_the_phase_lock() { #[test] fn start_proceeds_when_no_process_holds_the_phase_lock() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); let phase = PhaseId::new(91); init_repo(root, phase); - let fake_bin = fake_bin_dir("exit 0"); + let fake_bin = common::fake_bin_dir("exit 0"); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); - let output = child(root, phase, &fake_bin, "supervise") + let output = child(&home, root, phase, &fake_bin, "supervise") .output() .expect("run devflow start"); assert!( @@ -200,11 +171,12 @@ fn start_proceeds_when_no_process_holds_the_phase_lock() { #[test] fn start_clears_leftover_gate_files_after_taking_the_lock() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); let phase = PhaseId::new(92); init_repo(root, phase); - let fake_bin = fake_bin_dir("sleep 60"); + let fake_bin = common::fake_bin_dir("sleep 60"); let response_path = Gates::response_path(root, phase, Stage::Define); fs::create_dir_all(response_path.parent().unwrap()).unwrap(); fs::write( @@ -217,12 +189,13 @@ fn start_clears_leftover_gate_files_after_taking_the_lock() { .unwrap(), ) .unwrap(); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); - let mut process = child(root, phase, &fake_bin, "auto") + let mut process = child(&home, root, phase, &fake_bin, "auto") .env("DEVFLOW_GATE_TIMEOUT_SECS", "30") .spawn() .expect("spawn devflow start"); - wait_for( + common::wait_for( || { Gates::gate_path(root, phase, Stage::Define).exists() || process.try_wait().expect("poll start child").is_some() @@ -249,7 +222,7 @@ fn start_clears_leftover_gate_files_after_taking_the_lock() { !response_path.exists(), "the planted response must be removed" ); - kill_and_reap(&mut process); + common::kill_and_reap(&mut process); } // --------------------------------------------------------------------------- @@ -374,8 +347,14 @@ fn snapshot_paths(snapshot: &[(PathBuf, Vec)]) -> Vec<&Path> { snapshot.iter().map(|(path, _)| path.as_path()).collect() } -fn run_start(root: &Path, phase: PhaseId, fake_bin: &FakeBin, extra: &[&str]) -> Output { - child(root, phase, fake_bin, "supervise") +fn run_start( + home: &common::HermeticHome, + root: &Path, + phase: PhaseId, + fake_bin: &common::FakeBin, + extra: &[&str], +) -> Output { + child(home, root, phase, fake_bin, "supervise") .args(extra) .env("DEVFLOW_GATE_TIMEOUT_SECS", GATE_TIMEOUT_SECS) .output() @@ -393,13 +372,14 @@ fn events_contain_workflow_started(root: &Path) -> bool { /// the absence of every side effect. `refusal` is the first assertion's /// message and names the arm. fn assert_start_refuses_live_run( + home: &common::HermeticHome, root: &Path, phase: PhaseId, extra: &[&str], live_pid: u32, refusal: &str, ) { - let fake_bin = fake_bin_dir("exit 0"); + let fake_bin = common::fake_bin_dir("exit 0"); let before = phase_file_snapshot(root, phase); // Negative control on the snapshot itself: it must cover the planted // state and both planted gate files, or byte-identity would prove nothing. @@ -416,7 +396,7 @@ fn assert_start_refuses_live_run( ); } - let output = run_start(root, phase, &fake_bin, extra); + let output = run_start(home, root, phase, &fake_bin, extra); let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); assert!( @@ -517,6 +497,7 @@ fn assert_start_refuses_live_run( #[test] fn start_refuses_a_phase_whose_recorded_monitor_is_alive() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); let phase = PhaseId::new(95); @@ -525,6 +506,7 @@ fn start_refuses_a_phase_whose_recorded_monitor_is_alive() { plant_leftover_run(root, phase, Some(monitor.pid()), None); assert_start_refuses_live_run( + &home, root, phase, &[], @@ -535,6 +517,7 @@ fn start_refuses_a_phase_whose_recorded_monitor_is_alive() { #[test] fn start_refuses_a_phase_whose_recorded_agent_is_alive() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); let phase = PhaseId::new(96); @@ -543,6 +526,7 @@ fn start_refuses_a_phase_whose_recorded_agent_is_alive() { plant_leftover_run(root, phase, None, Some(agent.pid())); assert_start_refuses_live_run( + &home, root, phase, &["--force"], @@ -553,14 +537,16 @@ fn start_refuses_a_phase_whose_recorded_agent_is_alive() { #[test] fn start_replaces_a_leftover_state_whose_recorded_processes_are_dead() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); let phase = PhaseId::new(97); init_repo(root, phase); let planted = plant_leftover_run(root, phase, Some(NEVER_LIVE_PID), Some(NEVER_LIVE_PID)); - let fake_bin = fake_bin_dir("exit 0"); + let fake_bin = common::fake_bin_dir("exit 0"); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); - let output = run_start(root, phase, &fake_bin, &[]); + let output = run_start(&home, root, phase, &fake_bin, &[]); assert!( output.status.success(), "a leftover state whose recorded processes are dead must be replaced as today\n\ @@ -586,6 +572,7 @@ fn start_replaces_a_leftover_state_whose_recorded_processes_are_dead() { #[test] fn start_proceeds_when_no_state_exists_even_if_the_agent_pid_file_names_a_live_process() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); let phase = PhaseId::new(98); @@ -596,9 +583,10 @@ fn start_proceeds_when_no_state_exists_even_if_the_agent_pid_file_names_a_live_p !devflow_core::workflow::state_path(root, phase).exists(), "the carve-out fixture must have no phase state" ); - let fake_bin = fake_bin_dir("exit 0"); + let fake_bin = common::fake_bin_dir("exit 0"); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![agent.pid()]); - let output = run_start(root, phase, &fake_bin, &[]); + let output = run_start(&home, root, phase, &fake_bin, &[]); assert!( output.status.success(), "without phase state, start must proceed even if the agent pid file names a live process\n\ @@ -655,10 +643,17 @@ fn plant_resumable_run( } } -fn resume_command(root: &Path, phase: PhaseId, fake_bin: &FakeBin, cache_dir: &Path) -> Command { +/// `devflow resume` for `phase`; its `DEVFLOW_CACHE_DIR` is `home`'s cache dir, +/// set by `common::hermetic_devflow`. +fn resume_command( + home: &common::HermeticHome, + root: &Path, + phase: PhaseId, + fake_bin: &common::FakeBin, +) -> Command { let existing = std::env::var_os("PATH").unwrap_or_default(); let path = format!("{}:{}", fake_bin.path.display(), existing.to_string_lossy()); - let mut command = Command::new(devflow_bin()); + let mut command = common::hermetic_devflow(home); command .args([ "resume", @@ -669,49 +664,10 @@ fn resume_command(root: &Path, phase: PhaseId, fake_bin: &FakeBin, cache_dir: &P .arg(root) .current_dir(root) .env("PATH", path) - .env("DEVFLOW_GATE_TIMEOUT_SECS", GATE_TIMEOUT_SECS) - .env("DEVFLOW_CACHE_DIR", cache_dir); + .env("DEVFLOW_GATE_TIMEOUT_SECS", GATE_TIMEOUT_SECS); command } -struct LaunchedRunReaper { - root: PathBuf, - phase: PhaseId, - spare: Vec, -} - -impl LaunchedRunReaper { - fn new(root: &Path, phase: PhaseId, spare: Vec) -> Self { - Self { - root: root.to_path_buf(), - phase, - spare, - } - } -} - -impl Drop for LaunchedRunReaper { - fn drop(&mut self) { - let monitor = devflow_core::workflow::load_state(&self.root, self.phase) - .ok() - .and_then(|state| state.monitor_pid); - let agent = fs::read_to_string(devflow_core::agent_result::agent_pid_path( - &self.root, self.phase, - )) - .ok() - .and_then(|pid| pid.trim().parse::().ok()); - for pid in [monitor, agent].into_iter().flatten() { - if !self.spare.contains(&pid) && devflow_core::agent::agent_running(pid) { - let _ = devflow_core::agent::terminate_and_verify( - pid, - devflow_core::agent::TERMINATE_VERIFY_WAIT, - devflow_core::agent::TERMINATE_VERIFY_POLL, - ); - } - } - } -} - struct RefusalCase { role: &'static str, live_pid: u32, @@ -721,10 +677,10 @@ struct RefusalCase { } fn assert_resume_refuses_live_run( + home: &common::HermeticHome, root: &Path, phase: PhaseId, - fake_bin: &FakeBin, - cache_dir: &Path, + fake_bin: &common::FakeBin, case: &RefusalCase, ) { let before = phase_file_snapshot(root, phase); @@ -737,7 +693,7 @@ fn assert_resume_refuses_live_run( ); } - let output = resume_command(root, phase, fake_bin, cache_dir) + let output = resume_command(home, root, phase, fake_bin) .output() .expect("run devflow resume"); let stdout = String::from_utf8_lossy(&output.stdout); @@ -824,13 +780,13 @@ fn assert_resume_refuses_live_run( #[test] fn resume_refuses_a_phase_whose_recorded_monitor_is_alive() { let repo = tempfile::tempdir().unwrap(); - let cache = tempfile::tempdir().unwrap(); + let home = common::HermeticHome::new(); let root = repo.path(); let phase = PhaseId::new(85); init_repo(root, phase); let monitor = LiveProcess::spawn(); - let fake_bin = fake_bin_dir("exec sleep 60"); - let _reaper = LaunchedRunReaper::new(root, phase, vec![monitor.pid()]); + let fake_bin = common::fake_bin_dir("exec sleep 60"); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![monitor.pid()]); plant_resumable_run( root, phase, @@ -840,10 +796,10 @@ fn resume_refuses_a_phase_whose_recorded_monitor_is_alive() { ); assert_resume_refuses_live_run( + &home, root, phase, &fake_bin, - cache.path(), &RefusalCase { role: "monitor", live_pid: monitor.pid(), @@ -860,13 +816,13 @@ fn resume_refuses_a_phase_whose_recorded_monitor_is_alive() { #[test] fn resume_refuses_a_stopped_phase_whose_recorded_agent_is_alive() { let repo = tempfile::tempdir().unwrap(); - let cache = tempfile::tempdir().unwrap(); + let home = common::HermeticHome::new(); let root = repo.path(); let phase = PhaseId::new(86); init_repo(root, phase); let agent = LiveProcess::spawn(); - let fake_bin = fake_bin_dir("exec sleep 60"); - let _reaper = LaunchedRunReaper::new(root, phase, vec![agent.pid()]); + let fake_bin = common::fake_bin_dir("exec sleep 60"); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![agent.pid()]); plant_resumable_run(root, phase, Some(NEVER_LIVE_PID), Some(agent.pid()), true); let case = RefusalCase { role: "agent", @@ -879,8 +835,8 @@ fn resume_refuses_a_stopped_phase_whose_recorded_agent_is_alive() { refusal: "resume must refuse a phase whose recorded agent is alive", }; - assert_resume_refuses_live_run(root, phase, &fake_bin, cache.path(), &case); - assert_resume_refuses_live_run(root, phase, &fake_bin, cache.path(), &case); + assert_resume_refuses_live_run(&home, root, phase, &fake_bin, &case); + assert_resume_refuses_live_run(&home, root, phase, &fake_bin, &case); assert!( devflow_core::workflow::load_state(root, phase) .expect("load planted state") @@ -893,11 +849,11 @@ fn resume_refuses_a_stopped_phase_whose_recorded_agent_is_alive() { fn resume_relaunches_a_phase_whose_recorded_processes_are_dead() { for (phase, stopped) in [(PhaseId::new(87), false), (PhaseId::new(88), true)] { let repo = tempfile::tempdir().unwrap(); - let cache = tempfile::tempdir().unwrap(); + let home = common::HermeticHome::new(); let root = repo.path(); init_repo(root, phase); - let fake_bin = fake_bin_dir("exec sleep 60"); - let _reaper = LaunchedRunReaper::new(root, phase, vec![]); + let fake_bin = common::fake_bin_dir("exec sleep 60"); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); plant_resumable_run( root, phase, @@ -906,7 +862,7 @@ fn resume_relaunches_a_phase_whose_recorded_processes_are_dead() { stopped, ); - let output = resume_command(root, phase, &fake_bin, cache.path()) + let output = resume_command(&home, root, phase, &fake_bin) .output() .expect("run devflow resume"); assert!( @@ -928,12 +884,12 @@ fn resume_relaunches_a_phase_whose_recorded_processes_are_dead() { #[test] fn resume_without_phase_state_reports_the_missing_state_not_a_live_run() { let repo = tempfile::tempdir().unwrap(); - let cache = tempfile::tempdir().unwrap(); + let home = common::HermeticHome::new(); let root = repo.path(); let phase = PhaseId::new(84); init_repo(root, phase); let agent = LiveProcess::spawn(); - let fake_bin = fake_bin_dir("exec sleep 60"); + let fake_bin = common::fake_bin_dir("exec sleep 60"); write_agent_pid(root, phase, agent.pid()); assert!( !devflow_core::workflow::state_path(root, phase).exists(), @@ -941,7 +897,7 @@ fn resume_without_phase_state_reports_the_missing_state_not_a_live_run() { ); let before = phase_file_snapshot(root, phase); - let output = resume_command(root, phase, &fake_bin, cache.path()) + let output = resume_command(&home, root, phase, &fake_bin) .output() .expect("run devflow resume"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -961,15 +917,16 @@ fn resume_without_phase_state_reports_the_missing_state_not_a_live_run() { #[test] fn start_refusal_text_stays_byte_identical_for_a_live_monitor() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); let phase = PhaseId::new(94); init_repo(root, phase); let monitor = LiveProcess::spawn(); - let fake_bin = fake_bin_dir("exit 0"); + let fake_bin = common::fake_bin_dir("exit 0"); plant_leftover_run(root, phase, Some(monitor.pid()), None); - let output = run_start(root, phase, &fake_bin, &[]); + let output = run_start(&home, root, phase, &fake_bin, &[]); let stderr = String::from_utf8_lossy(&output.stderr); assert!( !output.status.success(), diff --git a/crates/devflow-cli/tests/stop_e2e.rs b/crates/devflow-cli/tests/stop_e2e.rs index 8f3f7bac..91874397 100644 --- a/crates/devflow-cli/tests/stop_e2e.rs +++ b/crates/devflow-cli/tests/stop_e2e.rs @@ -10,6 +10,8 @@ //! `gate_sweep_e2e.rs`) — this drives the real compiled binary via //! `Command::new`, not an in-process call. +mod common; + use devflow_core::gates::{GateResponse, Gates}; use devflow_core::mode::Mode; use devflow_core::phase_id::PhaseId; @@ -21,10 +23,6 @@ use std::path::Path; use std::process::{Child, Command}; use std::time::{Duration, Instant}; -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} - /// Hermetic `git` invocation for fixture setup (999.37) — never a bare /// `Command::new("git")`. fn git(root: &Path, args: &[&str]) { @@ -97,15 +95,9 @@ fn write_live_lock(root: &Path, phase: PhaseId, child: &Child, recorded_start: u std::fs::write(lock, format!("{}\n{recorded_start}", child.id())).unwrap(); } -fn kill_and_reap(child: &mut Child) { - if child.try_wait().expect("poll holder").is_none() { - let _ = child.kill(); - } - let _ = child.wait(); -} - #[test] fn stop_marks_stopped_after_the_signalled_lock_holder_exits() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(103); @@ -118,7 +110,7 @@ fn stop_marks_stopped_after_the_signalled_lock_holder_exits() { let start = devflow_core::agent::process_start_time(holder.id()).expect("holder start time"); write_live_lock(root, phase, &holder, start); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -141,6 +133,7 @@ fn stop_marks_stopped_after_the_signalled_lock_holder_exits() { #[test] fn stop_writes_no_state_while_the_lock_holder_survives_the_signal() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(104); @@ -155,7 +148,7 @@ fn stop_writes_no_state_while_the_lock_holder_survives_the_signal() { let start = devflow_core::agent::process_start_time(holder.id()).expect("holder start time"); write_live_lock(root, phase, &holder, start); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -170,11 +163,12 @@ fn stop_writes_no_state_while_the_lock_holder_survives_the_signal() { assert!(stderr.contains(&holder.id().to_string()) && stderr.contains("not marked")); assert_eq!(std::fs::read(&state_path).unwrap(), before); assert!(holder.try_wait().expect("poll holder").is_none()); - kill_and_reap(&mut holder); + common::kill_and_reap(&mut holder); } #[test] fn stop_never_signals_the_recorded_monitor_pid() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(105); @@ -186,7 +180,7 @@ fn stop_never_signals_the_recorded_monitor_pid() { state.monitor_pid = Some(monitor.id()); devflow_core::workflow::save_state(&state).unwrap(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -198,11 +192,12 @@ fn stop_never_signals_the_recorded_monitor_pid() { .unwrap() .stopped ); - kill_and_reap(&mut monitor); + common::kill_and_reap(&mut monitor); } #[test] fn stop_refuses_to_signal_a_lock_holder_whose_start_time_does_not_match() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(106); @@ -218,7 +213,7 @@ fn stop_refuses_to_signal_a_lock_holder_whose_start_time_does_not_match() { devflow_core::agent::process_start_time(holder.id()).expect("holder start time") + 1; write_live_lock(root, phase, &holder, wrong_start); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -227,7 +222,7 @@ fn stop_refuses_to_signal_a_lock_holder_whose_start_time_does_not_match() { assert!(String::from_utf8_lossy(&output.stderr).contains("refusing to signal")); assert!(holder.try_wait().expect("poll holder").is_none()); assert_eq!(std::fs::read(&state_path).unwrap(), before); - kill_and_reap(&mut holder); + common::kill_and_reap(&mut holder); } /// Saves a fresh state for `phase`, starts a live `sleep` as the process the @@ -253,6 +248,7 @@ fn gated_phase_with_foreign_holder( /// gate for the real recovery path. #[test] fn stop_at_a_ship_gate_with_a_recycled_holder_refuses_without_writing_a_response() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(108); @@ -267,7 +263,7 @@ fn stop_at_a_ship_gate_with_a_recycled_holder_refuses_without_writing_a_response write_live_lock(root, phase, &holder, wrong_start); Gates::write_gate(root, phase, Stage::Ship, "merge").unwrap(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -292,7 +288,7 @@ fn stop_at_a_ship_gate_with_a_recycled_holder_refuses_without_writing_a_response ); assert!(holder.try_wait().expect("poll holder").is_none()); assert_eq!(std::fs::read(&state_path).unwrap(), before); - kill_and_reap(&mut holder); + common::kill_and_reap(&mut holder); } /// Finding B: a legacy single-line lock makes the holder's identity @@ -300,6 +296,7 @@ fn stop_at_a_ship_gate_with_a_recycled_holder_refuses_without_writing_a_response /// response that no confirmed process can consume. #[test] fn stop_at_a_gate_with_an_unconfirmable_holder_refuses_unevidenced_success() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(109); @@ -315,7 +312,7 @@ fn stop_at_a_gate_with_an_unconfirmable_holder_refuses_unevidenced_success() { std::fs::write(lock, holder.id().to_string()).unwrap(); Gates::write_gate(root, phase, Stage::Code, "paused").unwrap(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -345,7 +342,7 @@ fn stop_at_a_gate_with_an_unconfirmable_holder_refuses_unevidenced_success() { ); assert!(holder.try_wait().expect("poll holder").is_none()); assert_eq!(std::fs::read(&state_path).unwrap(), before); - kill_and_reap(&mut holder); + common::kill_and_reap(&mut holder); } /// Finding C-3: an old gate request is not evidence that the current live @@ -353,6 +350,7 @@ fn stop_at_a_gate_with_an_unconfirmable_holder_refuses_unevidenced_success() { /// writing an abort response and reporting an unevidenced successful stop. #[test] fn stop_does_not_treat_a_stale_gate_as_the_live_holders_wait() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(112); @@ -369,7 +367,7 @@ fn stop_does_not_treat_a_stale_gate_as_the_live_holders_wait() { write_live_lock(root, phase, &holder, start); Gates::write_gate(root, phase, Stage::Code, "stale request").unwrap(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -381,7 +379,7 @@ fn stop_does_not_treat_a_stale_gate_as_the_live_holders_wait() { String::from_utf8_lossy(&output.stderr) ); let Some(status) = holder.try_wait().expect("poll holder") else { - kill_and_reap(&mut holder); + common::kill_and_reap(&mut holder); panic!("stop must signal the live holder instead of trusting the stale gate"); }; assert!(!status.success(), "SIGTERM must end the live holder"); @@ -403,12 +401,13 @@ fn stop_does_not_treat_a_stale_gate_as_the_live_holders_wait() { /// live holder exists. #[test] fn gate_approve_without_a_live_holder_does_not_claim_advance() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(110); Gates::write_gate(root, phase, Stage::Ship, "approve merge").unwrap(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["gate", "approve", &phase.to_string(), "--project"]) .arg(root) .output() @@ -476,6 +475,7 @@ fn wait_for_child_exit( /// class `23-ORPHAN-FORENSICS.md` documented, now remedied. #[test] fn stop_ends_a_gated_phase_through_its_own_abort_path_with_no_signal_sent() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(95); @@ -490,7 +490,7 @@ fn stop_ends_a_gated_phase_through_its_own_abort_path_with_no_signal_sent() { // failed stop still lets `wait_for_child_exit`'s cleanup `wait()` // return promptly. Set per-`Command` via `.env(...)`, never via a // process-global env mutation (999.37). - let mut child = Command::new(devflow_bin()) + let mut child = common::hermetic_devflow(&home) .args(["advance", "--phase", &phase.to_string()]) .arg(root) .env("DEVFLOW_GATE_TIMEOUT_SECS", "15") @@ -518,7 +518,7 @@ fn stop_ends_a_gated_phase_through_its_own_abort_path_with_no_signal_sent() { blocking on it" ); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -566,6 +566,7 @@ fn stop_ends_a_gated_phase_through_its_own_abort_path_with_no_signal_sent() { /// may race a second writer). #[test] fn stop_marks_state_stopped_and_records_reason() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(96); @@ -576,7 +577,7 @@ fn stop_marks_state_stopped_and_records_reason() { state.gate_pending = true; devflow_core::workflow::save_state(&state).unwrap(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -603,6 +604,7 @@ fn stop_marks_state_stopped_and_records_reason() { /// `--until` halt) must survive — appended to, not overwritten. #[test] fn stop_preserves_pre_existing_stop_reason() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(97); @@ -612,7 +614,7 @@ fn stop_preserves_pre_existing_stop_reason() { state.stop_reason = Some("stopped after plan completed (--until plan)".to_string()); devflow_core::workflow::save_state(&state).unwrap(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -636,6 +638,7 @@ fn stop_preserves_pre_existing_stop_reason() { /// halt stage for `devflow start --until`) and must be left untouched. #[test] fn stop_leaves_stop_until_unchanged() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(98); @@ -644,7 +647,7 @@ fn stop_leaves_stop_until_unchanged() { state.stop_until = Some(Stage::Plan); devflow_core::workflow::save_state(&state).unwrap(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -665,7 +668,8 @@ fn stop_leaves_stop_until_unchanged() { /// verification map. #[test] fn stop_help_documents_phase_flag() { - let output = Command::new(devflow_bin()) + let home = common::HermeticHome::new(); + let output = common::hermetic_devflow(&home) .args(["stop", "--help"]) .output() .expect("run devflow stop --help"); @@ -689,6 +693,7 @@ fn stop_help_documents_phase_flag() { /// one wrote. #[test] fn stop_is_idempotent_against_an_already_answered_gate() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(99); @@ -700,7 +705,7 @@ fn stop_is_idempotent_against_an_already_answered_gate() { devflow_core::workflow::save_state(&state).unwrap(); let run_stop = || { - Command::new(devflow_bin()) + common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -735,6 +740,7 @@ fn stop_is_idempotent_against_an_already_answered_gate() { /// success no-op here, not a clobber. #[test] fn stop_against_a_hand_written_response_is_a_success_no_op() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(100); @@ -754,7 +760,7 @@ fn stop_against_a_hand_written_response_is_a_success_no_op() { let response_path = Gates::response_path(root, phase, Stage::Ship); let before = std::fs::read(&response_path).unwrap(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -777,11 +783,12 @@ fn stop_against_a_hand_written_response_is_a_success_no_op() { /// not an error. #[test] fn stop_against_a_root_with_no_state_is_a_success() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(101); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -800,6 +807,7 @@ fn stop_against_a_root_with_no_state_is_a_success() { /// fail-closed guarantee is provably unweakened by this plan. #[test] fn stop_then_cleanup_composes_refuse_then_force() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(102); @@ -832,7 +840,7 @@ fn stop_then_cleanup_composes_refuse_then_force() { // No open gate, no lock — the lock-holder fallback is a no-op, and the // only observable effect of this stop is state.stopped/stop_reason. - let stopped = Command::new(devflow_bin()) + let stopped = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output() @@ -850,7 +858,7 @@ fn stop_then_cleanup_composes_refuse_then_force() { // cleanup without --force must still decline — the fail-closed // guarantee this plan's design depends on and does not weaken. - let refused = Command::new(devflow_bin()) + let refused = common::hermetic_devflow(&home) .args(["cleanup"]) .arg(root) .output() @@ -867,7 +875,7 @@ fn stop_then_cleanup_composes_refuse_then_force() { // cleanup --force is the documented escape hatch — the two verbs // compose in that order. - let forced = Command::new(devflow_bin()) + let forced = common::hermetic_devflow(&home) .args(["cleanup", "--force"]) .arg(root) .output() @@ -893,6 +901,7 @@ fn stop_then_cleanup_composes_refuse_then_force() { /// root and the stop is asserted to have landed THERE. #[test] fn stop_accepts_the_project_root_as_a_positional_argument() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(103); @@ -900,7 +909,7 @@ fn stop_accepts_the_project_root_as_a_positional_argument() { let state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); devflow_core::workflow::save_state(&state).unwrap(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string()]) .arg(root) .output() @@ -940,6 +949,7 @@ fn stop_accepts_the_project_root_as_a_positional_argument() { /// measurement of PRECEDENCE rather than of the walk. #[test] fn stop_root_flag_takes_precedence_over_the_positional() { + let home = common::HermeticHome::new(); let dir_a = tempfile::tempdir().unwrap(); let dir_b = tempfile::tempdir().unwrap(); let root_a = dir_a.path(); @@ -959,7 +969,7 @@ fn stop_root_flag_takes_precedence_over_the_positional() { // Ordering 1 — flag names A, positional names B. seed(root_a); seed(root_b); - let first = Command::new(devflow_bin()) + let first = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root_a) .arg(root_b) @@ -979,7 +989,7 @@ fn stop_root_flag_takes_precedence_over_the_positional() { // Ordering 2 — roles reversed: flag names B, positional names A. seed(root_a); seed(root_b); - let second = Command::new(devflow_bin()) + let second = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root_b) .arg(root_a) @@ -1008,6 +1018,7 @@ fn stop_root_flag_takes_precedence_over_the_positional() { /// discriminate. #[test] fn stop_against_a_nonexistent_positional_root_names_the_offending_path() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let missing = dir.path().join("no-such-project-root"); let phase = PhaseId::new(105); @@ -1017,7 +1028,7 @@ fn stop_against_a_nonexistent_positional_root_names_the_offending_path() { "fixture error: the offending path must NOT exist" ); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string()]) .arg(&missing) .output() @@ -1058,10 +1069,11 @@ fn stop_against_a_nonexistent_positional_root_names_the_offending_path() { /// is the precise legibility failure the plan was worried about. #[test] fn stop_with_an_empty_project_root_is_refused_identically_for_both_spellings() { + let home = common::HermeticHome::new(); let phase = PhaseId::new(106); // Spelling 1 — empty POSITIONAL. - let positional = Command::new(devflow_bin()) + let positional = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string()]) .arg("") .output() @@ -1085,7 +1097,7 @@ fn stop_with_an_empty_project_root_is_refused_identically_for_both_spellings() { ); // Spelling 2 — empty value supplied to the long FLAG. - let flag = Command::new(devflow_bin()) + let flag = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg("") .output() @@ -1120,6 +1132,7 @@ fn stop_with_an_empty_project_root_is_refused_identically_for_both_spellings() { #[cfg(unix)] #[test] fn stop_positional_root_survives_a_non_utf8_path() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let phase = PhaseId::new(107); @@ -1138,7 +1151,7 @@ fn stop_positional_root_survives_a_non_utf8_path() { "fixture error: the offending path must NOT exist" ); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["stop", "--phase", &phase.to_string()]) .arg(&missing) .output() From 4f18d28e6d0b463fd2433439bcc9a5924e16b94d Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 07:48:54 -0400 Subject: [PATCH 004/112] test(50-03): run phase7_cli and gate e2e suites hermetically - phase7_cli, gate_wedge_e2e and gate_notify_persist_e2e declare `mod common;` and spawn devflow only through common::hermetic_devflow - gate_notify_persist_e2e: isolated_cache/ReapOnDrop replaced by a test-level HermeticHome and a ParkedChild guard on common::kill_and_reap - LaunchedRunReaper bound right after every spawn that launches a run - no assertion, argument list or test count changed (28/5/9) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- .../tests/gate_notify_persist_e2e.rs | 97 +++++++------- crates/devflow-cli/tests/gate_wedge_e2e.rs | 57 +++++---- crates/devflow-cli/tests/phase7_cli.rs | 120 ++++++++++++++---- 3 files changed, 177 insertions(+), 97 deletions(-) diff --git a/crates/devflow-cli/tests/gate_notify_persist_e2e.rs b/crates/devflow-cli/tests/gate_notify_persist_e2e.rs index 6ab10fed..6bd38dcf 100644 --- a/crates/devflow-cli/tests/gate_notify_persist_e2e.rs +++ b/crates/devflow-cli/tests/gate_notify_persist_e2e.rs @@ -21,6 +21,8 @@ //! The variable is set or removed on each spawned child only (TEST-01), so the //! developer's own shell hook never reaches these runs. +mod common; + use devflow_core::gates::{GATE_NOTIFY_CMD_ENV, Gates}; use devflow_core::mode::Mode; use devflow_core::phase_id::PhaseId; @@ -32,10 +34,6 @@ use std::path::{Path, PathBuf}; use std::process::{Child, Command}; use std::time::{Duration, Instant}; -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} - fn git(root: &Path, args: &[&str]) { let output = devflow_core::test_support::git_command(root) .args(args) @@ -109,29 +107,16 @@ fn fake_bin_dir() -> FakeBin { } /// Owns a spawned `devflow` process so a panicking assertion never leaks a -/// parked process for the length of its gate timeout, and the private cache -/// directory it registers in (`Drop::drop` reaps the child before the field -/// is removed). -struct ReapOnDrop( - Child, - #[expect(dead_code, reason = "held only so the cache dir outlives the child")] - tempfile::TempDir, -); - -/// A private `DEVFLOW_CACHE_DIR` for one spawned child, so its root -/// registration never lands in the developer's real `~/.cache/devflow`. -fn isolated_cache(command: &mut Command) -> tempfile::TempDir { - let cache = tempfile::tempdir().unwrap(); - command.env("DEVFLOW_CACHE_DIR", cache.path()); - cache -} - -impl Drop for ReapOnDrop { +/// parked process for the length of its gate timeout: dropping it kills and +/// reaps the child through [`common::kill_and_reap`]. The child is built by +/// [`common::hermetic_devflow`], so its root registration lands in the test's +/// [`common::HermeticHome`], which each test binds before this guard and so +/// outlives it. +struct ParkedChild(Child); + +impl Drop for ParkedChild { fn drop(&mut self) { - if matches!(self.0.try_wait(), Ok(None)) { - let _ = self.0.kill(); - } - let _ = self.0.wait(); + common::kill_and_reap(&mut self.0); } } @@ -148,7 +133,7 @@ fn apply_notify_env(command: &mut Command, notify: NotifyEnv<'_>) { }; } -fn wait_until_parked(child: &mut ReapOnDrop, gate: &Path, what: &str) { +fn wait_until_parked(child: &mut ParkedChild, gate: &Path, what: &str) { let waited = Instant::now(); while !gate.exists() { if let Some(status) = child.0.try_wait().expect("poll child") { @@ -181,24 +166,26 @@ fn wait_for_notify_fired(root: &Path) { } fn spawn_parked_start( + home: &common::HermeticHome, root: &Path, phase: PhaseId, fake_bin: &FakeBin, notify: NotifyEnv<'_>, -) -> ReapOnDrop { - spawn_parked_start_with(root, phase, fake_bin, notify, true) +) -> ParkedChild { + spawn_parked_start_with(home, root, phase, fake_bin, notify, true) } fn spawn_parked_start_with( + home: &common::HermeticHome, root: &Path, phase: PhaseId, fake_bin: &FakeBin, notify: NotifyEnv<'_>, legacy_launch: bool, -) -> ReapOnDrop { +) -> ParkedChild { let existing = std::env::var_os("PATH").unwrap_or_default(); let path = format!("{}:{}", fake_bin.path.display(), existing.to_string_lossy()); - let mut command = Command::new(devflow_bin()); + let mut command = common::hermetic_devflow(home); command .args([ "start", @@ -217,8 +204,7 @@ fn spawn_parked_start_with( .env("DEVFLOW_GATE_TIMEOUT_SECS", "30") .env(TEST_MARKER_ENV, root); apply_notify_env(&mut command, notify); - let cache = isolated_cache(&mut command); - let mut start = ReapOnDrop(command.spawn().expect("spawn foreground start"), cache); + let mut start = ParkedChild(command.spawn().expect("spawn foreground start")); wait_until_parked( &mut start, &Gates::gate_path(root, phase, Stage::Define), @@ -228,23 +214,23 @@ fn spawn_parked_start_with( } fn spawn_parked_advance( + home: &common::HermeticHome, root: &Path, phase: PhaseId, persisted: Option<&str>, notify: NotifyEnv<'_>, -) -> ReapOnDrop { +) -> ParkedChild { let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); state.stage = Stage::Code; state.gate_notify_cmd = persisted.map(str::to_owned); devflow_core::workflow::save_state(&state).unwrap(); - let mut command = Command::new(devflow_bin()); + let mut command = common::hermetic_devflow(home); command .args(["advance", "--phase", &phase.to_string()]) .arg(root) .env("DEVFLOW_GATE_TIMEOUT_SECS", "30"); apply_notify_env(&mut command, notify); - let cache = isolated_cache(&mut command); - let mut advance = ReapOnDrop(command.spawn().expect("spawn foreground advance"), cache); + let mut advance = ParkedChild(command.spawn().expect("spawn foreground advance")); wait_until_parked( &mut advance, &Gates::gate_path(root, phase, Stage::Code), @@ -261,6 +247,7 @@ fn persisted_cmd(root: &Path, phase: PhaseId) -> Option { #[test] fn start_persists_the_notify_command_from_the_environment() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(94); @@ -269,26 +256,28 @@ fn start_persists_the_notify_command_from_the_environment() { let cmd = format!("touch {}", sentinel.path().join("fired").display()); let fake_bin = fake_bin_dir(); - let _start = spawn_parked_start(root, phase, &fake_bin, NotifyEnv::Set(&cmd)); + let _start = spawn_parked_start(&home, root, phase, &fake_bin, NotifyEnv::Set(&cmd)); assert_eq!(persisted_cmd(root, phase).as_deref(), Some(cmd.as_str())); } #[test] fn start_with_an_empty_notify_variable_persists_nothing() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(94); init_start_repo(root, phase); let fake_bin = fake_bin_dir(); - let _start = spawn_parked_start(root, phase, &fake_bin, NotifyEnv::Set("")); + let _start = spawn_parked_start(&home, root, phase, &fake_bin, NotifyEnv::Set("")); assert_eq!(persisted_cmd(root, phase), None); } #[test] fn advance_gate_runs_the_persisted_command_when_the_variable_is_unset() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(95); @@ -297,7 +286,7 @@ fn advance_gate_runs_the_persisted_command_when_the_variable_is_unset() { let sentinel = sentinel_dir.path().join("fired"); let cmd = format!("touch {}", sentinel.display()); - let _advance = spawn_parked_advance(root, phase, Some(&cmd), NotifyEnv::Unset); + let _advance = spawn_parked_advance(&home, root, phase, Some(&cmd), NotifyEnv::Unset); wait_for_notify_fired(root); assert!(sentinel.exists(), "the persisted notify command must run"); @@ -310,6 +299,7 @@ fn advance_gate_runs_the_persisted_command_when_the_variable_is_unset() { #[test] fn advance_gate_prefers_a_set_variable_over_the_persisted_command() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(95); @@ -321,6 +311,7 @@ fn advance_gate_prefers_a_set_variable_over_the_persisted_command() { let override_cmd = format!("touch {}", overriding.display()); let _advance = spawn_parked_advance( + &home, root, phase, Some(&persisted_hook), @@ -342,6 +333,7 @@ fn advance_gate_prefers_a_set_variable_over_the_persisted_command() { #[test] fn advance_gate_with_an_empty_variable_skips_the_persisted_command() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(95); @@ -350,7 +342,7 @@ fn advance_gate_with_an_empty_variable_skips_the_persisted_command() { let sentinel = sentinel_dir.path().join("fired"); let cmd = format!("touch {}", sentinel.display()); - let _advance = spawn_parked_advance(root, phase, Some(&cmd), NotifyEnv::Set("")); + let _advance = spawn_parked_advance(&home, root, phase, Some(&cmd), NotifyEnv::Set("")); wait_for_notify_fired(root); assert!( @@ -368,6 +360,7 @@ fn advance_gate_with_an_empty_variable_skips_the_persisted_command() { /// persisted command, so a command exported (or emptied) after `start` /// reaches the chain it launches. fn resume_with(notify: NotifyEnv<'_>) -> Option { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(97); @@ -379,7 +372,7 @@ fn resume_with(notify: NotifyEnv<'_>) -> Option { let _reaper = MarkedReaper(root.to_string_lossy().into_owned()); let existing = std::env::var_os("PATH").unwrap_or_default(); let path = format!("{}:{}", fake_bin.path.display(), existing.to_string_lossy()); - let mut command = Command::new(devflow_bin()); + let mut command = common::hermetic_devflow(&home); command .args([ "resume", @@ -393,8 +386,8 @@ fn resume_with(notify: NotifyEnv<'_>) -> Option { .env("DEVFLOW_GATE_TIMEOUT_SECS", "30") .env(TEST_MARKER_ENV, root); apply_notify_env(&mut command, notify); - let cache = isolated_cache(&mut command); - let mut resume = ReapOnDrop(command.spawn().expect("spawn resume"), cache); + let mut resume = ParkedChild(command.spawn().expect("spawn resume")); + let _run_reaper = common::LaunchedRunReaper::new(root, phase, Vec::new()); let waited = Instant::now(); loop { let saved = persisted_cmd(root, phase); @@ -467,8 +460,13 @@ fn marked_processes(marker: &str) -> Vec<(u32, String, Vec)> { /// Approve the parked Define gate, wait for the monitor chain to launch the /// fake agent, and return the agent's environment from `/proc`. Every marked /// process is killed before returning. -fn agent_environment_after_define(root: &Path, phase: PhaseId, fake_bin: &FakeBin) -> Vec { - let approve = Command::new(devflow_bin()) +fn agent_environment_after_define( + home: &common::HermeticHome, + root: &Path, + phase: PhaseId, + fake_bin: &FakeBin, +) -> Vec { + let approve = common::hermetic_devflow(home) .args(["gate", "approve", &phase.to_string(), "define"]) .arg(root) .output() @@ -495,7 +493,7 @@ fn agent_environment_after_define(root: &Path, phase: PhaseId, fake_bin: &FakeBi ); std::thread::sleep(Duration::from_millis(20)); }; - let _ = Command::new(devflow_bin()) + let _ = common::hermetic_devflow(home) .args(["stop", "--phase", &phase.to_string(), "--root"]) .arg(root) .output(); @@ -503,20 +501,23 @@ fn agent_environment_after_define(root: &Path, phase: PhaseId, fake_bin: &FakeBi } fn assert_chain_has_no_notify_variable(legacy_launch: bool) { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(96); init_start_repo(root, phase); let fake_bin = fake_bin_dir(); let _start = spawn_parked_start_with( + &home, root, phase, &fake_bin, NotifyEnv::Set("touch /nonexistent/notify-sentinel"), legacy_launch, ); + let _run_reaper = common::LaunchedRunReaper::new(root, phase, Vec::new()); - let environment = agent_environment_after_define(root, phase, &fake_bin); + let environment = agent_environment_after_define(&home, root, phase, &fake_bin); // Control: a variable set on `start` the same way does reach the agent, // so an absent entry below means stripped, not unread. diff --git a/crates/devflow-cli/tests/gate_wedge_e2e.rs b/crates/devflow-cli/tests/gate_wedge_e2e.rs index a2227284..d5b85169 100644 --- a/crates/devflow-cli/tests/gate_wedge_e2e.rs +++ b/crates/devflow-cli/tests/gate_wedge_e2e.rs @@ -7,6 +7,8 @@ //! - the `advance_*` arms cover a foreground `devflow advance` parked at a //! Code gate. +mod common; + use devflow_core::gates::Gates; use devflow_core::mode::Mode; use devflow_core::phase_id::PhaseId; @@ -18,10 +20,6 @@ use std::path::{Path, PathBuf}; use std::process::{Child, Command}; use std::time::{Duration, Instant}; -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} - fn git(root: &Path, args: &[&str]) { let output = devflow_core::test_support::git_command(root) .args(args) @@ -62,11 +60,11 @@ fn wait_for(mut predicate: impl FnMut() -> bool, what: &str) { } } -fn spawn_gated_advance(root: &Path, phase: PhaseId) -> Child { +fn spawn_gated_advance(home: &common::HermeticHome, root: &Path, phase: PhaseId) -> Child { let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); state.stage = Stage::Code; devflow_core::workflow::save_state(&state).unwrap(); - let child = Command::new(devflow_bin()) + let child = common::hermetic_devflow(home) .args(["advance", "--phase", &phase.to_string()]) .arg(root) .env("DEVFLOW_GATE_TIMEOUT_SECS", "15") @@ -83,8 +81,8 @@ fn spawn_gated_advance(root: &Path, phase: PhaseId) -> Child { child } -fn reject(root: &Path, phase: PhaseId) -> std::process::Output { - Command::new(devflow_bin()) +fn reject(home: &common::HermeticHome, root: &Path, phase: PhaseId) -> std::process::Output { + common::hermetic_devflow(home) .args([ "gate", "reject", @@ -142,10 +140,15 @@ fn fake_bin_dir(agent_body: &str) -> FakeBin { /// A foreground `devflow start --mode auto` with the fake agent first on the /// child's PATH. PATH is set on the child only (TEST-01). -fn start_child(root: &Path, phase: PhaseId, fake_bin: &FakeBin) -> Command { +fn start_child( + home: &common::HermeticHome, + root: &Path, + phase: PhaseId, + fake_bin: &FakeBin, +) -> Command { let existing = std::env::var_os("PATH").unwrap_or_default(); let path = format!("{}:{}", fake_bin.path.display(), existing.to_string_lossy()); - let mut command = Command::new(devflow_bin()); + let mut command = common::hermetic_devflow(home); command .args([ "start", @@ -180,13 +183,14 @@ impl Drop for ReapOnDrop { /// Spawns `start` (owned by `ReapOnDrop` from the moment it exists) and waits /// up to 20 s for it to park at the Define preflight gate. fn spawn_parked_start( + home: &common::HermeticHome, root: &Path, phase: PhaseId, fake_bin: &FakeBin, gate_timeout_secs: &str, ) -> ReapOnDrop { let mut start = ReapOnDrop( - start_child(root, phase, fake_bin) + start_child(home, root, phase, fake_bin) .env("DEVFLOW_GATE_TIMEOUT_SECS", gate_timeout_secs) .spawn() .expect("spawn foreground start"), @@ -205,8 +209,8 @@ fn spawn_parked_start( start } -fn reject_define(root: &Path, phase: PhaseId) -> std::process::Output { - Command::new(devflow_bin()) +fn reject_define(home: &common::HermeticHome, root: &Path, phase: PhaseId) -> std::process::Output { + common::hermetic_devflow(home) .args([ "gate", "reject", @@ -245,15 +249,16 @@ fn phase_gate_entries(root: &Path, phase: PhaseId) -> Vec { #[test] fn advance_wedge_arm_killed_advance_leaves_a_code_gate_that_reject_reports_honestly() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(4816); init_repo(root, phase); - let mut child = spawn_gated_advance(root, phase); + let mut child = spawn_gated_advance(&home, root, phase); child.kill().expect("kill foreground advance"); child.wait().expect("reap foreground advance"); - let output = reject(root, phase); + let output = reject(&home, root, phase); assert!( !output.status.success(), "wedge reject unexpectedly succeeded: {}", @@ -273,7 +278,7 @@ fn advance_wedge_arm_killed_advance_leaves_a_code_gate_that_reject_reports_hones "unexpected output: {output_text}" ); - let recovery = Command::new(devflow_bin()) + let recovery = common::hermetic_devflow(&home) .args(["recover", "--clean", "--phase", &phase.to_string()]) .arg(root) .output() @@ -323,13 +328,14 @@ fn advance_wedge_arm_killed_advance_leaves_a_code_gate_that_reject_reports_hones #[test] fn advance_self_resolving_arm_live_advance_consumes_the_code_rejection() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(4817); init_repo(root, phase); - let mut child = spawn_gated_advance(root, phase); + let mut child = spawn_gated_advance(&home, root, phase); let before = Instant::now(); - let output = reject(root, phase); + let output = reject(&home, root, phase); assert!( output.status.success(), "live reject failed: {}", @@ -359,6 +365,7 @@ fn advance_self_resolving_arm_live_advance_consumes_the_code_rejection() { #[test] fn dry_run_sweep_reports_no_waiter_gates_as_left_alone() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(4818); @@ -375,7 +382,7 @@ fn dry_run_sweep_reports_no_waiter_gates_as_left_alone() { std::fs::write(lock_path, format!("{pid}\n{start}")).unwrap(); Gates::write_gate(root, live_ship_phase, Stage::Ship, "live poller").unwrap(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args([ "gate", "sweep", @@ -425,12 +432,13 @@ fn dry_run_sweep_reports_no_waiter_gates_as_left_alone() { #[test] fn start_wedge_arm_interrupted_start_leaves_a_define_gate_nothing_answers() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(93); init_start_repo(root, phase); let fake_bin = fake_bin_dir("sleep 60"); - let mut start = spawn_parked_start(root, phase, &fake_bin, "30"); + let mut start = spawn_parked_start(&home, root, phase, &fake_bin, "30"); assert_eq!( devflow_core::lock::holder_identity(root, phase).map(|(pid, _)| pid), @@ -445,7 +453,7 @@ fn start_wedge_arm_interrupted_start_leaves_a_define_gate_nothing_answers() { "the interrupted start must leave its Define gate behind" ); - let output = reject_define(root, phase); + let output = reject_define(&home, root, phase); let output_text = format!( "{}{}", String::from_utf8_lossy(&output.stdout), @@ -502,7 +510,7 @@ fn start_wedge_arm_interrupted_start_leaves_a_define_gate_nothing_answers() { ); } - let recovery = Command::new(devflow_bin()) + let recovery = common::hermetic_devflow(&home) .args(["recover", "--clean", "--phase", &phase.to_string()]) .arg(root) .output() @@ -539,12 +547,13 @@ fn start_wedge_arm_interrupted_start_leaves_a_define_gate_nothing_answers() { #[test] fn start_self_resolving_arm_live_start_consumes_the_define_rejection() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(94); init_start_repo(root, phase); let fake_bin = fake_bin_dir("sleep 60"); - let mut start = spawn_parked_start(root, phase, &fake_bin, "60"); + let mut start = spawn_parked_start(&home, root, phase, &fake_bin, "60"); assert_eq!( devflow_core::lock::holder_identity(root, phase).map(|(pid, _)| pid), @@ -553,7 +562,7 @@ fn start_self_resolving_arm_live_start_consumes_the_define_rejection() { ); let before = Instant::now(); - let output = reject_define(root, phase); + let output = reject_define(&home, root, phase); let stdout = String::from_utf8_lossy(&output.stdout); assert!( output.status.success(), diff --git a/crates/devflow-cli/tests/phase7_cli.rs b/crates/devflow-cli/tests/phase7_cli.rs index 6603ee6c..f095382c 100644 --- a/crates/devflow-cli/tests/phase7_cli.rs +++ b/crates/devflow-cli/tests/phase7_cli.rs @@ -1,3 +1,5 @@ +mod common; + use devflow_core::phase_id::PhaseId; use devflow_core::stage::Stage; use devflow_core::workflow::load_state; @@ -12,10 +14,6 @@ struct FakeBin { path: PathBuf, } -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} - fn git(root: &Path, args: &[&str]) -> Output { // Hermetic: pinning cwd alone does not stop an inherited GIT_DIR from // retargeting the real repository (999.37). @@ -93,8 +91,8 @@ fn path_with_fake_bin(fake_bin: &Path) -> String { format!("{}:{}", fake_bin.display(), existing.to_string_lossy()) } -fn run_devflow(root: &Path, fake_bin: &Path, args: &[&str]) -> Output { - run_devflow_inner(root, fake_bin, args, false) +fn run_devflow(home: &common::HermeticHome, root: &Path, fake_bin: &Path, args: &[&str]) -> Output { + run_devflow_inner(home, root, fake_bin, args, false) } /// [`run_devflow`] with D-11's legacy-launch opt-out forced on for the spawned @@ -123,12 +121,23 @@ fn run_devflow(root: &Path, fake_bin: &Path, args: &[&str]) -> Output { /// coverage rather than stabilising it; see /// `parallel_creates_two_worktrees_and_spawns_two_monitors`, which deliberately /// does not use this helper. -fn run_devflow_legacy_launch(root: &Path, fake_bin: &Path, args: &[&str]) -> Output { - run_devflow_inner(root, fake_bin, args, true) +fn run_devflow_legacy_launch( + home: &common::HermeticHome, + root: &Path, + fake_bin: &Path, + args: &[&str], +) -> Output { + run_devflow_inner(home, root, fake_bin, args, true) } -fn run_devflow_inner(root: &Path, fake_bin: &Path, args: &[&str], legacy_launch: bool) -> Output { - let mut command = Command::new(devflow_bin()); +fn run_devflow_inner( + home: &common::HermeticHome, + root: &Path, + fake_bin: &Path, + args: &[&str], + legacy_launch: bool, +) -> Output { + let mut command = common::hermetic_devflow(home); command .args(args) .arg(root) @@ -266,6 +275,7 @@ fn seed_feature_branch(root: &Path, phase: PhaseId) { #[test] fn parallel_creates_two_worktrees_and_spawns_two_monitors() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -324,6 +334,7 @@ esac // test's subject is two worktrees and two monitors, which is // mode-independent; the agent diversity is the point and is preserved. let output = run_devflow( + &home, root, &fake_bin.path, &[ @@ -336,6 +347,8 @@ esac "supervise", ], ); + let _reaper7 = common::LaunchedRunReaper::new(root, PhaseId::new(7), Vec::new()); + let _reaper8 = common::LaunchedRunReaper::new(root, PhaseId::new(8), Vec::new()); let stdout = String::from_utf8_lossy(&output.stdout); assert!(stdout.contains("phase 7")); assert!(stdout.contains("phase 8")); @@ -382,6 +395,7 @@ esac #[test] fn start_defaults_to_worktree() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -397,6 +411,7 @@ fn start_defaults_to_worktree() { // puts the phase, not which transport its agent is launched over. See // `run_devflow_legacy_launch`. run_devflow_legacy_launch( + &home, root, &fake_bin.path, &[ @@ -412,6 +427,7 @@ fn start_defaults_to_worktree() { "supervise", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(11), Vec::new()); // start returns before the detached monitor finishes; wait for the // worktree directory like the other integration tests do. @@ -441,6 +457,7 @@ fn start_defaults_to_worktree() { /// phase's worktree, which always starts at ahead=0, behind=0. #[test] fn start_worktree_mode_ignores_main_checkout_divergence() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -473,6 +490,7 @@ fn start_worktree_mode_ignores_main_checkout_divergence() { // divergence check's scope, not the launch transport. See // `run_devflow_legacy_launch`. run_devflow_legacy_launch( + &home, root, &fake_bin.path, &[ @@ -488,6 +506,7 @@ fn start_worktree_mode_ignores_main_checkout_divergence() { "supervise", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(13), Vec::new()); wait_for(&root.join(".worktrees/phase-13")); assert!(root.join(".worktrees/phase-13").is_dir()); @@ -500,6 +519,7 @@ fn start_worktree_mode_ignores_main_checkout_divergence() { #[test] fn start_no_worktree_uses_feature_branch() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -512,6 +532,7 @@ fn start_no_worktree_uses_feature_branch() { // `--no-worktree` keeps the phase on its feature branch, not which // transport its agent is launched over. See `run_devflow_legacy_launch`. run_devflow_legacy_launch( + &home, root, &fake_bin.path, &[ @@ -528,6 +549,7 @@ fn start_no_worktree_uses_feature_branch() { "--no-worktree", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(12), Vec::new()); // start returns before the detached monitor finishes; wait for the // agent pid file that the monitor writes on the feature-branch path. @@ -555,6 +577,7 @@ fn start_no_worktree_uses_feature_branch() { /// case this plan adds. #[test] fn start_until_plan_halts_cleanly() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -569,6 +592,7 @@ fn start_until_plan_halts_cleanly() { // into `state.legacy_claude_launch` by `start`, so the detached monitor // that launches Plan reads it back. See `run_devflow_legacy_launch`. run_devflow_legacy_launch( + &home, root, &fake_bin.path, &[ @@ -586,6 +610,7 @@ fn start_until_plan_halts_cleanly() { "plan", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(44), Vec::new()); let state = wait_for_stopped(root, PhaseId::new(44)); assert_eq!( @@ -615,6 +640,7 @@ fn start_until_plan_halts_cleanly() { /// any stage runs, not silently accepted as if it intercepted anything. #[test] fn start_until_ship_is_rejected() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -623,7 +649,7 @@ fn start_until_ship_is_rejected() { "#!/bin/sh\nprintf 'DEVFLOW_RESULT: {\"status\":\"success\"}\\n'\n", )]); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args([ "start", "--phase", "45", "--agent", "claude", "--mode", "auto", "--until", "ship", ]) @@ -653,6 +679,7 @@ fn start_until_ship_is_rejected() { /// `start` is ever dispatched. #[test] fn start_until_unknown_stage_is_rejected_by_clap() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -661,7 +688,7 @@ fn start_until_unknown_stage_is_rejected_by_clap() { "#!/bin/sh\nprintf 'DEVFLOW_RESULT: {\"status\":\"success\"}\\n'\n", )]); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args([ "start", "--phase", "46", "--agent", "claude", "--mode", "auto", "--until", "bogus", ]) @@ -688,12 +715,13 @@ fn start_until_unknown_stage_is_rejected_by_clap() { /// `--until` had not been passed. #[test] fn start_dry_run_annotates_until_stage() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); let fake_bin = fake_bin_dir(&[]); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args([ "start", "--phase", @@ -738,12 +766,13 @@ fn start_dry_run_annotates_until_stage() { /// annotation — the full pipeline runs to Ship. #[test] fn start_dry_run_without_until_has_no_stop_annotation() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); let fake_bin = fake_bin_dir(&[]); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args([ "start", "--phase", @@ -770,6 +799,7 @@ fn start_dry_run_without_until_has_no_stop_annotation() { #[test] fn status_prints_cron_hint_when_cron_instructions_exist() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -781,7 +811,7 @@ fn status_prints_cron_hint_when_cron_instructions_exist() { devflow_core::ship::write_cron_instructions(root, &instructions).unwrap(); let fake_bin = fake_bin_dir(&[]); - let output = run_devflow(root, &fake_bin.path, &["status"]); + let output = run_devflow(&home, root, &fake_bin.path, &["status"]); let stdout = String::from_utf8_lossy(&output.stdout); // D-10 (44-03) replaced the old `--from-devflow ` invocation with a @@ -804,6 +834,7 @@ fn status_prints_cron_hint_when_cron_instructions_exist() { #[test] fn reference_and_cleanup_worktree_cli_flow() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -835,13 +866,14 @@ esac )]); // reference — creates static snapshot - let out = run_devflow(root, &fake_bin.path, &["reference"]); + let out = run_devflow(&home, root, &fake_bin.path, &["reference"]); let stdout = String::from_utf8_lossy(&out.stdout); assert!(stdout.contains("reference worktree")); assert!(root.join(".worktrees/reference").is_dir()); // start --worktree — creates phase worktree let out = run_devflow( + &home, root, &fake_bin.path, &[ @@ -855,6 +887,7 @@ esac "--worktree", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(8), Vec::new()); let stdout = String::from_utf8_lossy(&out.stdout); assert!(stdout.contains("phase 8")); assert!( @@ -863,7 +896,7 @@ esac ); // status — lists active worktrees - let out = run_devflow(root, &fake_bin.path, &["status"]); + let out = run_devflow(&home, root, &fake_bin.path, &["status"]); let stdout = String::from_utf8_lossy(&out.stdout); assert!( stdout.contains(".worktrees/reference"), @@ -902,6 +935,7 @@ esac let _reap = MonitorReapGuard::after_launch(&settled); run_devflow( + &home, root, &fake_bin.path, &[ @@ -917,7 +951,7 @@ esac wait_for_state_cleared(root, PhaseId::new(8)); // cleanup — removes worktrees - let out = run_devflow(root, &fake_bin.path, &["cleanup", "--force"]); + let out = run_devflow(&home, root, &fake_bin.path, &["cleanup", "--force"]); let stdout = String::from_utf8_lossy(&out.stdout); assert!(stdout.contains("removed")); @@ -932,6 +966,7 @@ esac /// monitor is created. #[test] fn start_codex_without_context_fails_preflight() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -939,7 +974,7 @@ fn start_codex_without_context_fails_preflight() { // (14-CR-05) passes and the CONTEXT.md artifact check must fire next. let fake_bin = fake_bin_dir(&[("codex", "#!/bin/sh\nexit 0\n")]); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args([ "start", "--phase", "42", "--agent", "codex", "--mode", "auto", ]) @@ -978,6 +1013,7 @@ fn start_codex_without_context_fails_preflight() { /// worktree unconditionally and exits 0. #[test] fn cleanup_force_refuses_on_live_agent_unknown_monitor() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -1009,7 +1045,7 @@ fn cleanup_force_refuses_on_live_agent_unknown_monitor() { devflow_core::workflow::save_state(&state).unwrap(); let fake_bin = fake_bin_dir(&[]); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["cleanup", "--force"]) .arg(root) .env("PATH", path_with_fake_bin(&fake_bin.path)) @@ -1044,6 +1080,7 @@ fn cleanup_force_refuses_on_live_agent_unknown_monitor() { /// classification alone. #[test] fn cleanup_force_refuses_on_dead_monitor_live_agent() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -1074,7 +1111,7 @@ fn cleanup_force_refuses_on_dead_monitor_live_agent() { devflow_core::workflow::save_state(&state).unwrap(); let fake_bin = fake_bin_dir(&[]); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["cleanup", "--force"]) .arg(root) .env("PATH", path_with_fake_bin(&fake_bin.path)) @@ -1112,6 +1149,7 @@ fn cleanup_force_refuses_on_dead_monitor_live_agent() { /// same phase; `cleanup` must recognize `state.stopped` too. #[test] fn cleanup_keeps_worktree_for_until_stopped_phase_without_force() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -1140,7 +1178,7 @@ fn cleanup_keeps_worktree_for_until_stopped_phase_without_force() { devflow_core::workflow::save_state(&state).unwrap(); let fake_bin = fake_bin_dir(&[]); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["cleanup"]) .arg(root) .env("PATH", path_with_fake_bin(&fake_bin.path)) @@ -1173,6 +1211,7 @@ fn cleanup_keeps_worktree_for_until_stopped_phase_without_force() { /// still be able to discard a stopped phase's worktree. #[test] fn cleanup_force_removes_worktree_for_until_stopped_phase() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -1197,7 +1236,7 @@ fn cleanup_force_removes_worktree_for_until_stopped_phase() { devflow_core::workflow::save_state(&state).unwrap(); let fake_bin = fake_bin_dir(&[]); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["cleanup", "--force"]) .arg(root) .env("PATH", path_with_fake_bin(&fake_bin.path)) @@ -1220,6 +1259,7 @@ fn cleanup_force_removes_worktree_for_until_stopped_phase() { /// the second run finds the worktree already gone and does not error. #[test] fn cleanup_is_idempotent_when_worktree_already_removed() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -1245,7 +1285,7 @@ fn cleanup_is_idempotent_when_worktree_already_removed() { devflow_core::workflow::save_state(&state).unwrap(); let fake_bin = fake_bin_dir(&[]); - let first = Command::new(devflow_bin()) + let first = common::hermetic_devflow(&home) .args(["cleanup", "--force"]) .arg(root) .env("PATH", path_with_fake_bin(&fake_bin.path)) @@ -1259,7 +1299,7 @@ fn cleanup_is_idempotent_when_worktree_already_removed() { ); assert!(!wt_path.is_dir(), "worktree must be removed on first run"); - let second = Command::new(devflow_bin()) + let second = common::hermetic_devflow(&home) .args(["cleanup", "--force"]) .arg(root) .env("PATH", path_with_fake_bin(&fake_bin.path)) @@ -1330,12 +1370,14 @@ fn wait_for_gate(root: &Path, phase: PhaseId) -> devflow_core::state::State { /// `Failed` and gates instead of advancing to Code. #[test] fn pi_marker_less_run_does_not_advance() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); let fake_bin = fake_bin_dir(&[("pi", &pi_stub("printf 'fake pi, no marker\\n'\nexit 0\n"))]); run_devflow( + &home, root, &fake_bin.path, &[ @@ -1348,6 +1390,7 @@ fn pi_marker_less_run_does_not_advance() { "supervise", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(7), Vec::new()); let state = wait_for_gate(root, PhaseId::new(7)); assert_eq!( @@ -1363,12 +1406,14 @@ fn pi_marker_less_run_does_not_advance() { /// not Success, so the pipeline gates instead of advancing. #[test] fn pi_nonzero_exit_does_not_advance() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); let fake_bin = fake_bin_dir(&[("pi", &pi_stub("printf 'fake pi, no marker\\n'\nexit 1\n"))]); run_devflow( + &home, root, &fake_bin.path, &[ @@ -1381,6 +1426,7 @@ fn pi_nonzero_exit_does_not_advance() { "supervise", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(7), Vec::new()); let state = wait_for_gate(root, PhaseId::new(7)); assert_eq!( @@ -1397,12 +1443,14 @@ fn pi_nonzero_exit_does_not_advance() { /// the monitor reaps it and gates (never a silent advance). #[test] fn pi_hung_process_is_detected_not_left_running() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); let fake_bin = fake_bin_dir(&[("pi", &pi_stub("exec sleep 30\n"))]); run_devflow( + &home, root, &fake_bin.path, &[ @@ -1415,6 +1463,7 @@ fn pi_hung_process_is_detected_not_left_running() { "supervise", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(7), Vec::new()); let pid_path = root.join(".devflow/phase-07-agent-pid"); let pid = wait_for_pid(&pid_path); @@ -1589,12 +1638,14 @@ impl Drop for MonitorReapGuard { /// no commits, so it gates instead of advancing to Code. #[test] fn marker_less_antigravity_never_advances() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); let fake_bin = fake_bin_dir(&[("agy", &antigravity_stub(StubMode::Quiet))]); run_devflow( + &home, root, &fake_bin.path, &[ @@ -1607,6 +1658,7 @@ fn marker_less_antigravity_never_advances() { "supervise", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(7), Vec::new()); let state = wait_for_gate(root, PhaseId::new(7)); let _reap = MonitorReapGuard::after_launch(&state); @@ -1625,12 +1677,14 @@ fn marker_less_antigravity_never_advances() { /// stopped, NOT gated. #[test] fn antigravity_parses_devflow_result_from_stream() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); let fake_bin = fake_bin_dir(&[("agy", &antigravity_stub(StubMode::MarkerStream))]); run_devflow( + &home, root, &fake_bin.path, &[ @@ -1645,6 +1699,7 @@ fn antigravity_parses_devflow_result_from_stream() { "plan", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(7), Vec::new()); let state = wait_for_stopped(root, PhaseId::new(7)); let _reap = MonitorReapGuard::after_launch(&state); @@ -1664,12 +1719,14 @@ fn antigravity_parses_devflow_result_from_stream() { /// advances a commit-gated stage, only the marker does (ANTG-03). #[test] fn antigravity_init_without_marker_gates_at_plan() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); let fake_bin = fake_bin_dir(&[("agy", &antigravity_stub(StubMode::InitOnly))]); run_devflow( + &home, root, &fake_bin.path, &[ @@ -1682,6 +1739,7 @@ fn antigravity_init_without_marker_gates_at_plan() { "supervise", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(7), Vec::new()); let state = wait_for_gate(root, PhaseId::new(7)); let _reap = MonitorReapGuard::after_launch(&state); @@ -1755,6 +1813,7 @@ fn suite_reap_audit() { /// normal guard. #[test] fn unguarded_monitor_is_detected_by_the_registry() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -1763,6 +1822,7 @@ fn unguarded_monitor_is_detected_by_the_registry() { let fake_bin = fake_bin_dir(&[("pi", &pi_stub("exec sleep 30\n"))]); run_devflow( + &home, root, &fake_bin.path, &[ @@ -1775,6 +1835,7 @@ fn unguarded_monitor_is_detected_by_the_registry() { "supervise", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(7), Vec::new()); let state = load_state(root, PhaseId::new(7)).expect("load state"); let pid = state @@ -1822,6 +1883,7 @@ fi /// commit-gated stage. Define advances on exit 0; Plan gates. #[test] fn hermes_marker_less_run_does_not_advance() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -1831,6 +1893,7 @@ fn hermes_marker_less_run_does_not_advance() { )]); run_devflow( + &home, root, &fake_bin.path, &[ @@ -1843,6 +1906,7 @@ fn hermes_marker_less_run_does_not_advance() { "supervise", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(7), Vec::new()); let state = wait_for_gate(root, PhaseId::new(7)); assert_eq!( @@ -1857,6 +1921,7 @@ fn hermes_marker_less_run_does_not_advance() { /// HRMS-03: a Hermes run that exits non-zero must not advance its stage. #[test] fn hermes_nonzero_exit_does_not_advance() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); @@ -1866,6 +1931,7 @@ fn hermes_nonzero_exit_does_not_advance() { )]); run_devflow( + &home, root, &fake_bin.path, &[ @@ -1878,6 +1944,7 @@ fn hermes_nonzero_exit_does_not_advance() { "supervise", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(7), Vec::new()); let state = wait_for_gate(root, PhaseId::new(7)); assert_eq!( @@ -1893,12 +1960,14 @@ fn hermes_nonzero_exit_does_not_advance() { /// does not falsely advance, and gates when killed. #[test] fn hermes_hung_process_is_detected_not_left_running() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo(root); let fake_bin = fake_bin_dir(&[("hermes", &hermes_stub("exec sleep 30\n"))]); run_devflow( + &home, root, &fake_bin.path, &[ @@ -1911,6 +1980,7 @@ fn hermes_hung_process_is_detected_not_left_running() { "supervise", ], ); + let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(7), Vec::new()); let pid_path = root.join(".devflow/phase-07-agent-pid"); let pid = wait_for_pid(&pid_path); From 3213f534b5143e43e5595cdff1e256935d8c15b3 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 07:52:30 -0400 Subject: [PATCH 005/112] test(50-03): migrate five more e2e suites and pin dry-run stray pass - gate_sweep_e2e, start_reachability_e2e, log_format_env, doctor_antigravity and auto_chain_leak_repair_e2e declare `mod common;` and spawn devflow only through common::hermetic_devflow - auto_chain_leak_repair_e2e: Fixture owns the HermeticHome (dropped after its reaper); LaunchedRunReaper bound after the __monitor spawn - gate_sweep_e2e: no_suite_on_the_common_builder_runs_a_real_reap_strays_pass source scan with planted controls (T-50-05) - no assertion, env var or argument list changed (counts 4+1/2/3/2/2) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- .../tests/auto_chain_leak_repair_e2e.rs | 15 +- .../devflow-cli/tests/doctor_antigravity.rs | 13 +- crates/devflow-cli/tests/gate_sweep_e2e.rs | 237 +++++++++++++++++- crates/devflow-cli/tests/log_format_env.rs | 30 ++- .../tests/start_reachability_e2e.rs | 14 +- 5 files changed, 270 insertions(+), 39 deletions(-) diff --git a/crates/devflow-cli/tests/auto_chain_leak_repair_e2e.rs b/crates/devflow-cli/tests/auto_chain_leak_repair_e2e.rs index 9cc5ecba..24765ad4 100644 --- a/crates/devflow-cli/tests/auto_chain_leak_repair_e2e.rs +++ b/crates/devflow-cli/tests/auto_chain_leak_repair_e2e.rs @@ -20,6 +20,8 @@ //! scaffolding below is copied from `auto_chain_flag_e2e.rs` rather than //! shared. Keep the two in step. +mod common; + use devflow_core::mode::Mode; use devflow_core::phase_id::PhaseId; use devflow_core::stage::Stage; @@ -32,10 +34,6 @@ use std::path::{Path, PathBuf}; use std::process::{Child, Command, Output}; use std::time::{Duration, Instant}; -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} - /// Hermetic git invocation pinned to `root` (999.37) — never a bare /// `Command::new("git")`. fn git(root: &Path, args: &[&str]) { @@ -205,6 +203,9 @@ struct Fixture { script: PathBuf, child_pid_file: PathBuf, reaper: Reaper, + /// Every `devflow` child's `HOME` and cache. Declared last so it drops + /// after `reaper` has reaped the monitor. + home: common::HermeticHome, } impl Fixture { @@ -245,6 +246,7 @@ impl Fixture { script, child_pid_file, reaper: Reaper::default(), + home: common::HermeticHome::new(), } } @@ -252,7 +254,7 @@ impl Fixture { /// sleeper, as a child of this test process so it can be signalled /// directly. fn spawn_monitor(&self) -> Child { - let child = Command::new(devflow_bin()) + let child = common::hermetic_devflow(&self.home) .arg("__monitor") .arg("--project") .arg(&self.root) @@ -298,7 +300,7 @@ impl Fixture { /// process outlives the test binary" criterion forbids. See the SUMMARY for /// what this does NOT establish. fn run_resume(&self, git_only_path: &Path) -> Output { - Command::new(devflow_bin()) + common::hermetic_devflow(&self.home) .arg("resume") .arg("--phase") .arg(self.phase.to_string()) @@ -379,6 +381,7 @@ fn auto_chain_flag_survives_sigkill_and_is_repaired_on_the_next_start() { let path_dir = git_only_path(); let mut monitor = fixture.spawn_monitor(); + let _run_reaper = common::LaunchedRunReaper::new(&fixture.root, fixture.phase, Vec::new()); // The fixture must actually engage the guard before anything downstream // means anything. A timeout here is "the fixture never set the flag", which diff --git a/crates/devflow-cli/tests/doctor_antigravity.rs b/crates/devflow-cli/tests/doctor_antigravity.rs index a01cf198..22c821eb 100644 --- a/crates/devflow-cli/tests/doctor_antigravity.rs +++ b/crates/devflow-cli/tests/doctor_antigravity.rs @@ -7,13 +7,10 @@ //! `devflow` binary, and PATH is scoped to each child via `Command::env` so //! no test-process-global environment mutation can poison parallel tests. +mod common; + use std::os::unix::fs::PermissionsExt; use std::path::Path; -use std::process::Command; - -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} fn write_stub_agy(root: &Path) -> std::path::PathBuf { let bin = root.join("bin"); @@ -33,11 +30,12 @@ fn write_stub_agy(root: &Path) -> std::path::PathBuf { /// antigravity check present with the stub's version line. #[test] fn doctor_reports_antigravity_present_with_stub_on_path() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let bin = write_stub_agy(root); - let out = Command::new(devflow_bin()) + let out = common::hermetic_devflow(&home) .arg("doctor") .arg(root) .env("PATH", &bin) @@ -59,12 +57,13 @@ fn doctor_reports_antigravity_present_with_stub_on_path() { /// missing/warn — never a hard failure, never a false green. #[test] fn doctor_reports_antigravity_absent_without_agy() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let empty = root.join("empty-bin"); std::fs::create_dir_all(&empty).unwrap(); - let out = Command::new(devflow_bin()) + let out = common::hermetic_devflow(&home) .arg("doctor") .arg(root) .env("PATH", &empty) diff --git a/crates/devflow-cli/tests/gate_sweep_e2e.rs b/crates/devflow-cli/tests/gate_sweep_e2e.rs index 605655d1..d928ee7b 100644 --- a/crates/devflow-cli/tests/gate_sweep_e2e.rs +++ b/crates/devflow-cli/tests/gate_sweep_e2e.rs @@ -9,19 +9,16 @@ //! `release_check.rs`) — this drives the real compiled binary via //! `Command::new`, not an in-process call. +mod common; + use devflow_core::gates::{GateAction, GateFile, Gates}; use devflow_core::mode::Mode; use devflow_core::phase_id::PhaseId; use devflow_core::stage::Stage; use devflow_core::state::{AgentKind, State}; use std::path::Path; -use std::process::Command; use std::time::{Duration, Instant}; -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} - /// Hermetic `git` invocation for fixture setup (999.37) — never a bare /// `Command::new("git")`. fn git(root: &Path, args: &[&str]) { @@ -162,6 +159,7 @@ const AGED_WELL_PAST_DEFAULT_THRESHOLD_SECS: u64 = 30 * 24 * 60 * 60; /// it up and resolves to `GateAction::Abort`. #[test] fn sweep_reaps_an_aged_gate_and_a_real_poller_resolves_to_abort() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(91); @@ -184,7 +182,7 @@ fn sweep_reaps_an_aged_gate_and_a_real_poller_resolves_to_abort() { std::thread::scope(|scope| { let poller = scope.spawn(move || Gates::poll_response(root, phase, stage, 30)); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["gate", "sweep", "--root"]) .arg(root) .output() @@ -216,6 +214,7 @@ fn sweep_reaps_an_aged_gate_and_a_real_poller_resolves_to_abort() { /// change. #[test] fn sweep_leaves_a_fresh_gate_untouched() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(92); @@ -224,7 +223,7 @@ fn sweep_leaves_a_fresh_gate_untouched() { // Freshly written — `unix_now()`'s age is ~0s, well under the 6h default. Gates::write_gate(root, phase, stage, "review gaps").unwrap(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args(["gate", "sweep", "--root"]) .arg(root) .output() @@ -254,7 +253,8 @@ fn sweep_leaves_a_fresh_gate_untouched() { /// in the subcommand's own `--help`. #[test] fn sweep_help_documents_max_age_and_dry_run() { - let output = Command::new(devflow_bin()) + let home = common::HermeticHome::new(); + let output = common::hermetic_devflow(&home) .args(["gate", "sweep", "--help"]) .output() .expect("run devflow gate sweep --help"); @@ -283,6 +283,7 @@ fn sweep_help_documents_max_age_and_dry_run() { /// OS process via `Command::new`, not driven in-process on a thread. #[test] fn sweep_ends_a_real_advance_process_through_its_own_abort_path() { + let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(95); @@ -300,7 +301,7 @@ fn sweep_ends_a_real_advance_process_through_its_own_abort_path() { // block for `gate_timeout_secs()`'s multi-day production default. // Set per-`Command` via `.env(...)`, not via a process-global env // mutation (999.37) — so this cannot race any other test. - let mut child = Command::new(devflow_bin()) + let mut child = common::hermetic_devflow(&home) .args(["advance", "--phase", &phase.to_string()]) .arg(root) .env("DEVFLOW_GATE_TIMEOUT_SECS", "15") @@ -371,3 +372,221 @@ fn sweep_ends_a_real_advance_process_through_its_own_abort_path() { // forms (must be zero), not asserted at runtime — there is no API // surface here that could send one. } + +// --------------------------------------------------------------------------- +// 50-03 (999.141, T-50-05): the machine-wide stray pass stays dry-run in every +// suite on the hermetic builder. +// +// `gate sweep --reap-strays` scans the whole machine's `/proc` and reaps every +// DevFlow-shaped process the caller owns, sparing only the ones the registry +// knows about. On `common::hermetic_devflow` the registry holds nothing but +// the test's own roots, so a real (non-dry-run) pass from a test would treat +// every live run on the developer's machine as a stray. This scan refuses one. +// --------------------------------------------------------------------------- + +/// The `"--reap-strays"` argument literal, quotes included. Spelled through +/// `concat!` split inside the token so this file's own source never holds it. +const REAP_STRAYS_LITERAL: &str = concat!("\"--reap-", "strays\""); + +/// The `"--dry-run"` argument literal, quotes included, spelled the same way. +const DRY_RUN_LITERAL: &str = concat!("\"--dry-", "run\""); + +/// `src` with every `//` comment (line, doc and trailing) removed. A `//` +/// inside a string, raw string or char literal is not a comment. A local copy +/// of `test_home_isolation.rs`'s scanner: integration test binaries cannot +/// share private helpers, and `common/mod.rs` holds no scan code. +fn code_only(src: &str) -> String { + let chars: Vec = src.chars().collect(); + let mut out = String::with_capacity(src.len()); + let mut index = 0; + while index < chars.len() { + index = match chars[index] { + '/' if chars.get(index + 1) == Some(&'/') => skip_to_line_end(&chars, index), + '"' => copy_string(&chars, index, &mut out), + 'r' if raw_string_hashes(&chars, index).is_some() => { + copy_raw_string(&chars, index, &mut out) + } + '\'' => copy_char_or_lifetime(&chars, index, &mut out), + other => { + out.push(other); + index + 1 + } + }; + } + out +} + +fn skip_to_line_end(chars: &[char], mut index: usize) -> usize { + while index < chars.len() && chars[index] != '\n' { + index += 1; + } + index +} + +/// Copies a `"…"` literal starting at `start`, honouring `\` escapes. +fn copy_string(chars: &[char], start: usize, out: &mut String) -> usize { + out.push('"'); + let mut index = start + 1; + while index < chars.len() { + out.push(chars[index]); + match chars[index] { + '\\' if index + 1 < chars.len() => { + out.push(chars[index + 1]); + index += 2; + } + '"' => return index + 1, + _ => index += 1, + } + } + index +} + +fn is_ident_char(c: char) -> bool { + c.is_alphanumeric() || c == '_' +} + +/// The `#` count of a raw string (`r"`, `r#"`, `br"`…) starting at the `r` at +/// `index`, or `None` when that `r` is part of an identifier or not followed +/// by `#*"`. +fn raw_string_hashes(chars: &[char], index: usize) -> Option { + let before = index.checked_sub(1).map(|i| chars[i]); + let prefix_ok = match before { + None => true, + Some('b') => index < 2 || !is_ident_char(chars[index - 2]), + Some(c) => !is_ident_char(c), + }; + if !prefix_ok { + return None; + } + let hashes = chars[index + 1..].iter().take_while(|&&c| c == '#').count(); + (chars.get(index + 1 + hashes) == Some(&'"')).then_some(hashes) +} + +fn copy_raw_string(chars: &[char], start: usize, out: &mut String) -> usize { + let hashes = raw_string_hashes(chars, start).expect("caller checked the raw-string opener"); + let body = start + 2 + hashes; + out.extend(&chars[start..body]); + let mut index = body; + while index < chars.len() { + let closes = chars[index] == '"' + && chars[index + 1..] + .iter() + .take(hashes) + .filter(|&&c| c == '#') + .count() + == hashes; + if closes { + out.extend(&chars[index..=index + hashes]); + return index + hashes + 1; + } + out.push(chars[index]); + index += 1; + } + index +} + +/// Copies a char literal (`'x'`, `'\''`, `'\u{1F600}'`) or, for a lifetime +/// or label, only its quote. +fn copy_char_or_lifetime(chars: &[char], start: usize, out: &mut String) -> usize { + let end = if chars.get(start + 1) == Some(&'\\') { + chars + .get(start + 2..) + .unwrap_or_default() + .iter() + .position(|&c| c == '\'') + .map(|offset| start + 2 + offset) + } else if chars.get(start + 2) == Some(&'\'') { + Some(start + 2) + } else { + None + }; + match end { + Some(end) => { + out.extend(&chars[start..=end]); + end + 1 + } + None => { + out.push('\''); + start + 1 + } + } +} + +/// The `;`-separated statements of comment-stripped `code` that pass +/// `"--reap-strays"` without `"--dry-run"`, trimmed. A test that adds +/// `--dry-run` in a separate statement is flagged too: the conservative +/// direction. +fn real_reap_strays_statements(code: &str) -> Vec { + code.split(';') + .filter(|statement| { + statement.contains(REAP_STRAYS_LITERAL) && !statement.contains(DRY_RUN_LITERAL) + }) + .map(|statement| statement.trim().to_string()) + .collect() +} + +#[test] +fn no_suite_on_the_common_builder_runs_a_real_reap_strays_pass() { + // (1) Planted controls: a real pass is flagged, a dry-run pass and a + // commented-out real pass are not. + let planted = concat!( + "let out = common::hermetic_devflow(&home).args([\"gate\", \"sweep\", \"--reap-", + "strays\"]).output();" + ); + assert_eq!( + real_reap_strays_statements(&code_only(planted)).len(), + 1, + "a planted real --reap-strays pass must be flagged" + ); + let planted_dry_run = concat!( + "let out = common::hermetic_devflow(&home).args([\"gate\", \"sweep\", \"--reap-", + "strays\", \"--dry-", + "run\"]).output();" + ); + assert_eq!( + real_reap_strays_statements(&code_only(planted_dry_run)), + Vec::::new(), + "a planted --reap-strays pass with --dry-run must pass" + ); + let commented = format!("// {planted}\n"); + assert_eq!( + real_reap_strays_statements(&code_only(&commented)), + Vec::::new(), + "a commented-out real --reap-strays pass must pass" + ); + + // (2) Every suite directly under `tests/` that declares `mod common;`; + // this file always does. + let tests_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests"); + let mut paths: Vec<_> = std::fs::read_dir(&tests_dir) + .unwrap_or_else(|err| panic!("read {}: {err}", tests_dir.display())) + .map(|entry| entry.expect("read tests dir entry").path()) + .filter(|path| path.is_file() && path.extension().is_some_and(|ext| ext == "rs")) + .collect(); + paths.sort(); + let mut scanned = 0; + let mut offenders = Vec::new(); + for path in paths { + let source = std::fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + let code = code_only(&source); + if !code.lines().any(|line| line.trim() == "mod common;") { + continue; + } + scanned += 1; + for statement in real_reap_strays_statements(&code) { + offenders.push(format!("{}: {statement}", path.display())); + } + } + assert!( + scanned > 0, + "no suite under {} declares `mod common;`, so the scan checked nothing", + tests_dir.display() + ); + assert!( + offenders.is_empty(), + "a suite on the hermetic builder runs a real (non-dry-run) --reap-strays pass, \ + whose safety set would shrink to the test's own roots:\n{}", + offenders.join("\n") + ); +} diff --git a/crates/devflow-cli/tests/log_format_env.rs b/crates/devflow-cli/tests/log_format_env.rs index 8ee165bd..908ab74f 100644 --- a/crates/devflow-cli/tests/log_format_env.rs +++ b/crates/devflow-cli/tests/log_format_env.rs @@ -19,17 +19,14 @@ //! Tests below now assert log lines on stderr and assert their absence from //! stdout. +mod common; + use devflow_core::gates::Gates; use devflow_core::mode::Mode; use devflow_core::phase_id::PhaseId; use devflow_core::stage::Stage; use devflow_core::state::{AgentKind, State}; use std::path::PathBuf; -use std::process::Command; - -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} /// Set up a project dir with a legacy `.devflow/state.json` (phase 1, no /// matching per-phase file), so `devflow status` triggers the migration's @@ -57,8 +54,13 @@ fn project_with_legacy_state() -> tempfile::TempDir { dir } -fn run_status(dir: &PathBuf, log_format: Option<&str>, rust_log: Option<&str>) -> (String, String) { - let mut cmd = Command::new(devflow_bin()); +fn run_status( + home: &common::HermeticHome, + dir: &PathBuf, + log_format: Option<&str>, + rust_log: Option<&str>, +) -> (String, String) { + let mut cmd = common::hermetic_devflow(home); cmd.arg("status").arg(dir); // Start from a clean env so ambient RUST_LOG/DEVFLOW_LOG_FORMAT from the @@ -85,8 +87,14 @@ fn run_status(dir: &PathBuf, log_format: Option<&str>, rust_log: Option<&str>) - #[test] fn rust_log_debug_is_honored_under_json_log_format() { + let home = common::HermeticHome::new(); let dir = project_with_legacy_state(); - let (stdout, stderr) = run_status(&dir.path().to_path_buf(), Some("json"), Some("debug")); + let (stdout, stderr) = run_status( + &home, + &dir.path().to_path_buf(), + Some("json"), + Some("debug"), + ); assert!( stderr.contains("migrated legacy state.json"), @@ -104,12 +112,13 @@ fn rust_log_debug_is_honored_under_json_log_format() { #[test] fn rust_log_default_suppresses_debug_under_json_log_format() { + let home = common::HermeticHome::new(); let dir = project_with_legacy_state(); // No RUST_LOG set — tracing-subscriber's EnvFilter::from_default_env() // defaults to ERROR-level-only when the env var is absent, so this DEBUG // line must NOT appear. If it does, RUST_LOG is being ignored (i.e. a // fixed, too-verbose level is hardcoded again), the exact WR-01 bug class. - let (stdout, stderr) = run_status(&dir.path().to_path_buf(), Some("json"), None); + let (stdout, stderr) = run_status(&home, &dir.path().to_path_buf(), Some("json"), None); assert!( !stderr.contains("migrated legacy state.json"), @@ -145,9 +154,10 @@ fn project_with_open_gate(phase: PhaseId, stage: Stage) -> tempfile::TempDir { #[test] fn rust_log_unset_still_shows_info_level_logs_by_default() { + let home = common::HermeticHome::new(); let dir = project_with_open_gate(PhaseId::new(15), Stage::Ship); - let mut cmd = Command::new(devflow_bin()); + let mut cmd = common::hermetic_devflow(&home); cmd.arg("gate") .arg("approve") .arg("15") diff --git a/crates/devflow-cli/tests/start_reachability_e2e.rs b/crates/devflow-cli/tests/start_reachability_e2e.rs index 8c90e4ab..9ec368b8 100644 --- a/crates/devflow-cli/tests/start_reachability_e2e.rs +++ b/crates/devflow-cli/tests/start_reachability_e2e.rs @@ -10,14 +10,12 @@ //! stderr, and filesystem effects — never on `preflight.rs`/`commands.rs` //! source text (test-signal-rejection pattern 4). +mod common; + use std::fs; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; -use std::process::{Command, Output}; - -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} +use std::process::Output; /// Hermetic git invocation pinned to `root` (999.37) — never a bare /// `Command::new("git")`. @@ -115,12 +113,13 @@ fn path_with_fake_bin(fake_bin: &Path) -> String { #[test] fn start_refuses_a_phase_promoted_only_on_the_working_branch_and_scaffolds_nothing() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo_with_phase_24_promoted_only_on_feature_branch(root); let fake_bin = fake_bin_dir(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args([ "start", "--phase", "24", "--agent", "claude", "--mode", "auto", ]) @@ -185,12 +184,13 @@ fn start_refuses_a_phase_promoted_only_on_the_working_branch_and_scaffolds_nothi /// branch is created. #[test] fn start_refuses_before_creating_the_feature_branch_in_no_worktree_mode() { + let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path(); init_repo_with_phase_24_promoted_only_on_feature_branch(root); let fake_bin = fake_bin_dir(); - let output = Command::new(devflow_bin()) + let output = common::hermetic_devflow(&home) .args([ "start", "--phase", From 98bb5cc0ed38eab6cb3ac16ca452da7bf30e7d7b Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 07:58:22 -0400 Subject: [PATCH 006/112] test(50-04): add failing test for recycled-pid lock reclaim - acquire_reclaims_a_lock_whose_live_pid_has_a_different_start_time fails today: the pid-only predicate returns Contended (RED) - controls: a live identity-matched holder and a live holder with no recorded start time both stay Contended (pass before the change) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/lock.rs | 86 +++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/crates/devflow-core/src/lock.rs b/crates/devflow-core/src/lock.rs index bfaea959..01b4f0d8 100644 --- a/crates/devflow-core/src/lock.rs +++ b/crates/devflow-core/src/lock.rs @@ -915,6 +915,92 @@ mod tests { drop(guard); } + /// Write a lock record naming this test process's own (live) pid, with + /// `recorded_start` as its second line when given. Returns the pid, the + /// observed start time of this process, and the lock path. + fn write_own_pid_lock( + dir: &Path, + phase: PhaseId, + recorded_start: impl FnOnce(u64) -> Option, + ) -> (u32, u64, PathBuf) { + let path = lock_path(dir, phase); + let pid = std::process::id(); + let observed = crate::agent::process_start_time(pid) + .expect("the test process must expose its start time"); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let record = match recorded_start(observed) { + Some(start) => format!("{pid}\n{start}"), + None => pid.to_string(), + }; + fs::write(&path, record).unwrap(); + (pid, observed, path) + } + + /// 999.128 (D-08): a live pid whose start time differs from the recorded + /// one is a recycled pid, not the holder — the recorded holder exited. + /// Reclaim must treat it like a dead holder. + #[test] + fn acquire_reclaims_a_lock_whose_live_pid_has_a_different_start_time() { + let dir = tempfile::tempdir().unwrap(); + let phase = PhaseId::new(1); + let (pid, observed, path) = + write_own_pid_lock(dir.path(), phase, |start| Some(start.saturating_add(1))); + + let guard = acquire(dir.path(), phase).expect("a recycled-pid lock must be reclaimed"); + + assert_eq!( + fs::read_to_string(&path).unwrap(), + format!("{pid}\n{observed}"), + "the reclaimed lock must name the caller with its real start time" + ); + assert_eq!(holder_status(dir.path(), phase), HolderStatus::Live { pid }); + drop(guard); + } + + /// Phase 48 D-02 one-writer rule: a live holder whose start time matches + /// the recorded one is the holder, and is never reclaimed. + #[test] + fn acquire_never_reclaims_a_live_holder_whose_start_time_matches() { + let dir = tempfile::tempdir().unwrap(); + let phase = PhaseId::new(1); + let (pid, _, path) = write_own_pid_lock(dir.path(), phase, Some); + let before = fs::read(&path).unwrap(); + + let err = acquire(dir.path(), phase).expect_err("a live matched holder must contend"); + + match err { + LockError::Contended { pid: holder, .. } => assert_eq!(holder, pid.to_string()), + other => panic!("expected Contended, got {other:?}"), + } + assert_eq!( + fs::read(&path).unwrap(), + before, + "the lock must be untouched" + ); + } + + /// A live pid with no recorded start time cannot be identity-checked + /// (Unconfirmable), so reclaim stays conservative and contends. + #[test] + fn acquire_never_reclaims_a_live_holder_with_no_recorded_start_time() { + let dir = tempfile::tempdir().unwrap(); + let phase = PhaseId::new(1); + let (pid, _, path) = write_own_pid_lock(dir.path(), phase, |_| None); + let before = fs::read(&path).unwrap(); + + let err = acquire(dir.path(), phase).expect_err("an unconfirmable holder must contend"); + + match err { + LockError::Contended { pid: holder, .. } => assert_eq!(holder, pid.to_string()), + other => panic!("expected Contended, got {other:?}"), + } + assert_eq!( + fs::read(&path).unwrap(), + before, + "the lock must be untouched" + ); + } + #[test] fn acquire_treats_a_corrupt_lock_record_as_contended_without_deleting_it() { let dir = tempfile::tempdir().unwrap(); From 06db94cd072a1fd0ba8aac862ff42fcf2e58c64a Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 07:59:48 -0400 Subject: [PATCH 007/112] feat(50-04): reclaim a lock whose live pid was recycled - acquire_after_existing now classifies the holder with classify_holder_record (pid liveness + recorded start time) inside the coordination flock it already holds - NoHolder and Recycled are reclaimed; Live and Unconfirmable stay Contended (Phase 48 D-02 one-writer rule) - the reclaim warning names the observed and recorded start times when it reclaims a recycled pid (999.128, D-08) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/lock.rs | 41 ++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/crates/devflow-core/src/lock.rs b/crates/devflow-core/src/lock.rs index 01b4f0d8..fee6ab46 100644 --- a/crates/devflow-core/src/lock.rs +++ b/crates/devflow-core/src/lock.rs @@ -245,18 +245,37 @@ fn acquire_after_existing( let Some(record) = read_holder_record(&path) else { return Err(contended(path)); }; - if crate::agent::agent_running(record.pid) { - return Err(LockError::Contended { - pid: record.pid.to_string(), - path, - }); + // 999.128 (D-08): reclaim through the same holder classifier `holder_status` + // uses, so a live pid whose start time differs from the recorded one (the + // recorded holder exited and its pid was reused) counts as stale. Only an + // identity-confirmed live holder, or one whose identity cannot be checked, + // keeps the lock. + let is_running = crate::agent::agent_running(record.pid); + let observed_start_time = crate::agent::process_start_time(record.pid); + match classify_holder_record(record, is_running, observed_start_time) { + HolderStatus::Live { .. } | HolderStatus::Unconfirmable { .. } => { + return Err(LockError::Contended { + pid: record.pid.to_string(), + path, + }); + } + HolderStatus::NoHolder => tracing::warn!( + "reclaiming stale devflow lock at {} (holder pid {} is not alive)", + path.display(), + record.pid + ), + HolderStatus::Recycled { pid } => { + let show = + |start: Option| start.map_or_else(|| "unknown".into(), |s| s.to_string()); + tracing::warn!( + "reclaiming devflow lock at {}: holder pid {pid} is alive but its start time {} \ + differs from the recorded {} — reclaiming a recycled pid", + path.display(), + show(observed_start_time), + show(record.recorded_start_time) + ); + } } - - tracing::warn!( - "reclaiming stale devflow lock at {} (holder pid {} is not alive)", - path.display(), - record.pid - ); before_reclaim(); match fs::remove_file(&path) { Ok(()) => {} From 3fedc453e473c9f1ba497cbf8753ffd347fdb4e6 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 08:02:02 -0400 Subject: [PATCH 008/112] test(50-04): pin the repair at a recycled-pid lock end to end - rename clean_phase_returns_without_cleanup_for_a_recycled_pid_lock to clean_phase_reclaims_a_recycled_pid_lock_and_cleans and invert it - add clean_phase_refuses_a_live_pid_lock_and_cleans_nothing, a hand-written live record so the refusal comes from classification - recover_clean_e2e: declare mod common, spawn devflow hermetically, add explicit_clean_at_a_recycled_lock_holder_reclaims_and_cleans (R-13) - recover_cmd: correct the explicit-phase comment (comment only) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 6 +- crates/devflow-cli/tests/recover_clean_e2e.rs | 59 +++++++++++++++++-- crates/devflow-core/src/recover.rs | 44 ++++++++++++-- 3 files changed, 97 insertions(+), 12 deletions(-) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 56ace307..09e315a4 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -2736,8 +2736,10 @@ pub(crate) fn recover_cmd( ) -> Result<(), CliError> { if do_clean { match phase { - // Explicit phase: clear it regardless of staleness (14-CR-01's - // escape hatch for a wedged-but-fresh run). + // Explicit phase: cleaned when its lock holder is dead or its pid + // was recycled (a live pid whose start time differs from the one + // recorded, 999.128). A live, identity-confirmed holder still + // refuses, so a wedged live run needs `devflow stop` first. Some(phase) => { let report = match recover::clean_phase_report(project_root, phase) { Ok(report) => report, diff --git a/crates/devflow-cli/tests/recover_clean_e2e.rs b/crates/devflow-cli/tests/recover_clean_e2e.rs index 257e9b39..38c06ef1 100644 --- a/crates/devflow-cli/tests/recover_clean_e2e.rs +++ b/crates/devflow-cli/tests/recover_clean_e2e.rs @@ -3,20 +3,18 @@ //! exits non-zero, and a sweep that cleared nothing does not claim a cleanup. //! Each refusal test has a control that must still clean and say so. +mod common; + use devflow_core::mode::Mode; use devflow_core::phase_id::PhaseId; use devflow_core::state::{AgentKind, State}; use std::path::Path; -use std::process::{Command, Output}; +use std::process::Output; use std::time::{SystemTime, UNIX_EPOCH}; /// A PID that is essentially certain not to map to a live process. const DEAD_PID: u32 = 0x7FFF_FFFE; -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} - fn init_repo(root: &Path) { let output = devflow_core::test_support::git_command(root) .args(["init", "-q"]) @@ -45,7 +43,8 @@ fn save_state_aged(root: &Path, phase: PhaseId, age_secs: u64) { } fn recover_clean(root: &Path, phase: Option) -> Output { - let mut command = Command::new(devflow_bin()); + let home = common::HermeticHome::new(); + let mut command = common::hermetic_devflow(&home); command.args(["recover", "--clean"]); if let Some(phase) = phase { command.args(["--phase", &phase.to_string()]); @@ -114,6 +113,54 @@ fn explicit_clean_without_a_lock_holder_cleans_and_says_so() { assert!(!devflow_core::workflow::state_path(root, phase).exists()); } +/// 999.128 (D-08), R-13: the repair `stop`/`doctor` name at a recycled-pid +/// gate must work. A lock record naming a live pid whose start time differs +/// from the recorded one belongs to a holder that exited, so an explicit clean +/// reclaims it and cleans the phase. +/// `explicit_clean_with_a_held_phase_lock_fails_and_cleans_nothing` is the +/// live-holder control. +#[test] +fn explicit_clean_at_a_recycled_lock_holder_reclaims_and_cleans() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + init_repo(root); + let phase = PhaseId::new(65); + save_state_aged(root, phase, 60); + let gate = devflow_core::gates::Gates::write_gate( + root, + phase, + devflow_core::stage::Stage::Code, + "wedged at a recycled pid", + ) + .unwrap(); + // This test process is alive; recording a start time one tick off its + // real one makes the record name a process instance that no longer exists. + let pid = std::process::id(); + let start = devflow_core::agent::process_start_time(pid).expect("test pid start time"); + let lock_path = root + .join(".devflow") + .join(format!("lock-{}", phase.padded())); + std::fs::write(&lock_path, format!("{pid}\n{}", start.saturating_add(1))).unwrap(); + assert_eq!( + devflow_core::lock::holder_status(root, phase), + devflow_core::lock::HolderStatus::Recycled { pid } + ); + + let output = recover_clean(root, Some(phase)); + + let text = combined(&output); + assert!( + output.status.success(), + "a clean at a recycled-pid lock must succeed: {text}" + ); + assert!( + text.contains("cleaned up workflow state for phase 65"), + "the clean must say what it cleaned: {text}" + ); + assert!(!devflow_core::workflow::state_path(root, phase).exists()); + assert!(!gate.exists(), "the phase's gate file must be removed"); +} + /// Fix-review finding C-3: `clean_phase` is idempotent, so an explicit clean /// of a phase with nothing on disk succeeded and printed "cleaned up". /// `explicit_clean_without_a_lock_holder_cleans_and_says_so` is the control. diff --git a/crates/devflow-core/src/recover.rs b/crates/devflow-core/src/recover.rs index 5bd7779c..20d82ebe 100644 --- a/crates/devflow-core/src/recover.rs +++ b/crates/devflow-core/src/recover.rs @@ -1113,8 +1113,11 @@ mod tests { drop(guard); } + /// 999.128 (D-08): a lock whose recorded pid is alive with a different + /// start time names a holder that exited and whose pid was reused, so the + /// explicit clean reclaims it and cleans the phase. #[test] - fn clean_phase_returns_without_cleanup_for_a_recycled_pid_lock() { + fn clean_phase_reclaims_a_recycled_pid_lock_and_cleans() { let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(50); @@ -1131,23 +1134,56 @@ mod tests { crate::lock::holder_status(root, phase), crate::lock::HolderStatus::Recycled { .. } )); + + clean_phase(root, phase).expect("a recycled-pid lock must be reclaimed and cleaned"); + + for path in paths { + assert!( + !path.exists(), + "cleaning at a recycled-pid lock must remove {}", + path.display() + ); + } + assert!( + !lock_path.exists(), + "the reclaimed lock must be released when the clean's guard drops" + ); + } + + /// Control for the reclaim above: the same hand-written record with the + /// observed start time names this live process, so classification (not the + /// coordination flock — no guard is held here) must refuse the clean. + #[test] + fn clean_phase_refuses_a_live_pid_lock_and_cleans_nothing() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let phase = PhaseId::new(50); + let paths = write_phase_artifacts(root, phase); + let lock_path = crate::lock::lock_path(root, phase); + let observed = + crate::agent::process_start_time(std::process::id()).expect("test pid start time"); + std::fs::write(&lock_path, format!("{}\n{}", std::process::id(), observed)).unwrap(); + assert!(matches!( + crate::lock::holder_status(root, phase), + crate::lock::HolderStatus::Live { .. } + )); let mut before = artifact_bytes(&paths); before.push((lock_path.clone(), std::fs::read(&lock_path).unwrap())); - let err = clean_phase(root, phase).expect_err("a recycled-pid lock must be refused"); + let err = clean_phase(root, phase).expect_err("a live identity-matched lock must refuse"); assert!( matches!( err, RecoverError::Lock(crate::lock::LockError::Contended { .. }) ), - "acquisition contention, not read-only classification, controls cleanup: {err:?}" + "a live holder must be reported as a lock refusal: {err:?}" ); for (path, contents) in before { assert_eq!( std::fs::read(&path).unwrap(), contents, - "a recycled-pid lock must preserve {}", + "a live-pid lock must preserve {}", path.display() ); } From 811d6f533b0a3335f4724349db62f25c9e836520 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 12:27:14 -0400 Subject: [PATCH 009/112] test(50-03): skip suite_reap_audit under cargo-nextest The audit reads process-wide counters that the other phase7_cli tests bump, so it only holds when they share its process (cargo test, which scripts/check.sh runs). nextest gives each test its own process, where REAPED stays 0 and the audit could only time out after 60s. Return early when NEXTEST is set. Resolves WINDOWS.md #18 (operator decision 2026-10-17). Verified: nextest -E 'test(=suite_reap_audit)' 1 passed in 0.008s; cargo test --exact suite_reap_audit alone still runs the audit and fails at its deadline (rc 101); cargo test --test phase7_cli 28 passed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/tests/phase7_cli.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/devflow-cli/tests/phase7_cli.rs b/crates/devflow-cli/tests/phase7_cli.rs index f095382c..7001e392 100644 --- a/crates/devflow-cli/tests/phase7_cli.rs +++ b/crates/devflow-cli/tests/phase7_cli.rs @@ -1762,8 +1762,17 @@ fn antigravity_init_without_marker_gates_at_plan() { /// or seen its monitor exit naturally), the registry must DRAIN. Polls /// bounded so a slow legitimate test does not flake; a genuinely leaked /// monitor keeps the registry non-empty and the audit panics naming the pid. +/// +/// The audit reads process-wide counters the other tests in this binary bump, +/// so it only means something when they share its process, as under +/// `cargo test` (what `scripts/check.sh` runs). cargo-nextest runs every test +/// in its own process and sets `NEXTEST`, where `REAPED` is always 0 and the +/// audit could only time out; there it returns without asserting. #[test] fn suite_reap_audit() { + if std::env::var_os("NEXTEST").is_some() { + return; + } let deadline = std::time::Instant::now() + Duration::from_secs(60); // Ordering barrier: wait for a SUSTAINED clean state, not a single empty // poll. A clean state means: no guard in flight (ACTIVE_GUARDS == 0), at From c13feca0dc6825f86a381e00a32a5a43b78377ea Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 12:34:19 -0400 Subject: [PATCH 010/112] test(50-05): add failing probe tests for Gone, Stale and Alive - supervisor module: SocketPath, publish via .part + hard_link, protocol framing - registry::socket_dir_in; path_digest widened to pub(crate) - probe is a stub returning NoReply, so V-01..V-03 fail at their probes Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/lib.rs | 1 + crates/devflow-core/src/registry.rs | 14 +- crates/devflow-core/src/supervisor.rs | 436 ++++++++++++++++++ .../devflow-core/src/supervisor/protocol.rs | 291 ++++++++++++ 4 files changed, 739 insertions(+), 3 deletions(-) create mode 100644 crates/devflow-core/src/supervisor.rs create mode 100644 crates/devflow-core/src/supervisor/protocol.rs diff --git a/crates/devflow-core/src/lib.rs b/crates/devflow-core/src/lib.rs index 53f4fe0b..09297a89 100644 --- a/crates/devflow-core/src/lib.rs +++ b/crates/devflow-core/src/lib.rs @@ -76,6 +76,7 @@ pub mod ship; pub mod ship_evidence; pub mod stage; pub mod state; +pub mod supervisor; /// Test-only helpers, also available to other crates' test targets via the /// `test-support` feature. Never compiled into a normal build. #[cfg(any(test, feature = "test-support"))] diff --git a/crates/devflow-core/src/registry.rs b/crates/devflow-core/src/registry.rs index 8bfe7e4a..0571edc9 100644 --- a/crates/devflow-core/src/registry.rs +++ b/crates/devflow-core/src/registry.rs @@ -5,8 +5,8 @@ //! answers "what is DevFlow doing across every project on this machine?" //! without shelling out to `ps` and `find` (see `23-ORPHAN-FORENSICS.md`). //! This module is that answer: a `(project_root, phase)` pair is registered -//! on the same code path that already writes `state.monitor_pid`, so a -//! running phase cannot be missing from the registry. +//! on the launch path, so a running phase cannot be missing from the +//! registry. //! //! **Storage shape (23-03 revision, cross-AI review BLOCKER 4):** one file //! per `(project_root, phase)` under a `roots/` subdirectory of the cache @@ -67,6 +67,14 @@ pub fn roots_dir_in(cache_dir: &Path) -> PathBuf { cache_dir.join("roots") } +/// The `sock/` subdirectory of a cache dir, where each launch's supervisor +/// socket and its sidecar live (see [`crate::supervisor`]). The socket path is +/// persisted in state when a launch is recorded and is never re-derived at +/// probe time. +pub fn socket_dir_in(cache_dir: &Path) -> PathBuf { + cache_dir.join("sock") +} + /// The deterministic per-registration entry file path for `(project_root, /// phase)`. The digest is only a filename disambiguator — the authoritative /// `project_root` lives inside the file itself, and `load_roots_in` reads it @@ -85,7 +93,7 @@ pub fn entry_path_in(cache_dir: &Path, project_root: &Path, phase: PhaseId) -> P /// DefaultHasher` is deliberately not used here: its output is explicitly /// not guaranteed stable across Rust releases, and an unstable filename /// would orphan every existing entry on a toolchain bump. -fn path_digest(path: &Path) -> u64 { +pub(crate) fn path_digest(path: &Path) -> u64 { use std::os::unix::ffi::OsStrExt; const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325; const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs new file mode 100644 index 00000000..83a6558c --- /dev/null +++ b/crates/devflow-core/src/supervisor.rs @@ -0,0 +1,436 @@ +//! Addressable monitor liveness: a per-launch Unix socket that a supervisor +//! listens on, and the probe that turns a recorded socket path into exactly +//! one of [`Liveness::Gone`], [`Liveness::Stale`] or [`Liveness::Alive`]. +//! +//! **Storage shape.** Each launch gets its own socket, and later a JSON +//! sidecar with the same stem, under `cache_dir/sock` +//! ([`registry::socket_dir_in`]): `--.sock`. The directory is mode `0700` and each socket `0600`. The socket +//! path is persisted in state when the launch is recorded and is never +//! re-derived at probe time, so a probe only ever asks about a path somebody +//! wrote down. +//! +//! **Publication.** A socket is bound and listening under a temporary +//! `.part` name first, then hard-linked to its final `.sock` +//! name. The final name therefore never exists before the listener is +//! listening, and a hard link onto an existing name fails instead of +//! replacing a live socket, as `rename` would. +//! +//! **Liveness reads no pid.** [`probe`] decides from the file at the path, a +//! connect, and one ping round trip. Only a same-uid socket whose connect is +//! refused is `Stale`; any other surprise at the path is a [`ProbeError`], so +//! a stray file can never be mistaken for a dead supervisor (R-6). + +pub mod protocol; + +use crate::phase_id::PhaseId; +use crate::registry; +use std::io; +use std::os::unix::fs::{DirBuilderExt, FileTypeExt, MetadataExt, PermissionsExt}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::{Path, PathBuf}; + +/// The longest socket path Linux accepts: `sun_path` is 108 bytes including +/// its terminating NUL. +pub const SUN_PATH_MAX_LINUX: usize = 107; + +/// The longest socket path macOS accepts: `sun_path` is 104 bytes including +/// its terminating NUL. Unverified on a macOS host. +pub const SUN_PATH_MAX_MACOS: usize = 103; + +/// The socket path limit for the target this was compiled for. +#[cfg(target_os = "macos")] +pub const SUN_PATH_MAX: usize = SUN_PATH_MAX_MACOS; + +/// The socket path limit for the target this was compiled for. +#[cfg(not(target_os = "macos"))] +pub const SUN_PATH_MAX: usize = SUN_PATH_MAX_LINUX; + +/// A socket path DevFlow refuses to build. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum SocketPathError { + /// The path, or its temporary publish name, is longer than the + /// `sun_path` limit; binding it would fail or truncate. + #[error("socket path is {len} bytes, longer than the {max}-byte sun_path limit")] + TooLong { + /// The offending path's length in bytes. + len: usize, + /// The limit it exceeds. + max: usize, + }, +} + +/// A supervisor socket path whose final name and temporary publish name both +/// fit `sun_path`. Callers pass a path ending in `.sock`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SocketPath(PathBuf); + +impl SocketPath { + /// Accept `path` when both it and its [`part_path`](Self::part_path) fit + /// [`SUN_PATH_MAX`] bytes; refuse it by name otherwise, before anything + /// is bound. + pub fn new(path: PathBuf) -> Result { + let socket = Self(path); + for candidate in [socket.0.clone(), socket.part_path()] { + let len = candidate.as_os_str().len(); + if len > SUN_PATH_MAX { + return Err(SocketPathError::TooLong { + len, + max: SUN_PATH_MAX, + }); + } + } + Ok(socket) + } + + /// The socket path for one launch: + /// `socket_dir_in(cache_dir)/--.sock`. + pub fn for_launch( + cache_dir: &Path, + project_root: &Path, + phase: PhaseId, + launch_id: &str, + ) -> Result { + let digest = registry::path_digest(project_root); + Self::new(registry::socket_dir_in(cache_dir).join(format!( + "{digest:016x}-{padded}-{launch_id}.sock", + padded = phase.padded() + ))) + } + + /// The final socket path. + pub fn as_path(&self) -> &Path { + &self.0 + } + + /// The sidecar beside the socket: same stem, `.json`. + pub fn sidecar_path(&self) -> PathBuf { + self.0.with_extension("json") + } + + /// The temporary publish name: same stem, `.part`, in the same directory. + /// It is exactly as long as the final `.sock` name, so the length limit + /// is the same for both, and a sweep that filters on `.sock` never sees a + /// half-published socket. + pub fn part_path(&self) -> PathBuf { + self.0.with_extension("part") + } +} + +/// A fresh launch id: 16 lowercase hex characters from this process's pid, +/// the current time in nanoseconds and a process-wide counter. Uniqueness is +/// also enforced at publish time, which refuses an existing name. +pub fn new_launch_id() -> String { + static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let count = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_nanos() as u64); + let mixed = u64::from(std::process::id()) ^ nanos ^ count.rotate_left(48); + format!("{mixed:016x}") +} + +/// The kernel boot id, trimmed, or `None` where it cannot be read. +pub fn boot_id() -> Option { + read_boot_id(Path::new("/proc/sys/kernel/random/boot_id")) +} + +/// [`boot_id`] against an explicit file, for tests. +pub(crate) fn read_boot_id(path: &Path) -> Option { + let contents = std::fs::read_to_string(path).ok()?; + let trimmed = contents.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) +} + +/// Why a socket could not be published. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum PublishError { + /// Something already exists at the final name; it was left untouched. + #[error("a socket already exists at {0}")] + AlreadyExists(PathBuf), + /// The socket directory is a symlink, not a directory, or owned by + /// another uid. + #[error("unsafe socket directory: {0}")] + UnsafeDir(String), + /// Any other filesystem or socket failure. + #[error("socket publish failed: {0}")] + Io(#[from] io::Error), +} + +/// Publish a listening socket at `path`: bind and listen under the temporary +/// name, restrict it to the owner, then hard-link the final name. The final +/// name never exists before the listener is listening, and an existing final +/// name is never unlinked or replaced. +pub fn publish(path: &SocketPath) -> Result { + publish_with(path, || {}) +} + +/// [`publish`] with a seam that runs after the socket is listening and +/// before the final name is linked, so tests can hold that window open. +pub(crate) fn publish_with( + path: &SocketPath, + before_link: impl FnOnce(), +) -> Result { + let dir = path + .as_path() + .parent() + .ok_or_else(|| PublishError::UnsafeDir("socket path has no parent directory".into()))?; + ensure_socket_dir(dir)?; + let part = path.part_path(); + let listener = UnixListener::bind(&part)?; + let linked = std::fs::set_permissions(&part, std::fs::Permissions::from_mode(0o600)) + .and_then(|()| { + before_link(); + std::fs::hard_link(&part, path.as_path()) + }); + let cleanup = std::fs::remove_file(&part); + match linked { + Ok(()) => { + if let Err(error) = cleanup { + tracing::warn!( + "published supervisor socket at {} but could not remove {}: {error}", + path.as_path().display(), + part.display() + ); + } + Ok(listener) + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { + Err(PublishError::AlreadyExists(path.as_path().to_path_buf())) + } + Err(error) => Err(PublishError::Io(error)), + } +} + +/// Create the socket directory with mode `0700` if absent, then refuse it +/// unless it is a real directory owned by this uid, and reset its mode. +fn ensure_socket_dir(dir: &Path) -> Result<(), PublishError> { + match std::fs::symlink_metadata(dir) { + Err(error) if error.kind() == io::ErrorKind::NotFound => { + if let Some(parent) = dir.parent() { + std::fs::create_dir_all(parent)?; + } + match std::fs::DirBuilder::new().mode(0o700).create(dir) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + } + Err(error) => return Err(error.into()), + Ok(_) => {} + } + let meta = std::fs::symlink_metadata(dir)?; + if meta.file_type().is_symlink() { + return Err(PublishError::UnsafeDir(format!( + "{} is a symlink", + dir.display() + ))); + } + if !meta.is_dir() { + return Err(PublishError::UnsafeDir(format!( + "{} is not a directory", + dir.display() + ))); + } + if meta.uid() != protocol::euid() { + return Err(PublishError::UnsafeDir(format!( + "{} is owned by uid {}, not this user", + dir.display(), + meta.uid() + ))); + } + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?; + Ok(()) +} + +/// What a recorded socket path says about its supervisor. Deliberately +/// exhaustive: these three states are the contract. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Liveness { + /// No file exists at the path. + Gone, + /// Our own socket is there, but its connect is refused: nothing listens. + Stale, + /// A connect followed by a valid ping reply. + Alive(protocol::PingReply), +} + +/// Why a probe could not answer Gone, Stale or Alive. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ProbeError { + /// The path could not be examined. + #[error("could not stat the socket path: {0}")] + Stat(#[source] io::Error), + /// The path holds something other than a socket (a file, a directory, a + /// symlink). + #[error("{} is not a socket", .0.display())] + NotASocket(PathBuf), + /// The socket belongs to another uid. + #[error("the socket is owned by uid {uid}, not this user")] + ForeignOwner { + /// The owning uid. + uid: u32, + }, + /// The connect failed with something other than a refusal, including a + /// socket removed between the stat and the connect. + #[error("could not connect to the socket: {0}")] + Connect(#[source] io::Error), + /// The connect succeeded but no valid ping reply arrived. + #[error("the socket gave no valid ping reply: {0}")] + NoReply(String), +} + +/// Classify the socket recorded at `path`. Reads no pid. +pub fn probe(path: &Path) -> Result { + let _ = path; + Err(ProbeError::NoReply("probe is not implemented yet".into())) +} + +#[cfg(test)] +mod tests { + use super::protocol::{PingReply, Reply, Request}; + use super::*; + use std::sync::mpsc; + use std::thread::{self, JoinHandle}; + + /// A short cache dir under `/tmp`, so socket paths stay far below + /// `sun_path` however long `$TMPDIR` is. + fn short_cache() -> tempfile::TempDir { + tempfile::Builder::new() + .prefix("dfs") + .tempdir_in("/tmp") + .expect("create a short cache dir under /tmp") + } + + fn sample_reply(launch_id: &str) -> PingReply { + PingReply { + launch_id: launch_id.to_string(), + phase: "50".to_string(), + stage: "code".to_string(), + launch: "stream".to_string(), + owner: "agent_running".to_string(), + agent_pid: 1, + agent_pgid: 1, + agent_start_time: 0, + boot_id: boot_id(), + supervisor_pid: std::process::id(), + } + } + + /// Publish a fresh launch's socket under `cache`, returning its path, its + /// launch id and the listening socket. + fn published(cache: &Path) -> (SocketPath, String, UnixListener) { + let launch_id = new_launch_id(); + let socket = SocketPath::for_launch( + cache, + Path::new("/project/root"), + PhaseId::new(50), + &launch_id, + ) + .expect("socket path fits sun_path"); + let listener = publish(&socket).expect("publish the socket"); + (socket, launch_id, listener) + } + + /// Answer one request on an accepted connection, as a supervisor would. + fn answer_one(stream: &UnixStream, launch_id: &str) { + assert!( + protocol::peer_is_same_user(stream), + "a same-user peer must pass the uid check" + ); + let reply = match protocol::read_request(stream) { + Ok(Request::Ping) => Reply::Ping(sample_reply(launch_id)), + _ => Reply::Error("unknown_op".to_string()), + }; + protocol::write_reply(stream, &reply).expect("write the reply"); + } + + /// Serve exactly `count` connections on `listener`, then hand it back. + fn serve_pings( + listener: UnixListener, + launch_id: String, + count: usize, + ) -> JoinHandle { + thread::spawn(move || { + for _ in 0..count { + let (stream, _) = listener.accept().expect("accept a probe"); + answer_one(&stream, &launch_id); + } + listener + }) + } + + fn expect_alive(path: &Path, launch_id: &str) { + match probe(path) { + Ok(Liveness::Alive(reply)) => assert_eq!(reply.launch_id, launch_id), + other => panic!("expected Alive with launch id {launch_id}, got {other:?}"), + } + } + + #[test] + fn probe_reports_alive_only_after_a_ping_reply() { + let cache = short_cache(); + let (socket, launch_id, listener) = published(cache.path()); + let server = serve_pings(listener, launch_id.clone(), 1); + expect_alive(socket.as_path(), &launch_id); + drop(server.join().expect("ping server thread")); + + // Control: a listener that accepts and never replies is not Alive. + let (silent, _, listener) = published(cache.path()); + let (release, released) = mpsc::channel::<()>(); + let holder = thread::spawn(move || { + let (stream, _) = listener.accept().expect("accept the probe"); + released.recv().expect("release signal"); + drop(stream); + }); + let outcome = probe(silent.as_path()); + release.send(()).expect("release the holder"); + holder.join().expect("holder thread"); + assert!( + matches!(outcome, Err(ProbeError::NoReply(_))), + "a connect with no reply must be NoReply, got {outcome:?}" + ); + } + + #[test] + fn probe_reports_stale_only_for_a_socket_whose_listener_is_gone() { + let cache = short_cache(); + let (socket, launch_id, listener) = published(cache.path()); + + // Control: while the listener is held and answering, the path is Alive. + let server = serve_pings(listener, launch_id.clone(), 1); + expect_alive(socket.as_path(), &launch_id); + drop(server.join().expect("ping server thread")); + + assert!( + socket.as_path().exists(), + "dropping the listener must leave the socket file" + ); + let outcome = probe(socket.as_path()); + assert!( + matches!(outcome, Ok(Liveness::Stale)), + "a socket whose listener is gone must be Stale, got {outcome:?}" + ); + } + + #[test] + fn probe_reports_gone_only_when_no_socket_file_exists() { + let cache = short_cache(); + let (socket, launch_id, listener) = published(cache.path()); + + // Control: the same directory with a live listener at the path. + let server = serve_pings(listener, launch_id.clone(), 1); + expect_alive(socket.as_path(), &launch_id); + drop(server.join().expect("ping server thread")); + + std::fs::remove_file(socket.as_path()).expect("remove the socket file"); + let outcome = probe(socket.as_path()); + assert!( + matches!(outcome, Ok(Liveness::Gone)), + "a path with no file must be Gone, got {outcome:?}" + ); + } +} diff --git a/crates/devflow-core/src/supervisor/protocol.rs b/crates/devflow-core/src/supervisor/protocol.rs new file mode 100644 index 00000000..1010d81b --- /dev/null +++ b/crates/devflow-core/src/supervisor/protocol.rs @@ -0,0 +1,291 @@ +//! The supervisor's wire protocol: one request per connection, one JSON +//! object per line, newline-terminated. +//! +//! A line is at most [`MAX_LINE`] bytes including its newline, and every read +//! and write on a connection is bounded by [`IO_TIMEOUT`], so an oversized or +//! slow peer cannot wedge either side (T-50-11). There is no version field: +//! DevFlow makes no cross-version promise for this socket (D-20). +//! +//! | Request | Reply | +//! |---|---| +//! | `{"op":"ping"}` | `{"ok":true, ...PingReply fields}` | +//! | `{"op":"shutdown"}` | `{"ok":true,"accepted":true,"owner":…}` then `{"ok":true,"stopped":true,"exit_code":143,"advance_suppressed":true}`, or `{"ok":false,"error":"handed_off"}` | +//! | anything else | `{"ok":false,"error":"unknown_op"}` | +//! +//! The pids in a ping reply are for display only: nothing on the liveness +//! path reads them. + +use serde::{Deserialize, Serialize}; +use std::io::{self, BufRead, BufReader, Read, Write}; +use std::os::unix::net::UnixStream; +use std::time::Duration; + +/// The longest line either side accepts, newline included. +pub const MAX_LINE: usize = 4096; + +/// The read and write timeout applied to every protocol connection. +pub const IO_TIMEOUT: Duration = Duration::from_secs(2); + +/// A request a client sends to a supervisor, serialized as +/// `{"op":"ping"}` or `{"op":"shutdown"}`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +#[non_exhaustive] +pub enum Request { + /// Ask the supervisor to describe itself. + Ping, + /// Ask the supervisor to stop its agent and exit. + Shutdown, +} + +/// The body of a successful ping reply. On the wire it also carries +/// `"ok":true`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct PingReply { + /// The per-launch id this supervisor was started with. + pub launch_id: String, + /// The phase label, as `PhaseId` displays it. + pub phase: String, + /// The stage the supervised agent runs. + pub stage: String, + /// The launch shape: `capture` (stdin null, stdout to a file) or + /// `stream` (the pipe-owning shape). + pub launch: String, + /// Who owns the run right now: `agent_running`, `advancing`, `gate_wait` + /// or `handed_off`. + pub owner: String, + /// The agent's pid (display only). + pub agent_pid: u32, + /// The agent's process group id (display only). + pub agent_pgid: u32, + /// The agent's start time in clock ticks since boot (display only). + pub agent_start_time: u64, + /// The kernel boot id the supervisor read at start, when readable. + pub boot_id: Option, + /// The supervisor's own pid (display only). + pub supervisor_pid: u32, +} + +/// The first shutdown reply: the request was accepted. On the wire, +/// `{"ok":true,"accepted":true,"owner":…}`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct ShutdownAccepted { + /// The owner phase at the moment the request was accepted. + pub owner: String, +} + +/// The final shutdown reply, sent once the stop is persisted. On the wire, +/// `{"ok":true,"stopped":true,"exit_code":143,"advance_suppressed":true}`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct ShutdownStopped { + /// The exit code the supervisor ends with. + pub exit_code: i32, + /// Whether the stage advance was suppressed by the stop. + pub advance_suppressed: bool, +} + +/// Every reply a server writes, one variant per wire shape. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + not(test), + expect( + dead_code, + reason = "server side of the protocol; the supervisor runtime (50-07) is its first non-test caller" + ) +)] +pub(crate) enum Reply { + /// `{"ok":true, ...}` answering a ping. + Ping(PingReply), + /// `{"ok":true,"accepted":true,"owner":…}`. + Accepted(ShutdownAccepted), + /// `{"ok":true,"stopped":true,...}`. + Stopped(ShutdownStopped), + /// `{"ok":false,"error":…}`, e.g. `unknown_op` or `handed_off`. + Error(String), +} + +impl Reply { + #[cfg_attr( + not(test), + expect(dead_code, reason = "used only by write_reply; see Reply") + )] + fn to_json(&self) -> serde_json::Result { + let mut object = match self { + Self::Ping(reply) => serde_json::to_value(reply)?, + Self::Accepted(reply) => { + let mut value = serde_json::to_value(reply)?; + value["accepted"] = serde_json::Value::Bool(true); + value + } + Self::Stopped(reply) => { + let mut value = serde_json::to_value(reply)?; + value["stopped"] = serde_json::Value::Bool(true); + value + } + Self::Error(error) => { + return Ok(serde_json::json!({ "ok": false, "error": error })); + } + }; + object["ok"] = serde_json::Value::Bool(true); + Ok(object) + } +} + +/// Read one newline-terminated line of at most [`MAX_LINE`] bytes (newline +/// included). A longer line, a line cut off by end of stream, an empty +/// stream and invalid UTF-8 are all errors. +pub(crate) fn read_line(reader: &mut impl BufRead) -> io::Result { + let mut buf = Vec::new(); + reader.take(MAX_LINE as u64).read_until(b'\n', &mut buf)?; + if buf.last() != Some(&b'\n') { + let (kind, message) = if buf.len() >= MAX_LINE { + ( + io::ErrorKind::InvalidData, + format!("protocol line exceeds {MAX_LINE} bytes"), + ) + } else if buf.is_empty() { + ( + io::ErrorKind::UnexpectedEof, + "connection closed before a line arrived".to_string(), + ) + } else { + ( + io::ErrorKind::UnexpectedEof, + "connection closed in the middle of a line".to_string(), + ) + }; + return Err(io::Error::new(kind, message)); + } + buf.pop(); + String::from_utf8(buf).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) +} + +fn set_timeouts(stream: &UnixStream) -> io::Result<()> { + stream.set_read_timeout(Some(IO_TIMEOUT))?; + stream.set_write_timeout(Some(IO_TIMEOUT)) +} + +/// Server side: read the connection's single request. A request that is not +/// valid JSON or names an unknown op is `InvalidData`; the caller answers it +/// with `Reply::Error("unknown_op")`. +#[cfg_attr( + not(test), + expect(dead_code, reason = "server side of the protocol; see Reply") +)] +pub(crate) fn read_request(stream: &UnixStream) -> io::Result { + set_timeouts(stream)?; + let line = read_line(&mut BufReader::new(stream))?; + serde_json::from_str(&line).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) +} + +/// Server side: write one reply line. +#[cfg_attr( + not(test), + expect(dead_code, reason = "server side of the protocol; see Reply") +)] +pub(crate) fn write_reply(mut stream: &UnixStream, reply: &Reply) -> io::Result<()> { + set_timeouts(stream)?; + let mut line = serde_json::to_string(&reply.to_json()?)?; + line.push('\n'); + if line.len() > MAX_LINE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("reply line of {} bytes exceeds {MAX_LINE}", line.len()), + )); + } + stream.write_all(line.as_bytes())?; + stream.flush() +} + +/// The effective uid of this process. +pub(crate) fn euid() -> u32 { + // SAFETY: geteuid has no preconditions and cannot fail. + unsafe { libc::geteuid() } +} + +/// Server side: whether the connected peer runs as this process's effective +/// uid. Linux reads the peer uid with `SO_PEERCRED`; macOS with `getpeereid`. +/// A failed read returns false, and so does every other target, so the uid +/// check is never skipped (T-50-10, T-50-87). +#[cfg(target_os = "linux")] +#[cfg_attr( + not(test), + expect(dead_code, reason = "server side of the protocol; see Reply") +)] +pub(crate) fn peer_is_same_user(stream: &UnixStream) -> bool { + use std::os::fd::AsRawFd; + let mut cred = libc::ucred { + pid: 0, + uid: 0, + gid: 0, + }; + let expected_len = std::mem::size_of::(); + let mut len = expected_len as libc::socklen_t; + // SAFETY: `cred` and `len` are live, correctly sized out-parameters for + // SO_PEERCRED, and the fd is borrowed from a stream that outlives the call. + let rc = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + (&raw mut cred).cast(), + &raw mut len, + ) + }; + rc == 0 && len as usize == expected_len && cred.uid == euid() +} + +/// Server side: whether the connected peer runs as this process's effective +/// uid, read with `getpeereid`. Compile-checked only: no macOS host has run it +/// (backlog 999.160). +#[cfg(target_os = "macos")] +#[cfg_attr( + not(test), + expect(dead_code, reason = "server side of the protocol; see Reply") +)] +pub(crate) fn peer_is_same_user(stream: &UnixStream) -> bool { + use std::os::fd::AsRawFd; + let mut uid: libc::uid_t = 0; + let mut gid: libc::gid_t = 0; + // SAFETY: `uid` and `gid` are live out-parameters, and the fd is borrowed + // from a stream that outlives the call. + let rc = unsafe { libc::getpeereid(stream.as_raw_fd(), &raw mut uid, &raw mut gid) }; + rc == 0 && uid == euid() +} + +/// This target has no peer-uid call DevFlow uses, so every peer is rejected: +/// fail closed. A client then reads no reply (`NoReply`), never `Stale`, so +/// the backstop cannot arm on this target. +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +#[cfg_attr( + not(test), + expect(dead_code, reason = "server side of the protocol; see Reply") +)] +pub(crate) fn peer_is_same_user(_stream: &UnixStream) -> bool { + false +} + +/// Client side: send one ping on a connected stream and parse the reply. +/// Every failure (timeout, closed connection, oversized or malformed line, +/// `"ok":false`) is returned as a message; a connect alone is never proof of +/// life. +pub(crate) fn ping(stream: UnixStream) -> Result { + set_timeouts(&stream).map_err(|error| format!("could not set socket timeouts: {error}"))?; + let request = serde_json::to_string(&Request::Ping) + .map_err(|error| format!("could not encode ping: {error}"))?; + (&stream) + .write_all(format!("{request}\n").as_bytes()) + .map_err(|error| format!("could not send ping: {error}"))?; + let line = read_line(&mut BufReader::new(&stream)) + .map_err(|error| format!("no ping reply: {error}"))?; + let value: serde_json::Value = serde_json::from_str(&line) + .map_err(|error| format!("ping reply is not JSON ({error}): {line}"))?; + if value.get("ok") != Some(&serde_json::Value::Bool(true)) { + return Err(format!("ping reply is not ok: {line}")); + } + serde_json::from_value(value) + .map_err(|error| format!("ping reply is missing fields ({error}): {line}")) +} From dc98124f391bb2adb080708a16706c3acdd1248d Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 12:36:23 -0400 Subject: [PATCH 011/112] feat(50-05): probe a recorded socket into Gone, Stale or Alive - probe: lstat, socket type and uid before connect; only ECONNREFUSED is Stale; a connect counts only after a valid ping reply; reads no pid - peer_is_same_user: SO_PEERCRED on Linux, getpeereid on macOS, false elsewhere - protocol wire-shape and line-cap tests - DEV-SETUP-CHECKLIST: aarch64-apple-darwin target for the macOS compile gate Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor.rs | 41 +++++++++-- .../devflow-core/src/supervisor/protocol.rs | 68 +++++++++++++++++++ 2 files changed, 104 insertions(+), 5 deletions(-) diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs index 83a6558c..f1d12896 100644 --- a/crates/devflow-core/src/supervisor.rs +++ b/crates/devflow-core/src/supervisor.rs @@ -180,8 +180,8 @@ pub(crate) fn publish_with( ensure_socket_dir(dir)?; let part = path.part_path(); let listener = UnixListener::bind(&part)?; - let linked = std::fs::set_permissions(&part, std::fs::Permissions::from_mode(0o600)) - .and_then(|()| { + let linked = + std::fs::set_permissions(&part, std::fs::Permissions::from_mode(0o600)).and_then(|()| { before_link(); std::fs::hard_link(&part, path.as_path()) }); @@ -284,9 +284,40 @@ pub enum ProbeError { } /// Classify the socket recorded at `path`. Reads no pid. +/// +/// 1. No file at the path (`lstat` NotFound) is `Gone`; any other stat +/// failure is [`ProbeError::Stat`]. +/// 2. Anything but a socket (a symlink included) is +/// [`ProbeError::NotASocket`]; a socket of another uid is +/// [`ProbeError::ForeignOwner`]. Both are checked before connecting, +/// because a connect to a regular file or a directory is also refused. +/// 3. A refused connect is `Stale`. Every other connect error is +/// [`ProbeError::Connect`], NotFound included: a socket removed between +/// the stat and the connect is reported, never guessed to be `Gone`. +/// 4. A connect is not proof of life (a listen backlog accepts it with +/// nobody serving): only a valid ping reply makes it `Alive`. pub fn probe(path: &Path) -> Result { - let _ = path; - Err(ProbeError::NoReply("probe is not implemented yet".into())) + let meta = match std::fs::symlink_metadata(path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Liveness::Gone), + Err(error) => return Err(ProbeError::Stat(error)), + Ok(meta) => meta, + }; + if !meta.file_type().is_socket() { + return Err(ProbeError::NotASocket(path.to_path_buf())); + } + if meta.uid() != protocol::euid() { + return Err(ProbeError::ForeignOwner { uid: meta.uid() }); + } + let stream = match UnixStream::connect(path) { + Err(error) if error.kind() == io::ErrorKind::ConnectionRefused => { + return Ok(Liveness::Stale); + } + Err(error) => return Err(ProbeError::Connect(error)), + Ok(stream) => stream, + }; + protocol::ping(stream) + .map(Liveness::Alive) + .map_err(ProbeError::NoReply) } #[cfg(test)] @@ -311,7 +342,7 @@ mod tests { phase: "50".to_string(), stage: "code".to_string(), launch: "stream".to_string(), - owner: "agent_running".to_string(), + owner: "advancing".to_string(), agent_pid: 1, agent_pgid: 1, agent_start_time: 0, diff --git a/crates/devflow-core/src/supervisor/protocol.rs b/crates/devflow-core/src/supervisor/protocol.rs index 1010d81b..e2214d4f 100644 --- a/crates/devflow-core/src/supervisor/protocol.rs +++ b/crates/devflow-core/src/supervisor/protocol.rs @@ -289,3 +289,71 @@ pub(crate) fn ping(stream: UnixStream) -> Result { serde_json::from_value(value) .map_err(|error| format!("ping reply is missing fields ({error}): {line}")) } + +#[cfg(test)] +mod tests { + use super::*; + + fn wire(reply: &Reply) -> serde_json::Value { + reply.to_json().expect("serialize a reply") + } + + #[test] + fn every_reply_has_its_wire_shape() { + let accepted = wire(&Reply::Accepted(ShutdownAccepted { + owner: "agent_running".to_string(), + })); + assert_eq!( + accepted, + serde_json::json!({"ok": true, "accepted": true, "owner": "agent_running"}) + ); + let stopped = wire(&Reply::Stopped(ShutdownStopped { + exit_code: 143, + advance_suppressed: true, + })); + assert_eq!( + stopped, + serde_json::json!({"ok": true, "stopped": true, "exit_code": 143, "advance_suppressed": true}) + ); + assert_eq!( + wire(&Reply::Error("handed_off".to_string())), + serde_json::json!({"ok": false, "error": "handed_off"}) + ); + let ping = wire(&Reply::Ping(PingReply { + launch_id: "00000000000000ab".to_string(), + phase: "50".to_string(), + stage: "code".to_string(), + launch: "capture".to_string(), + owner: "gate_wait".to_string(), + agent_pid: 10, + agent_pgid: 10, + agent_start_time: 99, + boot_id: None, + supervisor_pid: 9, + })); + assert_eq!(ping["ok"], serde_json::Value::Bool(true)); + assert_eq!(ping["launch"], "capture"); + assert_eq!(ping["supervisor_pid"], 9); + assert_eq!( + serde_json::to_string(&Request::Shutdown).expect("encode"), + r#"{"op":"shutdown"}"# + ); + assert!( + serde_json::from_str::(r#"{"op":"reboot"}"#).is_err(), + "an unknown op must not parse" + ); + } + + #[test] + fn a_line_longer_than_max_line_is_refused() { + let long = format!("{}\n", "x".repeat(MAX_LINE)); + let error = read_line(&mut long.as_bytes()).expect_err("an oversized line must fail"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + // Control: the longest allowed line, newline included, is read whole. + let fits = format!("{}\n", "x".repeat(MAX_LINE - 1)); + assert_eq!( + read_line(&mut fits.as_bytes()).expect("a line at the cap is read"), + "x".repeat(MAX_LINE - 1) + ); + } +} From 7931f6415fcfedfd8efe70e2a69e6018967d47e5 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 12:44:56 -0400 Subject: [PATCH 012/112] test(50-05): pin probe totality and the per-fixture liveness table - V-05: regular file, directory, symlink, unstat-able path and mode-000 socket are probe errors, never Stale; the same socket made readable is Stale - V-04: six fixtures, incl. a SIGKILLed holder child, each give one outcome; Gone, Stale and Alive are distinct - test_support::spawn_test_in_child shares run_test_in_child's command - cargo mutants on probe: 6 caught, 2 timeouts, 0 missed Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor.rs | 201 ++++++++++++++++++++++++ crates/devflow-core/src/test_support.rs | 29 +++- 2 files changed, 228 insertions(+), 2 deletions(-) diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs index f1d12896..b191433f 100644 --- a/crates/devflow-core/src/supervisor.rs +++ b/crates/devflow-core/src/supervisor.rs @@ -464,4 +464,205 @@ mod tests { "a path with no file must be Gone, got {outcome:?}" ); } + + /// The V-04 test, re-invoked as a child process to hold a socket until it + /// is SIGKILLed. + const HOLDER_TEST: &str = "supervisor::tests::each_fixture_produces_exactly_one_liveness_state"; + /// The socket path the holder child publishes at. + const HOLDER_SOCKET_ENV: &str = "DEVFLOW_TEST_HOLDER_SOCKET"; + + /// What a probe answered, without the payload, so fixtures can be + /// compared and counted. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + enum Outcome { + Gone, + Stale, + Alive, + NotASocket, + NoReply, + OtherError, + } + + fn outcome(result: &Result) -> Outcome { + match result { + Ok(Liveness::Gone) => Outcome::Gone, + Ok(Liveness::Stale) => Outcome::Stale, + Ok(Liveness::Alive(_)) => Outcome::Alive, + Err(ProbeError::NotASocket(_)) => Outcome::NotASocket, + Err(ProbeError::NoReply(_)) => Outcome::NoReply, + Err(_) => Outcome::OtherError, + } + } + + /// The child half of the SIGKILLed-holder fixture: publish at the path the + /// parent named and answer pings until killed. + fn hold_a_published_socket_until_killed() -> ! { + let path = std::env::var_os(HOLDER_SOCKET_ENV).expect("the parent names the socket path"); + let socket = SocketPath::new(PathBuf::from(path)).expect("holder socket path fits"); + let listener = publish(&socket).expect("holder publishes its socket"); + loop { + let (stream, _) = listener.accept().expect("holder accepts a probe"); + answer_one(&stream, "sigkilled-holder"); + } + } + + /// A socket whose holder was a separate process, SIGKILLed while + /// listening, so nothing in it ran an orderly shutdown. + fn sigkilled_holder_socket(cache: &Path) -> PathBuf { + let socket = SocketPath::for_launch( + cache, + Path::new("/project/holder"), + PhaseId::new(50), + &new_launch_id(), + ) + .expect("socket path fits sun_path"); + let mut child = crate::test_support::spawn_test_in_child( + HOLDER_TEST, + cache, + &[(HOLDER_SOCKET_ENV, socket.as_path().as_os_str())], + ); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while !matches!(probe(socket.as_path()), Ok(Liveness::Alive(_))) { + let exited = child.try_wait().expect("poll the holder child").is_some(); + if exited || std::time::Instant::now() >= deadline { + let _ = child.kill(); + let output = child.wait_with_output().expect("collect holder output"); + panic!( + "the holder child never answered a ping (exited early: {exited}); stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + thread::sleep(std::time::Duration::from_millis(20)); + } + // `Child::kill` sends SIGKILL on Unix, to the pid this test spawned + // and still owns (unreaped), never to a pid read from disk. + child.kill().expect("SIGKILL the holder child"); + child.wait().expect("reap the holder child"); + socket.as_path().to_path_buf() + } + + #[test] + fn each_fixture_produces_exactly_one_liveness_state() { + if crate::test_support::in_child_test(HOLDER_TEST) { + hold_a_published_socket_until_killed(); + } + let cache = short_cache(); + + let no_file = cache.path().join("absent.sock"); + + let (dropped, _, listener) = published(cache.path()); + drop(listener); + + let sigkilled = sigkilled_holder_socket(cache.path()); + + let (answering, answering_id, listener) = published(cache.path()); + let server = serve_pings(listener, answering_id, 1); + + let regular = cache.path().join("regular.sock"); + std::fs::write(®ular, b"not a socket").expect("write a regular file"); + + let (silent, _, silent_listener) = published(cache.path()); + + let fixtures = [ + ("no file", no_file.as_path(), Outcome::Gone), + ("dropped listener", dropped.as_path(), Outcome::Stale), + ("SIGKILLed holder", sigkilled.as_path(), Outcome::Stale), + ( + "ping-answering listener", + answering.as_path(), + Outcome::Alive, + ), + ("regular file", regular.as_path(), Outcome::NotASocket), + ( + "never-replying listener", + silent.as_path(), + Outcome::NoReply, + ), + ]; + let mut positives = std::collections::HashSet::new(); + for (name, path, expected) in fixtures { + let result = probe(path); + assert_eq!( + outcome(&result), + expected, + "fixture `{name}` must produce only {expected:?}, got {result:?}" + ); + if matches!(expected, Outcome::Gone | Outcome::Stale | Outcome::Alive) { + positives.insert(outcome(&result)); + } + } + drop(silent_listener); + drop(server.join().expect("ping server thread")); + assert_eq!( + positives.len(), + 3, + "the positive fixtures must yield three distinct answers, got {positives:?}" + ); + } + + #[test] + fn a_non_socket_or_unreadable_path_is_a_probe_error_not_stale() { + use std::os::unix::fs::symlink; + let cache = short_cache(); + + let regular = cache.path().join("regular.sock"); + std::fs::write(®ular, b"not a socket").expect("write a regular file"); + let directory = cache.path().join("directory.sock"); + std::fs::create_dir(&directory).expect("create a directory"); + let (live, live_id, listener) = published(cache.path()); + let server = serve_pings(listener, live_id.clone(), 1); + expect_alive(live.as_path(), &live_id); + let link = cache.path().join("link.sock"); + symlink(live.as_path(), &link).expect("symlink to the live socket"); + + for (name, path) in [ + ("regular file", regular.as_path()), + ("directory", directory.as_path()), + ("symlink to a live socket", link.as_path()), + ] { + let result = probe(path); + assert!( + matches!(result, Err(ProbeError::NotASocket(_))), + "{name} must be NotASocket, never Stale, got {result:?}" + ); + } + drop(server.join().expect("ping server thread")); + + // A path that cannot be examined (a component is a regular file, so + // lstat fails ENOTDIR) is a Stat error, never Gone. + let under_a_file = regular.join("child.sock"); + let result = probe(&under_a_file); + assert!( + matches!(result, Err(ProbeError::Stat(_))), + "a path that cannot be stat'ed must be a Stat error, got {result:?}" + ); + + let (unreadable, _, listener) = published(cache.path()); + drop(listener); + if protocol::euid() == 0 { + // Root bypasses permission bits, so a mode-000 socket is no + // obstacle to its connect and this row cannot be built (999.113). + eprintln!( + "skipping the mode-000 row: running as root, which bypasses the 0o000 fixture" + ); + } else { + std::fs::set_permissions(unreadable.as_path(), std::fs::Permissions::from_mode(0o000)) + .expect("chmod the socket 000"); + let result = probe(unreadable.as_path()); + assert!( + matches!(&result, Err(ProbeError::Connect(error)) if error.kind() == io::ErrorKind::PermissionDenied), + "an unreadable socket must be a Connect error, never Stale, got {result:?}" + ); + } + + // Control: the same socket file, readable again, is a real stale socket. + std::fs::set_permissions(unreadable.as_path(), std::fs::Permissions::from_mode(0o600)) + .expect("chmod the socket back to 600"); + let result = probe(unreadable.as_path()); + assert!( + matches!(result, Ok(Liveness::Stale)), + "a readable socket with no listener must be Stale, got {result:?}" + ); + } } diff --git a/crates/devflow-core/src/test_support.rs b/crates/devflow-core/src/test_support.rs index c012c024..92ac0eb3 100644 --- a/crates/devflow-core/src/test_support.rs +++ b/crates/devflow-core/src/test_support.rs @@ -156,6 +156,33 @@ pub fn run_test_in_child( path_dir: &std::path::Path, extra_env: &[(&str, &std::ffi::OsStr)], ) -> std::process::Output { + child_test_command(test_name, path_dir, extra_env) + .output() + .expect("spawn child test process with an isolated PATH") +} + +/// [`run_test_in_child`], but spawned rather than waited for: the caller gets +/// the running [`std::process::Child`] (stdout and stderr piped) and owns it, +/// so it can act on the child while it runs and must wait on it afterwards. +pub fn spawn_test_in_child( + test_name: &str, + path_dir: &std::path::Path, + extra_env: &[(&str, &std::ffi::OsStr)], +) -> std::process::Child { + child_test_command(test_name, path_dir, extra_env) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn child test process with an isolated PATH") +} + +/// The command both child-test helpers run: this test binary, filtered to +/// exactly `test_name`, with a child-local `PATH` and environment. +fn child_test_command( + test_name: &str, + path_dir: &std::path::Path, + extra_env: &[(&str, &std::ffi::OsStr)], +) -> std::process::Command { assert!( path_dir.is_dir(), "child test PATH must be an existing directory: {}", @@ -174,8 +201,6 @@ pub fn run_test_in_child( command.env(name, value); } command - .output() - .expect("spawn child test process with an isolated PATH") } /// Asserts that a child test process ran precisely its requested test and did From 861ed1f3e3161c2061f95b4e414ed8cc8612c801 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 12:47:38 -0400 Subject: [PATCH 013/112] test(50-05): pin the sun_path bound and the atomic socket publish - V-06: 106 and 107 bytes publish with a .part name of equal length; 108 is TooLong { len: 108, max: 107 } before any bind; new() checks .part too - V-11: final name reads Gone inside the publish window; a second publish is AlreadyExists and the first listener still answers; raw bind without listen reads Stale (control) - sock/ is 0700 (tightened if looser), socket 0600; symlinked or non-dir sock/ refused Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor.rs | 214 ++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs index b191433f..2f3ac6bf 100644 --- a/crates/devflow-core/src/supervisor.rs +++ b/crates/devflow-core/src/supervisor.rs @@ -665,4 +665,218 @@ mod tests { "a readable socket with no listener must be Stale, got {result:?}" ); } + + /// A `.sock` path in `cache`'s socket directory, padded to exactly `len` + /// bytes, so the boundary does not depend on the tempdir's length. + #[cfg(target_os = "linux")] + fn sock_path_of_len(cache: &Path, len: usize) -> PathBuf { + let dir = registry::socket_dir_in(cache); + let fixed = dir.as_os_str().len() + "/".len() + ".sock".len(); + let path = dir.join(format!("{}.sock", "a".repeat(len - fixed))); + assert_eq!(path.as_os_str().len(), len, "fixture path length"); + path + } + + #[cfg(target_os = "linux")] + #[test] + fn socket_path_of_107_bytes_binds_and_108_is_refused_by_name() { + assert_eq!(SUN_PATH_MAX_LINUX, 107); + assert_eq!( + SUN_PATH_MAX_MACOS, 103, + "the macOS limit is a constant only" + ); + assert_eq!(SUN_PATH_MAX, SUN_PATH_MAX_LINUX); + let cache = short_cache(); + + for len in [106, 107] { + let socket = SocketPath::new(sock_path_of_len(cache.path(), len)) + .unwrap_or_else(|error| panic!("a {len}-byte path must be accepted: {error}")); + assert_eq!( + socket.part_path().as_os_str().len(), + len, + "the temporary name must be exactly as long as the final name" + ); + let listener = publish(&socket) + .unwrap_or_else(|error| panic!("a {len}-byte path must publish: {error}")); + let meta = std::fs::symlink_metadata(socket.as_path()).expect("published path"); + assert!( + meta.file_type().is_socket(), + "{len} bytes: a socket is published" + ); + drop(listener); + } + + let too_long = sock_path_of_len(cache.path(), 108); + assert_eq!( + SocketPath::new(too_long.clone()), + Err(SocketPathError::TooLong { len: 108, max: 107 }) + ); + assert!( + !too_long.exists() && !too_long.with_extension("part").exists(), + "a refused path must be refused before anything is bound" + ); + + // `new` checks the temporary name too: a final path with no `.sock` + // extension fits, but its `.part` name is five bytes longer. + let bare = sock_path_of_len(cache.path(), 112).with_extension(""); + assert_eq!(bare.as_os_str().len(), 107, "fits as a final path"); + assert_eq!( + SocketPath::new(bare), + Err(SocketPathError::TooLong { len: 112, max: 107 }) + ); + } + + #[test] + fn publish_never_exposes_an_unlistening_socket_or_replaces_a_live_one() { + let cache = short_cache(); + let launch_id = new_launch_id(); + let socket = SocketPath::for_launch( + cache.path(), + Path::new("/project/root"), + PhaseId::new(50), + &launch_id, + ) + .expect("socket path fits sun_path"); + + let in_window = std::sync::Arc::new(std::sync::Barrier::new(2)); + let leave_window = std::sync::Arc::new(std::sync::Barrier::new(2)); + let publisher = { + let socket = socket.clone(); + let in_window = in_window.clone(); + let leave_window = leave_window.clone(); + thread::spawn(move || { + publish_with(&socket, || { + in_window.wait(); + leave_window.wait(); + }) + }) + }; + in_window.wait(); + assert!( + std::fs::symlink_metadata(socket.part_path()).is_ok(), + "the temporary name is bound and listening inside the window" + ); + let during = probe(socket.as_path()); + leave_window.wait(); + assert!( + matches!(during, Ok(Liveness::Gone)), + "the final name must not exist before the listener is listening, got {during:?}" + ); + let listener = publisher + .join() + .expect("publisher thread") + .expect("first publish"); + assert!( + !socket.part_path().exists(), + "the temporary name is removed" + ); + + let server = serve_pings(listener, launch_id.clone(), 1); + let second = publish(&socket); + assert!( + matches!(&second, Err(PublishError::AlreadyExists(path)) if path == socket.as_path()), + "a second publish onto a live name must fail, got {second:?}" + ); + assert!( + !socket.part_path().exists(), + "the refused publish removes its temporary name" + ); + expect_alive(socket.as_path(), &launch_id); + drop(server.join().expect("ping server thread")); + + // Control: a socket bound with no listen() reads Stale, which is the + // hazard binding under the temporary name avoids. + let raw = registry::socket_dir_in(cache.path()).join("bound-not-listening.sock"); + let bytes = raw.as_os_str().as_encoded_bytes(); + // SAFETY: plain socket(2) call; the result is checked before use. + let fd = unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0) }; + assert!(fd >= 0, "socket(2) failed: {}", io::Error::last_os_error()); + // SAFETY: fd is a freshly created, owned descriptor. + let _owned = unsafe { ::from_raw_fd(fd) }; + // SAFETY: sockaddr_un is plain old data; all-zero is a valid value. + let mut addr: libc::sockaddr_un = unsafe { std::mem::zeroed() }; + addr.sun_family = libc::AF_UNIX as libc::sa_family_t; + assert!(bytes.len() < addr.sun_path.len(), "raw path fits sun_path"); + for (slot, byte) in addr.sun_path.iter_mut().zip(bytes) { + *slot = *byte as libc::c_char; + } + // SAFETY: addr is a valid, NUL-terminated sockaddr_un of the given size. + let rc = unsafe { + libc::bind( + fd, + (&raw const addr).cast(), + std::mem::size_of::() as libc::socklen_t, + ) + }; + assert_eq!(rc, 0, "bind(2) failed: {}", io::Error::last_os_error()); + let result = probe(&raw); + assert!( + matches!(result, Ok(Liveness::Stale)), + "a bound socket with no listener must read Stale, got {result:?}" + ); + } + + #[test] + fn the_socket_dir_is_private_and_the_socket_is_owner_only() { + let mode = |path: &Path| { + std::fs::symlink_metadata(path) + .expect("stat") + .permissions() + .mode() + & 0o777 + }; + let cache = short_cache(); + let (socket, _, _listener) = published(cache.path()); + assert_eq!(mode(®istry::socket_dir_in(cache.path())), 0o700); + assert_eq!(mode(socket.as_path()), 0o600); + + // An existing, looser socket directory is tightened, not trusted. + let loose = short_cache(); + let loose_dir = registry::socket_dir_in(loose.path()); + std::fs::create_dir(&loose_dir).expect("create sock/"); + std::fs::set_permissions(&loose_dir, std::fs::Permissions::from_mode(0o755)) + .expect("chmod sock/ 755"); + let (_, _, _loose_listener) = published(loose.path()); + assert_eq!(mode(&loose_dir), 0o700); + + // A symlinked or non-directory sock/ is refused. + let target = short_cache(); + let linked = short_cache(); + std::os::unix::fs::symlink(target.path(), registry::socket_dir_in(linked.path())) + .expect("symlink sock/"); + let socket = SocketPath::for_launch( + linked.path(), + Path::new("/project/root"), + PhaseId::new(50), + &new_launch_id(), + ) + .expect("socket path fits"); + let result = publish(&socket); + assert!( + matches!(&result, Err(PublishError::UnsafeDir(why)) if why.contains("is a symlink")), + "a symlinked sock/ must be refused as a symlink, got {result:?}" + ); + assert_eq!( + std::fs::read_dir(target.path()) + .expect("read target") + .count(), + 0, + "nothing is created through the symlink" + ); + + let filed = short_cache(); + std::fs::write(registry::socket_dir_in(filed.path()), b"").expect("sock as a file"); + let socket = SocketPath::for_launch( + filed.path(), + Path::new("/project/root"), + PhaseId::new(50), + &new_launch_id(), + ) + .expect("socket path fits"); + let result = publish(&socket); + assert!( + matches!(result, Err(PublishError::UnsafeDir(_))), + "a sock/ that is not a directory must be refused, got {result:?}" + ); + } } From 18e6c95bf53a5c4883c47dcf9ab00f97405cbe74 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 12:59:00 -0400 Subject: [PATCH 014/112] test(50-06): add failing tests for sidecar and state-less sweep - sidecar_round_trips_and_is_owner_only: read-back, 0600, truncated is Json - a_published_socket_is_found_by_root_and_phase_without_state (tracer) - a_socket_whose_sidecar_will_not_parse_is_still_found_by_its_name (V-36) - Sidecar, SidecarError, SweepEntry types; sweep/read/write bodies stubbed Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor.rs | 453 ++++++++++++++++++++++++++ 1 file changed, 453 insertions(+) diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs index 2f3ac6bf..068997c1 100644 --- a/crates/devflow-core/src/supervisor.rs +++ b/crates/devflow-core/src/supervisor.rs @@ -25,6 +25,7 @@ pub mod protocol; use crate::phase_id::PhaseId; use crate::registry; +use serde::{Deserialize, Serialize}; use std::io; use std::os::unix::fs::{DirBuilderExt, FileTypeExt, MetadataExt, PermissionsExt}; use std::os::unix::net::{UnixListener, UnixStream}; @@ -320,6 +321,142 @@ pub fn probe(path: &Path) -> Result { .map_err(ProbeError::NoReply) } +/// The JSON file beside a socket (same stem, `.json`, mode `0600`) that +/// records who the socket belongs to, so a sweep can answer by project root +/// and phase without loading any state file (R-7). +/// +/// Its identity fields (`agent_pgid`, `agent_start_time`, `boot_id`) are only +/// an input to the stale-socket backstop's own validation; nothing signals a +/// process because a sidecar names it (T-50-13). Built only inside this +/// crate; other crates read it through [`read_sidecar`] and [`sweep_in`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct Sidecar { + /// The per-launch id, also the last segment of the socket's file name. + pub launch_id: String, + /// The project root the launch runs for. + pub project_root: PathBuf, + /// The phase the launch runs for. + pub phase: PhaseId, + /// The stage the supervised agent runs, when one is known. + pub stage: Option, + /// The agent kind, as its label. + pub agent: String, + /// The launch shape: `capture` or `stream`. + pub launch: String, + /// The agent's pid. + pub agent_pid: u32, + /// The agent's process group id. + pub agent_pgid: u32, + /// The agent's start time in clock ticks since boot. + pub agent_start_time: u64, + /// The kernel boot id at launch. + pub boot_id: String, + /// The supervisor's pid. + pub supervisor_pid: u32, + /// The supervisor's start time in clock ticks since boot. + pub supervisor_start_time: u64, + /// When the sidecar was written, RFC 3339 in UTC + /// (`YYYY-MM-DDTHH:MM:SSZ`). + pub created_at: String, +} + +/// Why a sidecar could not be read. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum SidecarError { + /// The file could not be read. + #[error("could not read the sidecar: {0}")] + Io(#[from] io::Error), + /// The file is not a complete sidecar JSON object. + #[error("the sidecar does not parse: {0}")] + Json(#[from] serde_json::Error), +} + +/// `unix_secs` as an RFC 3339 UTC timestamp, `YYYY-MM-DDTHH:MM:SSZ`. +pub(crate) fn rfc3339_from_unix(unix_secs: u64) -> String { + let _ = unix_secs; + String::new() // RED stub +} + +/// The current time as [`rfc3339_from_unix`] formats it, for +/// [`Sidecar::created_at`]. +#[cfg_attr( + not(test), + expect( + dead_code, + reason = "the launcher (50-07) is the first non-test writer of a sidecar" + ) +)] +pub(crate) fn rfc3339_now() -> String { + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()); + rfc3339_from_unix(secs) +} + +/// Write `sidecar` beside `socket` atomically: a uniquely named `.tmp` file +/// in the socket directory, set to mode `0600`, then renamed onto +/// [`SocketPath::sidecar_path`]. The sidecar is a regular file, so `rename` +/// is safe here; only sockets must never be renamed into place. The socket +/// directory is created `0700` if absent and refused if it is unsafe, as +/// [`publish`] does. +#[cfg_attr( + not(test), + expect( + dead_code, + reason = "the launcher (50-07) is the first non-test writer of a sidecar" + ) +)] +pub(crate) fn write_sidecar(socket: &SocketPath, sidecar: &Sidecar) -> io::Result<()> { + let _ = (socket, sidecar); + Ok(()) // RED stub +} + +/// Read and parse the sidecar at `path`. Read-only. +pub fn read_sidecar(path: &Path) -> Result { + let _ = path; + Err(SidecarError::Io(io::Error::other("RED stub"))) +} + +/// One socket stem found under `sock/`: its socket (if present), its sidecar +/// (if present and readable) and its liveness. +#[derive(Debug)] +#[non_exhaustive] +pub struct SweepEntry { + /// `sock/.sock`, whether or not that file exists. + pub socket_path: PathBuf, + /// Whether `.sock` was listed in the directory. + pub socket_present: bool, + /// The parsed sidecar, when `.json` exists and parses. + pub sidecar: Option, + /// `Some` only when `.json` exists but does not read or parse, so a + /// missing sidecar (`None` here and in [`Self::sidecar`]) is never + /// confused with a broken one. + pub sidecar_error: Option, + /// The launch id segment of a stem that fits the + /// `<016x digest>--` scheme, else `None`. + pub launch_id_in_name: Option, + /// [`probe`]'s answer for a present socket; `Ok(Liveness::Gone)` for a + /// sidecar with no socket. + pub liveness: Result, +} + +/// Every socket stem under `cache_dir`'s socket directory, sorted by socket +/// path. Read-only and never acting (D-06): it lists, probes and reads, and +/// removes no file and sends no shutdown. +pub fn sweep_in(cache_dir: &Path) -> Vec { + let _ = cache_dir; + Vec::new() // RED stub +} + +/// The [`sweep_in`] entries that may belong to `project_root` and `phase`. +/// Read-only and never acting (D-06). +pub fn sweep_for_in(cache_dir: &Path, project_root: &Path, phase: PhaseId) -> Vec { + let _ = (cache_dir, project_root, phase); + Vec::new() // RED stub +} + #[cfg(test)] mod tests { use super::protocol::{PingReply, Reply, Request}; @@ -879,4 +1016,320 @@ mod tests { "a sock/ that is not a directory must be refused, got {result:?}" ); } + + const ROOT: &str = "/project/root"; + const OTHER_ROOT: &str = "/project/other"; + + fn sample_sidecar(launch_id: &str, root: &Path, phase: PhaseId) -> Sidecar { + Sidecar { + launch_id: launch_id.to_string(), + project_root: root.to_path_buf(), + phase, + stage: Some("code".to_string()), + agent: "claude".to_string(), + launch: "stream".to_string(), + agent_pid: 4242, + agent_pgid: 4242, + agent_start_time: 123_456, + boot_id: "boot-test".to_string(), + supervisor_pid: std::process::id(), + supervisor_start_time: 654_321, + created_at: rfc3339_from_unix(1_700_000_000), + } + } + + fn reply_for(launch_id: &str, phase: PhaseId) -> PingReply { + let mut reply = sample_reply(launch_id); + reply.phase = phase.to_string(); + reply + } + + /// A ping server that answers every connection until stopped and records + /// each request it read, so a test can prove no shutdown was sent. + struct PingServer { + stop: std::sync::Arc, + handle: JoinHandle<(UnixListener, Vec)>, + } + + impl PingServer { + fn stop(self) -> (UnixListener, Vec) { + self.stop.store(true, std::sync::atomic::Ordering::SeqCst); + self.handle.join().expect("ping server thread") + } + } + + fn serve_until_stopped(listener: UnixListener, reply: PingReply) -> PingServer { + listener + .set_nonblocking(true) + .expect("make the listener non-blocking"); + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = stop.clone(); + let handle = thread::spawn(move || { + let mut seen = Vec::new(); + while !flag.load(std::sync::atomic::Ordering::SeqCst) { + match listener.accept() { + Ok((stream, _)) => { + stream + .set_nonblocking(false) + .expect("make the accepted stream blocking"); + let answer = match protocol::read_request(&stream) { + Ok(request) => { + seen.push(request); + match request { + Request::Ping => Reply::Ping(reply.clone()), + _ => Reply::Error("unknown_op".to_string()), + } + } + Err(_) => Reply::Error("unknown_op".to_string()), + }; + let _ = protocol::write_reply(&stream, &answer); + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + thread::sleep(std::time::Duration::from_millis(5)); + } + Err(error) => panic!("the ping server could not accept: {error}"), + } + } + (listener, seen) + }); + PingServer { stop, handle } + } + + fn paths(entries: &[SweepEntry]) -> Vec<&Path> { + entries + .iter() + .map(|entry| entry.socket_path.as_path()) + .collect() + } + + #[test] + fn a_published_socket_is_found_by_root_and_phase_without_state() { + let cache = short_cache(); + let root = Path::new(ROOT); + let phase = PhaseId::new(7); + let launch_id = new_launch_id(); + let socket = SocketPath::for_launch(cache.path(), root, phase, &launch_id) + .expect("socket path fits sun_path"); + write_sidecar(&socket, &sample_sidecar(&launch_id, root, phase)).expect("write a sidecar"); + let listener = publish(&socket).expect("publish the socket"); + let server = serve_until_stopped(listener, reply_for(&launch_id, phase)); + + let found = sweep_for_in(cache.path(), root, phase); + let other_phase = sweep_for_in(cache.path(), root, PhaseId::new(8)); + let other_root = sweep_for_in(cache.path(), Path::new(OTHER_ROOT), phase); + drop(server.stop()); + + let top_level: Vec<_> = std::fs::read_dir(cache.path()) + .expect("read the cache dir") + .map(|entry| entry.expect("cache entry").file_name()) + .collect(); + assert_eq!( + top_level, + [std::ffi::OsString::from("sock")], + "no state file exists: the cache holds only sock/" + ); + assert_eq!(found.len(), 1, "exactly one entry for (root, 7): {found:?}"); + let entry = &found[0]; + assert_eq!(entry.socket_path, socket.as_path()); + assert!(entry.socket_present, "the socket is listed"); + assert_eq!( + entry + .sidecar + .as_ref() + .map(|sidecar| sidecar.launch_id.as_str()), + Some(launch_id.as_str()), + "the entry carries its sidecar" + ); + assert!( + matches!(&entry.liveness, Ok(Liveness::Alive(reply)) if reply.launch_id == launch_id), + "the entry is Alive, got {:?}", + entry.liveness + ); + assert!( + other_phase.is_empty(), + "another phase finds nothing: {other_phase:?}" + ); + assert!( + other_root.is_empty(), + "another root finds nothing: {other_root:?}" + ); + } + + #[test] + fn sidecar_round_trips_and_is_owner_only() { + let cache = short_cache(); + let root = Path::new(ROOT); + let phase = PhaseId::new(7); + let launch_id = new_launch_id(); + let socket = SocketPath::for_launch(cache.path(), root, phase, &launch_id) + .expect("socket path fits sun_path"); + let sidecar = sample_sidecar(&launch_id, root, phase); + let path = socket.sidecar_path(); + + let written = write_sidecar(&socket, &sidecar); + assert!(written.is_ok(), "write_sidecar failed: {written:?}"); + let read = read_sidecar(&path); + assert_eq!( + read.as_ref().ok(), + Some(&sidecar), + "read_sidecar returns what was written, got {read:?}" + ); + let mode = std::fs::metadata(&path) + .map(|meta| meta.permissions().mode() & 0o777) + .ok(); + assert_eq!(mode, Some(0o600), "the sidecar is owner-only"); + let names: Vec<_> = std::fs::read_dir(registry::socket_dir_in(cache.path())) + .expect("read sock/") + .map(|entry| entry.expect("sock/ entry").file_name()) + .collect(); + assert_eq!( + names, + [path.file_name().expect("sidecar file name").to_os_string()], + "the temporary file is gone after the write" + ); + + // A second write replaces the first in place. + let mut changed = sidecar.clone(); + changed.stage = Some("verify".to_string()); + write_sidecar(&socket, &changed).expect("rewrite the sidecar"); + assert_eq!(read_sidecar(&path).ok(), Some(changed)); + + assert_eq!(rfc3339_from_unix(0), "1970-01-01T00:00:00Z"); + assert_eq!(rfc3339_from_unix(1_700_000_000), "2023-11-14T22:13:20Z"); + assert_eq!(rfc3339_from_unix(951_825_599), "2000-02-29T11:59:59Z"); + let now = rfc3339_now(); + assert!( + now.len() == "1970-01-01T00:00:00Z".len() + && now.starts_with("20") + && now.ends_with('Z'), + "rfc3339_now has the RFC 3339 UTC shape, got {now:?}" + ); + + let bytes = std::fs::read(&path).expect("read the sidecar bytes"); + std::fs::write(&path, &bytes[..bytes.len() / 2]).expect("truncate the sidecar"); + let truncated = read_sidecar(&path); + assert!( + matches!(truncated, Err(SidecarError::Json(_))), + "a truncated sidecar is a Json error, got {truncated:?}" + ); + let missing = read_sidecar(&cache.path().join("absent.json")); + assert!( + matches!(&missing, Err(SidecarError::Io(error)) if error.kind() == io::ErrorKind::NotFound), + "a missing sidecar is an Io error, got {missing:?}" + ); + } + + #[test] + fn a_socket_whose_sidecar_will_not_parse_is_still_found_by_its_name() { + const NAMED: &str = "aaaa0000aaaa0000"; + let cache = short_cache(); + let root = Path::new(ROOT); + let other_root = Path::new(OTHER_ROOT); + let seven = PhaseId::new(7); + let eight = PhaseId::new(8); + + let named = SocketPath::for_launch(cache.path(), root, seven, NAMED) + .expect("socket path fits sun_path"); + write_sidecar(&named, &sample_sidecar(NAMED, root, seven)).expect("write a sidecar"); + let named_server = serve_until_stopped( + publish(&named).expect("publish the named socket"), + reply_for(NAMED, seven), + ); + + // (1) A broken sidecar: found by the name's root digest and phase. + std::fs::write(named.sidecar_path(), b"{").expect("break the sidecar"); + let found = sweep_for_in(cache.path(), root, seven); + assert_eq!( + paths(&found), + [named.as_path()], + "a broken sidecar is found by name" + ); + let entry = &found[0]; + assert!(entry.sidecar.is_none(), "the broken sidecar is not parsed"); + assert!( + entry.sidecar_error.is_some(), + "a broken sidecar is reported as broken" + ); + assert_eq!(entry.launch_id_in_name.as_deref(), Some(NAMED)); + assert!( + matches!(&entry.liveness, Ok(Liveness::Alive(_))), + "the named socket is Alive, got {:?}", + entry.liveness + ); + assert!( + sweep_for_in(cache.path(), other_root, seven).is_empty(), + "another root's digest does not claim it" + ); + assert!( + sweep_for_in(cache.path(), root, eight).is_empty(), + "another phase does not claim it" + ); + + // (2) A missing sidecar: still found, and told apart from a broken one. + std::fs::remove_file(named.sidecar_path()).expect("remove the sidecar"); + let found = sweep_for_in(cache.path(), root, seven); + assert_eq!( + paths(&found), + [named.as_path()], + "a missing sidecar is found by name" + ); + assert!(found[0].sidecar.is_none()); + assert!( + found[0].sidecar_error.is_none(), + "a missing sidecar is not reported as broken" + ); + assert_eq!(found[0].launch_id_in_name.as_deref(), Some(NAMED)); + + // (3) A name outside the scheme with no sidecar: attributed only by the + // phase its ping reply names, for any root. + let foreign = SocketPath::new(registry::socket_dir_in(cache.path()).join("foreign.sock")) + .expect("foreign path fits"); + let foreign_server = serve_until_stopped( + publish(&foreign).expect("publish the foreign socket"), + reply_for("foreign", seven), + ); + for owner in [root, other_root] { + let found = sweep_for_in(cache.path(), owner, seven); + let entry = found + .iter() + .find(|entry| entry.socket_path == foreign.as_path()); + assert!( + entry.is_some(), + "the foreign socket answering phase 7 may belong to {}: {found:?}", + owner.display() + ); + assert_eq!( + entry.and_then(|entry| entry.launch_id_in_name.clone()), + None + ); + } + assert!( + !paths(&sweep_for_in(cache.path(), root, eight)).contains(&foreign.as_path()), + "the foreign socket answering phase 7 is not claimed for phase 8" + ); + + // (4) A parsed sidecar wins over the name. + const MISLABELED: &str = "bbbb0000bbbb0000"; + let mislabeled = SocketPath::for_launch(cache.path(), root, seven, MISLABELED) + .expect("socket path fits sun_path"); + write_sidecar(&mislabeled, &sample_sidecar(MISLABELED, other_root, seven)) + .expect("write the mislabeled sidecar"); + let mislabeled_server = serve_until_stopped( + publish(&mislabeled).expect("publish the mislabeled socket"), + reply_for(MISLABELED, seven), + ); + let for_other = sweep_for_in(cache.path(), other_root, seven); + let for_root = sweep_for_in(cache.path(), root, seven); + drop(mislabeled_server.stop()); + drop(foreign_server.stop()); + drop(named_server.stop()); + assert!( + paths(&for_other).contains(&mislabeled.as_path()), + "the sidecar's root claims it: {for_other:?}" + ); + assert!( + !paths(&for_root).contains(&mislabeled.as_path()), + "the name's root does not, once a sidecar parses: {for_root:?}" + ); + } } From 103354c8a40c9c9931a68a816d08c16b2a243753 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:02:10 -0400 Subject: [PATCH 015/112] feat(50-06): sidecar and read-only sweep by root and phase - write_sidecar: 0600 temp in sock/, renamed onto .json - read_sidecar, Sidecar, SidecarError; created_at is RFC 3339 UTC - sweep_in lists .sock/.json stems, probes and reads; acts on nothing - sweep_for_in: sidecar wins; else name digest+phase, else ping phase - ship::civil_from_days widened to pub(crate) for the timestamp Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/ship.rs | 2 +- crates/devflow-core/src/supervisor.rs | 234 ++++++++++++++++++++++++-- 2 files changed, 221 insertions(+), 15 deletions(-) diff --git a/crates/devflow-core/src/ship.rs b/crates/devflow-core/src/ship.rs index 5b06f321..daa7f323 100644 --- a/crates/devflow-core/src/ship.rs +++ b/crates/devflow-core/src/ship.rs @@ -469,7 +469,7 @@ fn days_from_civil(year: i32, month: u32, day: u32) -> i64 { era * 146_097 + doe - 719_468 } -fn civil_from_days(days: i64) -> (i32, u32, u32) { +pub(crate) fn civil_from_days(days: i64) -> (i32, u32, u32) { let z = days + 719_468; let era = z.div_euclid(146_097); let doe = z - era * 146_097; diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs index 068997c1..ac617dac 100644 --- a/crates/devflow-core/src/supervisor.rs +++ b/crates/devflow-core/src/supervisor.rs @@ -375,8 +375,19 @@ pub enum SidecarError { /// `unix_secs` as an RFC 3339 UTC timestamp, `YYYY-MM-DDTHH:MM:SSZ`. pub(crate) fn rfc3339_from_unix(unix_secs: u64) -> String { - let _ = unix_secs; - String::new() // RED stub + const SECS_PER_MINUTE: u64 = 60; + const SECS_PER_HOUR: u64 = 60 * SECS_PER_MINUTE; + const SECS_PER_DAY: u64 = 24 * SECS_PER_HOUR; + // u64::MAX / SECS_PER_DAY is below 2^48, so the day count fits i64. + let days = i64::try_from(unix_secs / SECS_PER_DAY).unwrap_or(i64::MAX); + let (year, month, day) = crate::ship::civil_from_days(days); + let of_day = unix_secs % SECS_PER_DAY; + format!( + "{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}Z", + hour = of_day / SECS_PER_HOUR, + minute = of_day % SECS_PER_HOUR / SECS_PER_MINUTE, + second = of_day % SECS_PER_MINUTE + ) } /// The current time as [`rfc3339_from_unix`] formats it, for @@ -409,14 +420,48 @@ pub(crate) fn rfc3339_now() -> String { ) )] pub(crate) fn write_sidecar(socket: &SocketPath, sidecar: &Sidecar) -> io::Result<()> { - let _ = (socket, sidecar); - Ok(()) // RED stub + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + static TEMP_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + + let target = socket.sidecar_path(); + let (Some(dir), Some(stem)) = (target.parent(), target.file_stem()) else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{} has no directory or stem", target.display()), + )); + }; + ensure_socket_dir(dir).map_err(|error| match error { + PublishError::Io(error) => error, + unsafe_dir => io::Error::new(io::ErrorKind::PermissionDenied, unsafe_dir.to_string()), + })?; + let contents = serde_json::to_vec_pretty(sidecar)?; + let count = TEMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let mut temp_name = stem.to_os_string(); + temp_name.push(format!(".{}-{count}.tmp", std::process::id())); + let temp = dir.join(temp_name); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temp)?; + let written = file + .write_all(&contents) + .and_then(|()| file.sync_all()) + .and_then(|()| std::fs::set_permissions(&temp, std::fs::Permissions::from_mode(0o600))) + .and_then(|()| std::fs::rename(&temp, &target)); + if written.is_err() { + // The temp name is unique to this call, so removing it touches + // nothing another writer owns. + let _ = std::fs::remove_file(&temp); + } + written } /// Read and parse the sidecar at `path`. Read-only. pub fn read_sidecar(path: &Path) -> Result { - let _ = path; - Err(SidecarError::Io(io::Error::other("RED stub"))) + let contents = std::fs::read_to_string(path)?; + Ok(serde_json::from_str(&contents)?) } /// One socket stem found under `sock/`: its socket (if present), its sidecar @@ -442,19 +487,141 @@ pub struct SweepEntry { pub liveness: Result, } -/// Every socket stem under `cache_dir`'s socket directory, sorted by socket -/// path. Read-only and never acting (D-06): it lists, probes and reads, and -/// removes no file and sends no shutdown. +/// Every socket stem under `cache_dir`'s socket directory, one entry per +/// stem that has a `.sock` or a `.json` file, sorted by socket path. Other +/// names, including a half-published `.part` and a sidecar's in-flight +/// `.tmp`, are skipped. Each present socket is [`probe`]d and each present +/// sidecar read. +/// +/// Read-only and never acting (D-06): it lists, probes and reads, and removes +/// no file and sends no shutdown; `remove_launch_files` is the acting +/// counterpart, and only `stop` and the backstop call it. +/// +/// A socket directory that is absent or cannot be listed yields an empty +/// `Vec`, as `registry::load_roots_in` does: enumeration degrades rather than +/// fails. pub fn sweep_in(cache_dir: &Path) -> Vec { - let _ = cache_dir; - Vec::new() // RED stub + let dir = registry::socket_dir_in(cache_dir); + let Ok(listing) = std::fs::read_dir(&dir) else { + return Vec::new(); + }; + // stem -> (has a `.sock`, has a `.json`) + let mut stems: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + for entry in listing.flatten() { + let name = entry.file_name(); + let Some(name) = name.to_str() else { continue }; + if let Some(stem) = name.strip_suffix(".sock") { + stems.entry(stem.to_string()).or_default().0 = true; + } else if let Some(stem) = name.strip_suffix(".json") { + stems.entry(stem.to_string()).or_default().1 = true; + } + } + let mut entries: Vec = stems + .into_iter() + .map(|(stem, (has_socket, has_sidecar))| sweep_entry(&dir, &stem, has_socket, has_sidecar)) + .collect(); + // Sorted by path, not stem: `a-b.sock` sorts before `a.sock`. + entries.sort_by(|left, right| left.socket_path.cmp(&right.socket_path)); + entries +} + +/// Build one [`SweepEntry`] from what the listing saw. Read-only. +fn sweep_entry(dir: &Path, stem: &str, has_socket: bool, has_sidecar: bool) -> SweepEntry { + let socket_path = dir.join(format!("{stem}.sock")); + let liveness = if has_socket { + probe(&socket_path) + } else { + Ok(Liveness::Gone) + }; + let (sidecar, sidecar_error) = if has_sidecar { + match read_sidecar(&dir.join(format!("{stem}.json"))) { + Ok(sidecar) => (Some(sidecar), None), + // Removed between the listing and the read: missing, not broken. + Err(SidecarError::Io(error)) if error.kind() == io::ErrorKind::NotFound => (None, None), + Err(error) => (None, Some(error.to_string())), + } + } else { + (None, None) + }; + SweepEntry { + socket_path, + socket_present: has_socket, + sidecar, + sidecar_error, + launch_id_in_name: socket_name_parts(stem).map(|(_, _, launch_id)| launch_id.to_string()), + liveness, + } } /// The [`sweep_in`] entries that may belong to `project_root` and `phase`. -/// Read-only and never acting (D-06). +/// +/// An entry whose sidecar parsed matches when the sidecar names that root and +/// phase; the sidecar wins whenever it parses. An entry whose sidecar is +/// missing or broken is attributed by its file name instead (RV-05; D-09 lets +/// the pairing be recovered from the name): a stem that fits the +/// `<016x digest>--` scheme matches when its digest +/// is `project_root`'s and its phase segment is `phase.padded()`. A stem +/// outside the scheme names no root, so it matches any root when its socket +/// is Alive and its ping reply names `phase`: it may belong, and the D-09 +/// guard fails closed on it. +/// +/// Read-only and never acting (D-06): it filters [`sweep_in`]'s output, so it +/// removes no file and sends no shutdown. The name fallback only widens what +/// is reported. pub fn sweep_for_in(cache_dir: &Path, project_root: &Path, phase: PhaseId) -> Vec { - let _ = (cache_dir, project_root, phase); - Vec::new() // RED stub + sweep_in(cache_dir) + .into_iter() + .filter(|entry| may_belong(entry, project_root, phase)) + .collect() +} + +/// Whether `entry` may belong to `project_root` and `phase`; see +/// [`sweep_for_in`]. +fn may_belong(entry: &SweepEntry, project_root: &Path, phase: PhaseId) -> bool { + if let Some(sidecar) = &entry.sidecar { + return sidecar.project_root == project_root && sidecar.phase == phase; + } + let stem = entry + .socket_path + .file_stem() + .and_then(std::ffi::OsStr::to_str) + .unwrap_or_default(); + match socket_name_parts(stem) { + Some((digest, padded, _)) => { + digest == registry::path_digest(project_root) && padded == phase.padded() + } + None => matches!( + &entry.liveness, + Ok(Liveness::Alive(reply)) if reply.phase.parse::().is_ok_and(|named| named == phase) + ), + } +} + +/// Split a socket stem of the form `<016x digest>--` +/// into its parts: 16 lowercase hex digits, a non-empty run of digits and +/// dots (`PhaseId::padded` never contains `-`), and a non-empty launch id, +/// which may itself contain `-`. Anything else is `None`. +fn socket_name_parts(stem: &str) -> Option<(u64, &str, &str)> { + const DIGEST_HEX_LEN: usize = 16; + let (hex, rest) = stem.split_at_checked(DIGEST_HEX_LEN)?; + if !hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return None; + } + let (padded, launch_id) = rest.strip_prefix('-')?.split_once('-')?; + if padded.is_empty() + || launch_id.is_empty() + || !padded + .bytes() + .all(|byte| byte.is_ascii_digit() || byte == b'.') + { + return None; + } + let digest = u64::from_str_radix(hex, 16).ok()?; + Some((digest, padded, launch_id)) } #[cfg(test)] @@ -1219,6 +1386,45 @@ mod tests { ); } + #[test] + fn socket_name_parts_accepts_only_the_launch_scheme() { + let digest = registry::path_digest(Path::new(ROOT)); + let built = SocketPath::for_launch( + Path::new("/c"), + Path::new(ROOT), + "35.1".parse::().expect("phase"), + "orphan-a", + ) + .expect("fits"); + let stem = built + .as_path() + .file_stem() + .and_then(std::ffi::OsStr::to_str) + .expect("utf-8 stem"); + assert_eq!( + socket_name_parts(stem), + Some((digest, "35.1", "orphan-a")), + "a launch id may contain '-'" + ); + for outside in [ + "foreign", + "", + "0123456789ABCDEF-07-id", + "0123456789abcde-07-id", + "0123456789abcdef07-id", + "0123456789abcdef--id", + "0123456789abcdef-07-", + "0123456789abcdef-7a-id", + "0123456789abcdef-07", + ] { + assert_eq!( + socket_name_parts(outside), + None, + "{outside:?} is outside the scheme" + ); + } + } + #[test] fn a_socket_whose_sidecar_will_not_parse_is_still_found_by_its_name() { const NAMED: &str = "aaaa0000aaaa0000"; From 3ec0637b2f7531887d2c1bc095f0391b293b7f60 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:04:19 -0400 Subject: [PATCH 016/112] test(50-06): add failing tests for read-only sweep and shutdown client - sweep_reports_without_removing_or_signalling (V-22) with removal control - request_shutdown_reads_accepted_then_stopped: stop, hand-off, refusal, close, timeout and malformed final line - remove_launch_files, ShutdownOutcome, request_shutdown stubbed Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor.rs | 332 +++++++++++++++++++++++++- 1 file changed, 331 insertions(+), 1 deletion(-) diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs index ac617dac..4305561d 100644 --- a/crates/devflow-core/src/supervisor.rs +++ b/crates/devflow-core/src/supervisor.rs @@ -494,7 +494,7 @@ pub struct SweepEntry { /// sidecar read. /// /// Read-only and never acting (D-06): it lists, probes and reads, and removes -/// no file and sends no shutdown; `remove_launch_files` is the acting +/// no file and sends no shutdown; [`remove_launch_files`] is the acting /// counterpart, and only `stop` and the backstop call it. /// /// A socket directory that is absent or cannot be listed yields an empty @@ -624,6 +624,51 @@ fn socket_name_parts(stem: &str) -> Option<(u64, &str, &str)> { Some((digest, padded, launch_id)) } +/// Remove a launch's socket and its sidecar. A file that is already gone is +/// not an error. +/// +/// This is the acting counterpart of the read-only [`sweep_in`], +/// [`sweep_for_in`], [`probe`] and [`read_sidecar`] (D-06): only `stop` and +/// the stale-socket backstop call it. +pub fn remove_launch_files(socket: &SocketPath) -> io::Result<()> { + let _ = socket; + Ok(()) // RED stub +} + +/// How a supervisor answered a shutdown request. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum ShutdownOutcome { + /// The request was accepted and the final line reported the stop. + Stopped { + /// The exit code the supervisor ends with. + exit_code: i32, + /// Whether the stage advance was suppressed by the stop. + advance_suppressed: bool, + }, + /// The supervisor had already handed the run off and did not stop it. + HandedOff, + /// The supervisor refused the request with this error. + Refused(String), +} + +/// Ask the supervisor at `path` to stop. Applies [`probe`]'s pre-connect +/// checks (a socket, owned by this uid), connects, sends +/// `{"op":"shutdown"}`, reads the immediate `accepted` line within +/// [`protocol::IO_TIMEOUT`], then the final line within `final_timeout`. +/// +/// Only a final `stopped` line is [`ShutdownOutcome::Stopped`]; a +/// `handed_off` answer is [`ShutdownOutcome::HandedOff`] and any other +/// refusal is [`ShutdownOutcome::Refused`]. A missing, late or malformed +/// line is an error, never a stop. +pub fn request_shutdown( + path: &Path, + final_timeout: std::time::Duration, +) -> Result { + let _ = (path, final_timeout); + Err(ProbeError::NoReply("RED stub".to_string())) +} + #[cfg(test)] mod tests { use super::protocol::{PingReply, Reply, Request}; @@ -1538,4 +1583,289 @@ mod tests { "the name's root does not, once a sidecar parses: {for_root:?}" ); } + + /// Every file under `sock/` as `(name, length, mtime)`, sorted. + fn snapshot(cache: &Path) -> Vec<(String, u64, std::time::SystemTime)> { + let mut files: Vec<_> = std::fs::read_dir(registry::socket_dir_in(cache)) + .expect("read sock/") + .map(|entry| { + let entry = entry.expect("sock/ entry"); + let meta = std::fs::symlink_metadata(entry.path()).expect("stat a sock/ entry"); + ( + entry.file_name().to_string_lossy().into_owned(), + meta.len(), + meta.modified().expect("mtime"), + ) + }) + .collect(); + files.sort(); + files + } + + #[test] + fn sweep_reports_without_removing_or_signalling() { + let cache = short_cache(); + let root = Path::new(ROOT); + let phase = PhaseId::new(7); + let launch = |id: &str| { + let socket = SocketPath::for_launch(cache.path(), root, phase, id) + .expect("socket path fits sun_path"); + write_sidecar(&socket, &sample_sidecar(id, root, phase)).expect("write a sidecar"); + socket + }; + let alive_id = new_launch_id(); + let alive = launch(&alive_id); + let server = serve_until_stopped( + publish(&alive).expect("publish the alive socket"), + reply_for(&alive_id, phase), + ); + let stale = launch(&new_launch_id()); + drop(publish(&stale).expect("publish the stale socket")); + let leftover = launch(&new_launch_id()); + let half = SocketPath::for_launch(cache.path(), root, phase, &new_launch_id()) + .expect("socket path fits sun_path"); + std::fs::write(half.part_path(), b"half-published").expect("write a .part file"); + + let before = snapshot(cache.path()); + let entries = sweep_in(cache.path()); + let after = snapshot(cache.path()); + let still_alive = probe(alive.as_path()); + let (_listener, requests) = server.stop(); + + let mut expected = vec![alive.as_path(), stale.as_path(), leftover.as_path()]; + expected.sort(); + assert_eq!( + paths(&entries), + expected, + "one entry per .sock/.json stem; the .part is not one" + ); + let liveness_of = |socket: &SocketPath| { + let entry = entries + .iter() + .find(|entry| entry.socket_path == socket.as_path()) + .expect("entry present"); + (entry.socket_present, &entry.liveness) + }; + assert!(matches!( + liveness_of(&alive), + (true, Ok(Liveness::Alive(_))) + )); + assert!(matches!(liveness_of(&stale), (true, Ok(Liveness::Stale)))); + assert!(matches!( + liveness_of(&leftover), + (false, Ok(Liveness::Gone)) + )); + assert_eq!( + before, after, + "the sweep leaves every name, length and mtime under sock/ unchanged" + ); + assert!( + matches!(still_alive, Ok(Liveness::Alive(_))), + "the alive listener still answers after the sweep, got {still_alive:?}" + ); + assert!( + requests.iter().all(|request| *request == Request::Ping), + "the sweep sent only pings, got {requests:?}" + ); + + // Control: the acting function changes the same snapshot, so the + // comparison above can see a removal. + remove_launch_files(&stale).expect("remove the stale launch's files"); + let removed = snapshot(cache.path()); + assert_ne!(before, removed, "the snapshot detects a removal"); + let stale_names = [stale.as_path(), stale.sidecar_path().as_path()].map(|path| { + path.file_name() + .expect("name") + .to_string_lossy() + .into_owned() + }); + assert_eq!( + removed, + before + .iter() + .filter(|(name, _, _)| !stale_names.contains(name)) + .cloned() + .collect::>(), + "exactly the stale socket and its sidecar are removed" + ); + assert!( + remove_launch_files(&stale).is_ok(), + "removing files that are already gone succeeds" + ); + } + + /// Accept one connection on `listener`, require a shutdown request, then + /// run `script` on the connection. + fn serve_shutdown( + listener: UnixListener, + script: impl FnOnce(&UnixStream) + Send + 'static, + ) -> JoinHandle<()> { + thread::spawn(move || { + // Poll, so a client that never connects ends the thread instead + // of wedging the test on `accept`. + listener + .set_nonblocking(true) + .expect("make the listener non-blocking"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + if std::time::Instant::now() >= deadline { + return; + } + thread::sleep(std::time::Duration::from_millis(5)); + } + Err(error) => panic!("the scripted server could not accept: {error}"), + } + }; + stream + .set_nonblocking(false) + .expect("make the accepted stream blocking"); + let request = protocol::read_request(&stream); + assert!( + matches!(request, Ok(Request::Shutdown)), + "the client sends a shutdown request, got {request:?}" + ); + script(&stream); + }) + } + + fn write_raw(mut stream: &UnixStream, line: &str) { + use std::io::Write; + stream + .write_all(format!("{line}\n").as_bytes()) + .expect("write a raw line"); + } + + #[test] + fn request_shutdown_reads_accepted_then_stopped() { + use std::time::Duration; + let cache = short_cache(); + let accepted = || { + Reply::Accepted(protocol::ShutdownAccepted { + owner: "agent_running".to_string(), + }) + }; + let run = |script: Box, final_timeout: Duration| { + let (socket, _, listener) = published(cache.path()); + let server = serve_shutdown(listener, script); + let outcome = request_shutdown(socket.as_path(), final_timeout); + server.join().expect("scripted server thread"); + outcome + }; + + let stopped = run( + Box::new(move |stream| { + protocol::write_reply(stream, &accepted()).expect("write accepted"); + thread::sleep(Duration::from_millis(200)); + protocol::write_reply( + stream, + &Reply::Stopped(protocol::ShutdownStopped { + exit_code: 143, + advance_suppressed: true, + }), + ) + .expect("write stopped"); + }), + Duration::from_secs(5), + ); + assert!( + matches!( + stopped, + Ok(ShutdownOutcome::Stopped { + exit_code: 143, + advance_suppressed: true + }) + ), + "accepted then stopped is Stopped, got {stopped:?}" + ); + + let handed_off = run( + Box::new(|stream| { + protocol::write_reply(stream, &Reply::Error("handed_off".to_string())) + .expect("write handed_off"); + }), + Duration::from_secs(5), + ); + assert!( + matches!(handed_off, Ok(ShutdownOutcome::HandedOff)), + "handed_off is HandedOff, got {handed_off:?}" + ); + + let late_hand_off = run( + Box::new(move |stream| { + protocol::write_reply(stream, &accepted()).expect("write accepted"); + protocol::write_reply(stream, &Reply::Error("handed_off".to_string())) + .expect("write handed_off"); + }), + Duration::from_secs(5), + ); + assert!( + matches!(late_hand_off, Ok(ShutdownOutcome::HandedOff)), + "a hand-off after accepted is HandedOff, never Stopped, got {late_hand_off:?}" + ); + + let refused = run( + Box::new(|stream| { + protocol::write_reply(stream, &Reply::Error("unknown_op".to_string())) + .expect("write the refusal"); + }), + Duration::from_secs(5), + ); + assert!( + matches!(&refused, Ok(ShutdownOutcome::Refused(error)) if error == "unknown_op"), + "another refusal is Refused, got {refused:?}" + ); + + let closed = run( + Box::new(move |stream| { + protocol::write_reply(stream, &accepted()).expect("write accepted"); + }), + Duration::from_secs(5), + ); + assert!( + matches!(closed, Err(ProbeError::NoReply(_))), + "a close after accepted is an error, never Stopped, got {closed:?}" + ); + + let silent = run( + Box::new(move |stream| { + use std::io::Read; + protocol::write_reply(stream, &accepted()).expect("write accepted"); + // Hold the connection open until the client gives up and + // closes it, past its final timeout. + let _ = (&mut &*stream).read(&mut [0_u8; 1]); + }), + Duration::from_millis(300), + ); + assert!( + matches!(silent, Err(ProbeError::NoReply(_))), + "no final line within the timeout is an error, got {silent:?}" + ); + + let malformed = run( + Box::new(move |stream| { + protocol::write_reply(stream, &accepted()).expect("write accepted"); + write_raw( + stream, + r#"{"ok":true,"exit_code":143,"advance_suppressed":true}"#, + ); + }), + Duration::from_secs(5), + ); + assert!( + matches!(malformed, Err(ProbeError::NoReply(_))), + "a final line without stopped:true is an error, got {malformed:?}" + ); + + // The pre-connect checks are probe's: a regular file is never asked. + let regular = cache.path().join("regular.sock"); + std::fs::write(®ular, b"not a socket").expect("write a regular file"); + let not_a_socket = request_shutdown(®ular, Duration::from_secs(1)); + assert!( + matches!(not_a_socket, Err(ProbeError::NotASocket(_))), + "a regular file is NotASocket, got {not_a_socket:?}" + ); + } } From 002fc029c22d6ca930038ca97ce83b9da1ae1b7b Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:06:15 -0400 Subject: [PATCH 017/112] feat(50-06): acting launch-file removal and shutdown client - remove_launch_files: removes .sock and sidecar, NotFound is success - request_shutdown: probe's pre-connect checks, then accepted within IO_TIMEOUT and the final line within final_timeout; only a stopped line is Stopped, handed_off is HandedOff, other refusals Refused - probe's lstat/socket/uid checks factored into own_socket_at Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor.rs | 146 +++++++++++++++++++++++--- 1 file changed, 132 insertions(+), 14 deletions(-) diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs index 4305561d..1b3a6031 100644 --- a/crates/devflow-core/src/supervisor.rs +++ b/crates/devflow-core/src/supervisor.rs @@ -298,16 +298,8 @@ pub enum ProbeError { /// 4. A connect is not proof of life (a listen backlog accepts it with /// nobody serving): only a valid ping reply makes it `Alive`. pub fn probe(path: &Path) -> Result { - let meta = match std::fs::symlink_metadata(path) { - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Liveness::Gone), - Err(error) => return Err(ProbeError::Stat(error)), - Ok(meta) => meta, - }; - if !meta.file_type().is_socket() { - return Err(ProbeError::NotASocket(path.to_path_buf())); - } - if meta.uid() != protocol::euid() { - return Err(ProbeError::ForeignOwner { uid: meta.uid() }); + if !own_socket_at(path)? { + return Ok(Liveness::Gone); } let stream = match UnixStream::connect(path) { Err(error) if error.kind() == io::ErrorKind::ConnectionRefused => { @@ -321,6 +313,25 @@ pub fn probe(path: &Path) -> Result { .map_err(ProbeError::NoReply) } +/// [`probe`]'s pre-connect checks, shared with [`request_shutdown`]: +/// `Ok(false)` when no file is at `path` (`lstat` NotFound), `Ok(true)` for a +/// socket owned by this uid, and a [`ProbeError`] for a stat failure, a +/// non-socket (a symlink included) or another uid's socket. +fn own_socket_at(path: &Path) -> Result { + let meta = match std::fs::symlink_metadata(path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(ProbeError::Stat(error)), + Ok(meta) => meta, + }; + if !meta.file_type().is_socket() { + return Err(ProbeError::NotASocket(path.to_path_buf())); + } + if meta.uid() != protocol::euid() { + return Err(ProbeError::ForeignOwner { uid: meta.uid() }); + } + Ok(true) +} + /// The JSON file beside a socket (same stem, `.json`, mode `0600`) that /// records who the socket belongs to, so a sweep can answer by project root /// and phase without loading any state file (R-7). @@ -631,8 +642,17 @@ fn socket_name_parts(stem: &str) -> Option<(u64, &str, &str)> { /// [`sweep_for_in`], [`probe`] and [`read_sidecar`] (D-06): only `stop` and /// the stale-socket backstop call it. pub fn remove_launch_files(socket: &SocketPath) -> io::Result<()> { - let _ = socket; - Ok(()) // RED stub + let mut first_error = None; + for path in [socket.as_path().to_path_buf(), socket.sidecar_path()] { + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + first_error.get_or_insert(error); + } + } + } + first_error.map_or(Ok(()), Err) } /// How a supervisor answered a shutdown request. @@ -665,8 +685,106 @@ pub fn request_shutdown( path: &Path, final_timeout: std::time::Duration, ) -> Result { - let _ = (path, final_timeout); - Err(ProbeError::NoReply("RED stub".to_string())) + use std::io::Write; + if !own_socket_at(path)? { + return Err(ProbeError::Stat(io::Error::new( + io::ErrorKind::NotFound, + format!("no socket at {}", path.display()), + ))); + } + let stream = UnixStream::connect(path).map_err(ProbeError::Connect)?; + let timeout_error = + |error: io::Error| ProbeError::NoReply(format!("could not set socket timeouts: {error}")); + stream + .set_read_timeout(Some(protocol::IO_TIMEOUT)) + .and_then(|()| stream.set_write_timeout(Some(protocol::IO_TIMEOUT))) + .map_err(timeout_error)?; + let request = serde_json::to_string(&protocol::Request::Shutdown) + .map_err(|error| ProbeError::NoReply(format!("could not encode shutdown: {error}")))?; + (&stream) + .write_all(format!("{request}\n").as_bytes()) + .map_err(|error| ProbeError::NoReply(format!("could not send shutdown: {error}")))?; + // One reader across both lines: it may already hold the final line. + let mut reader = io::BufReader::new(&stream); + match read_shutdown_line(&mut reader, "accepted")? { + ShutdownLine::Accepted => {} + ShutdownLine::Refused(error) => return Ok(refusal(error)), + ShutdownLine::Stopped(_) => { + return Err(ProbeError::NoReply( + "a stopped line arrived before the accepted line".to_string(), + )); + } + } + stream + .set_read_timeout(Some(final_timeout)) + .map_err(timeout_error)?; + match read_shutdown_line(&mut reader, "final")? { + ShutdownLine::Stopped(stopped) => Ok(ShutdownOutcome::Stopped { + exit_code: stopped.exit_code, + advance_suppressed: stopped.advance_suppressed, + }), + ShutdownLine::Refused(error) => Ok(refusal(error)), + ShutdownLine::Accepted => Err(ProbeError::NoReply( + "a second accepted line arrived instead of the final line".to_string(), + )), + } +} + +/// One parsed shutdown reply line. +enum ShutdownLine { + Accepted, + Stopped(protocol::ShutdownStopped), + Refused(String), +} + +/// A refusal's error string as an outcome: `handed_off` is its own outcome. +fn refusal(error: String) -> ShutdownOutcome { + if error == "handed_off" { + ShutdownOutcome::HandedOff + } else { + ShutdownOutcome::Refused(error) + } +} + +/// Read and classify one shutdown reply line. A missing, oversized or +/// malformed line, `"ok":true` with neither marker, and `"ok":false` with no +/// error string are all errors. +fn read_shutdown_line( + reader: &mut impl io::BufRead, + which: &str, +) -> Result { + let line = protocol::read_line(reader) + .map_err(|error| ProbeError::NoReply(format!("no {which} shutdown line: {error}")))?; + let value: serde_json::Value = serde_json::from_str(&line).map_err(|error| { + ProbeError::NoReply(format!( + "{which} shutdown line is not JSON ({error}): {line}" + )) + })?; + let is_true = |key: &str| value.get(key) == Some(&serde_json::Value::Bool(true)); + let (ok, stopped, accepted) = (is_true("ok"), is_true("stopped"), is_true("accepted")); + if value.get("ok") == Some(&serde_json::Value::Bool(false)) { + return match value.get("error").and_then(serde_json::Value::as_str) { + Some(error) => Ok(ShutdownLine::Refused(error.to_string())), + None => Err(ProbeError::NoReply(format!( + "{which} shutdown line refuses without an error: {line}" + ))), + }; + } + if ok && stopped { + return serde_json::from_value(value) + .map(ShutdownLine::Stopped) + .map_err(|error| { + ProbeError::NoReply(format!( + "{which} shutdown line is missing fields ({error}): {line}" + )) + }); + } + if ok && accepted { + return Ok(ShutdownLine::Accepted); + } + Err(ProbeError::NoReply(format!( + "{which} shutdown line is neither accepted nor stopped: {line}" + ))) } #[cfg(test)] From cfe873ba54f74a39ab9b4ee3c81c3f03b3d5585d Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:07:28 -0400 Subject: [PATCH 018/112] test(50-06): add failing test for persisted supervisor record - supervisor_record_round_trips_and_absent_reads_as_none via save/load - SupervisorRecord and State::supervisor added; field stubbed serde(skip) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/state.rs | 94 ++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/crates/devflow-core/src/state.rs b/crates/devflow-core/src/state.rs index 248b71e7..0d9eb022 100644 --- a/crates/devflow-core/src/state.rs +++ b/crates/devflow-core/src/state.rs @@ -286,6 +286,17 @@ pub struct State { /// liveness probe reports Unknown, never Stuck. #[serde(default)] pub monitor_pid: Option, + /// The published, ping-confirmed address of the supervisor that owns the + /// current stage (D-04): its socket path, launch id, and the agent + /// identity the stale-socket backstop validates. Written by the launcher, + /// under the per-phase lock, in one save that replaces the previous + /// record; it is never cleared ahead of that save. `None` means no + /// supervisor has been published for this state, OR the state was written + /// by a binary predating this field. The socket path is the only handle a + /// liveness question starts from: it is passed to + /// [`crate::supervisor::probe`] as recorded, never re-derived. + #[serde(skip)] // RED stub: not persisted yet + pub supervisor: Option, /// The Claude session id captured from the most recent captured stdout /// envelope for this phase's current stage (D-04, 28-02), read via /// [`crate::agent_result::session_id_from_capture`]. `None` means EITHER @@ -437,6 +448,47 @@ pub struct State { pub gate_notify_cmd: Option, } +/// The persisted handle on a launch's supervisor ([`State::supervisor`]). +/// +/// The socket path is where [`crate::supervisor::probe`] asks; the launch id, +/// agent process group, agent start time and boot id are what the +/// stale-socket backstop checks before it may act, so a record from another +/// launch or another boot never authorizes a signal. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] +pub struct SupervisorRecord { + /// The published socket, `cache_dir/sock/.sock`. + pub socket_path: PathBuf, + /// The per-launch id, also the last segment of the socket's file name. + pub launch_id: String, + /// The agent's process group id. + pub agent_pgid: u32, + /// The agent's start time in clock ticks since boot. + pub agent_start_time: u64, + /// The kernel boot id at launch. + pub boot_id: String, +} + +impl SupervisorRecord { + /// Build a record; the only way other crates can, since the struct is + /// `#[non_exhaustive]`. + pub fn new( + socket_path: PathBuf, + launch_id: String, + agent_pgid: u32, + agent_start_time: u64, + boot_id: String, + ) -> Self { + Self { + socket_path, + launch_id, + agent_pgid, + agent_start_time, + boot_id, + } + } +} + /// One human-only checkpoint recorded as seen for this run. /// /// Stores the declaration's own text rather than a hash: a hash would make @@ -591,6 +643,7 @@ impl State { project_root, worktree_path: None, monitor_pid: None, + supervisor: None, session_id: None, checkpoint_resumes: 0, stop_until: None, @@ -1196,6 +1249,47 @@ mod tests { ); } + /// `supervisor` survives a real save and load, and a state file without + /// the key (an older binary's) loads with `None` (D-04, 50-06). + #[test] + fn supervisor_record_round_trips_and_absent_reads_as_none() { + let dir = tempfile::tempdir().expect("tempdir"); + let phase = PhaseId::new(50); + let mut state = State::new( + phase, + AgentKind::Claude, + Mode::Auto, + dir.path().to_path_buf(), + ); + assert_eq!(state.supervisor, None, "a new state has no supervisor"); + let record = SupervisorRecord::new( + PathBuf::from("/c/sock/0123456789abcdef-50-00000000000000ab.sock"), + "00000000000000ab".to_string(), + 4242, + 123_456, + "boot-id".to_string(), + ); + state.supervisor = Some(record.clone()); + crate::workflow::save_state(&state).expect("save the state"); + let loaded = crate::workflow::load_state(dir.path(), phase).expect("load the state"); + assert_eq!( + loaded.supervisor, + Some(record), + "the supervisor record must survive a save and load" + ); + + let absent = r#"{ + "stage": "code", + "phase": 50, + "agent": "claude", + "mode": "auto", + "started_at": "0", + "project_root": "/repo" + }"#; + let loaded: State = serde_json::from_str(absent).expect("an older state file loads"); + assert_eq!(loaded.supervisor, None, "an absent key reads as None"); + } + /// A serde-absent `monitor_pid` (state written by a pre-18b binary) must /// deserialize to `None`, not `Some(0)` — a `Some(0)` default would let a /// pre-18b state file render as a monitor at pid 0. From dd2f1305b3422a5e293ac3e11d0629d24daab14b Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:09:36 -0400 Subject: [PATCH 019/112] feat(50-06): persist State::supervisor - serde(default) Option: saved records reload equal, older state files without the key load with None Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/devflow-core/src/state.rs b/crates/devflow-core/src/state.rs index 0d9eb022..e91a1a46 100644 --- a/crates/devflow-core/src/state.rs +++ b/crates/devflow-core/src/state.rs @@ -295,7 +295,7 @@ pub struct State { /// by a binary predating this field. The socket path is the only handle a /// liveness question starts from: it is passed to /// [`crate::supervisor::probe`] as recorded, never re-derived. - #[serde(skip)] // RED stub: not persisted yet + #[serde(default)] pub supervisor: Option, /// The Claude session id captured from the most recent captured stdout /// envelope for this phase's current stage (D-04, 28-02), read via From 29a76603816e7e49c3701b33adc603d0d603506a Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:16:27 -0400 Subject: [PATCH 020/112] test(50-21): migrate three e2e suites to the hermetic builder - auto_chain_flag_e2e, yes_ship_config, release_check spawn devflow only through common::hermetic_devflow; their local devflow_bin() is deleted - auto_chain_flag_e2e binds a LaunchedRunReaper after each foreground __monitor run, covering a regressed stop_until halt - new hermetic_spawn_sites.rs: permanent source scan (R6-G4) with planted controls and a real-data control per suite Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- .../devflow-cli/tests/auto_chain_flag_e2e.rs | 30 +++- .../devflow-cli/tests/hermetic_spawn_sites.rs | 135 ++++++++++++++++++ crates/devflow-cli/tests/release_check.rs | 16 +-- crates/devflow-cli/tests/yes_ship_config.rs | 14 +- 4 files changed, 174 insertions(+), 21 deletions(-) create mode 100644 crates/devflow-cli/tests/hermetic_spawn_sites.rs diff --git a/crates/devflow-cli/tests/auto_chain_flag_e2e.rs b/crates/devflow-cli/tests/auto_chain_flag_e2e.rs index d7684851..d1487f9f 100644 --- a/crates/devflow-cli/tests/auto_chain_flag_e2e.rs +++ b/crates/devflow-cli/tests/auto_chain_flag_e2e.rs @@ -13,6 +13,12 @@ //! the guard scoped to the wrong stack frame — that is RESEARCH Pitfall 2, and //! it is the entire reason this is an end-to-end test rather than a unit test //! of the writer. +//! +//! Every `devflow` child is built by `common::hermetic_devflow`, so the +//! monitor's registry entries land in the fixture's own tempdir `HOME`, never +//! in the operator's real `~/.cache/devflow` (999.141). + +mod common; use devflow_core::mode::Mode; use devflow_core::phase_id::PhaseId; @@ -22,11 +28,7 @@ use devflow_core::workflow::save_state; use std::fs; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; -use std::process::{Command, Output}; - -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} +use std::process::Output; /// Hermetic git invocation pinned to `root` (999.37) — never a bare /// `Command::new("git")`. @@ -146,6 +148,9 @@ struct Fixture { observation: PathBuf, prompt_file: PathBuf, script: PathBuf, + /// Every `devflow` child's `HOME` and cache. Declared last so it drops + /// after the rest of the fixture. + home: common::HermeticHome, } impl Fixture { @@ -191,12 +196,13 @@ impl Fixture { observation, prompt_file, script, + home: common::HermeticHome::new(), } } /// Run the REAL binary's hidden `__monitor` subcommand over `argv`. fn run_monitor(&self, argv: &[&str]) -> Output { - Command::new(devflow_bin()) + common::hermetic_devflow(&self.home) .arg("__monitor") .arg("--project") .arg(&self.root) @@ -247,6 +253,10 @@ fn auto_mode_code_stage_child_observes_the_flag_set() { let fixture = Fixture::new(Mode::Auto, false); let output = fixture.run_observer(); + // The foreground monitor halts at the Code boundary (`stop_until`), so + // it should launch nothing. If that halt ever regresses, the next stage's + // detached monitor and agent are reaped here instead of outliving the test. + let _run_reaper = common::LaunchedRunReaper::new(&fixture.root, fixture.phase, Vec::new()); assert!( output.status.success(), "monitor exited {:?}\nstdout:\n{}\nstderr:\n{}", @@ -287,6 +297,10 @@ fn supervise_mode_code_stage_child_observes_the_flag_clear() { let before = fs::read(&fixture.config).unwrap(); let output = fixture.run_observer(); + // The foreground monitor halts at the Code boundary (`stop_until`), so + // it should launch nothing. If that halt ever regresses, the next stage's + // detached monitor and agent are reaped here instead of outliving the test. + let _run_reaper = common::LaunchedRunReaper::new(&fixture.root, fixture.phase, Vec::new()); assert!( output.status.success(), "monitor exited {:?}\nstdout:\n{}\nstderr:\n{}", @@ -323,6 +337,10 @@ fn guard_clears_the_flag_when_the_supervised_child_fails() { let fixture = Fixture::new(Mode::Auto, true); let output = fixture.run_monitor(&["definitely-not-a-real-program-35-1-01"]); + // The foreground monitor halts at the Code boundary (`stop_until`), so + // it should launch nothing. If that halt ever regresses, the next stage's + // detached monitor and agent are reaped here instead of outliving the test. + let _run_reaper = common::LaunchedRunReaper::new(&fixture.root, fixture.phase, Vec::new()); assert!( !output.status.success(), "a child that cannot be spawned must fail the monitor, not pass silently" diff --git a/crates/devflow-cli/tests/hermetic_spawn_sites.rs b/crates/devflow-cli/tests/hermetic_spawn_sites.rs new file mode 100644 index 00000000..9d5af90b --- /dev/null +++ b/crates/devflow-cli/tests/hermetic_spawn_sites.rs @@ -0,0 +1,135 @@ +//! Permanent source scan for 999.141 / D-07 (50-21, R6-G4): every e2e suite +//! spawns `devflow` through `common::hermetic_devflow`, so no suite's child +//! reaches the operator's real `~/.cache/devflow` (and, from 50-11, no suite +//! binds a socket there). +//! +//! It replaces the shell counts the migration plans used as gates. It spawns +//! nothing and does not include the shared test helpers. The scanned tokens +//! are spelled through `concat!`, so this file never holds the binary's env +//! name or a raw spawn as written and cannot match itself. +//! +//! The comment rule is deliberately simple: a line whose trimmed start is +//! `//` is a comment and is skipped; every other line is code. + +use std::fs; +use std::path::{Path, PathBuf}; + +/// Suites migrated to the hermetic builder whose spawn sites this scan pins. +const HERMETIC_SUITES: &[&str] = &[ + "auto_chain_flag_e2e.rs", + "yes_ship_config.rs", + "release_check.rs", +]; + +/// `crates/devflow-cli/tests`. +fn tests_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests") +} + +/// The lines of `source` whose trimmed start is not `//`. +fn code_lines(source: &str) -> Vec<&str> { + source + .lines() + .filter(|line| !line.trim_start().starts_with("//")) + .collect() +} + +/// Cargo's env var naming the compiled `devflow` binary, assembled so this +/// file never spells it. +fn binary_env_name() -> &'static str { + concat!("CARGO_BIN_EXE_", "devflow") +} + +/// Occurrences, in the code lines of `source`, of a raw `devflow` spawn +/// (`Command::new` over the local or the shared `devflow_bin()`) or of the +/// binary's env name. +fn raw_spawn_sites(source: &str) -> usize { + let tokens = [ + concat!("Command::new(", "devflow_bin())"), + concat!("Command::new(common::", "devflow_bin())"), + binary_env_name(), + ]; + code_lines(source) + .iter() + .map(|line| { + tokens + .iter() + .map(|token| line.matches(token).count()) + .sum::() + }) + .sum() +} + +/// The source of `tests/`. Panics naming the path when it cannot be +/// read, so a missing or renamed suite fails rather than reading as clean. +fn read_suite(name: &str) -> String { + let path = tests_dir().join(name); + fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("cannot read suite {}: {err}", path.display())) +} + +#[test] +fn migrated_suites_spawn_devflow_only_through_the_hermetic_builder() { + // Planted controls first, on inline strings: the matcher must flag each + // raw form and pass a comment and the hermetic builder. + let raw_spawn = concat!("let out = Command::new(", "devflow_bin()).output();"); + assert_eq!( + raw_spawn_sites(raw_spawn), + 1, + "a raw local spawn must count" + ); + let shared_spawn = concat!( + "let out = Command::new(common::", + "devflow_bin()).output();" + ); + assert_eq!( + raw_spawn_sites(shared_spawn), + 1, + "a raw spawn over the shared devflow_bin() must count" + ); + let env_line = format!("let bin = env!(\"{}\");", binary_env_name()); + assert_eq!( + raw_spawn_sites(&env_line), + 1, + "a code line naming the binary's env var must count" + ); + let commented = format!("// {raw_spawn}"); + assert_eq!( + raw_spawn_sites(&commented), + 0, + "a commented-out spawn must not count" + ); + let hermetic = "let out = common::hermetic_devflow(&home).output();"; + assert_eq!( + raw_spawn_sites(hermetic), + 0, + "the hermetic builder must not count" + ); + + // Then every listed suite, read from disk. + let mut failures = Vec::new(); + for name in HERMETIC_SUITES { + let source = read_suite(name); + let raw = raw_spawn_sites(&source); + if raw != 0 { + failures.push(format!( + "{name}: {raw} raw devflow spawn site(s) or binary env name(s); expected 0" + )); + } + // Real-data control: the scan read the migrated file, not an empty + // or unrelated string. + let uses_builder = code_lines(&source) + .iter() + .any(|line| line.contains("common::hermetic_devflow(")); + if !uses_builder { + failures.push(format!( + "{name}: no `common::hermetic_devflow(` call in its code lines" + )); + } + } + assert!( + failures.is_empty(), + "suites not spawning devflow only through the hermetic builder:\n{}", + failures.join("\n") + ); +} diff --git a/crates/devflow-cli/tests/release_check.rs b/crates/devflow-cli/tests/release_check.rs index 900a6a8c..a48ab921 100644 --- a/crates/devflow-cli/tests/release_check.rs +++ b/crates/devflow-cli/tests/release_check.rs @@ -4,27 +4,25 @@ //! `pub(crate)` inside `devflow-cli`, and driving the actual CLI is what //! proves the `--check` gate and the self-pin comparison end-to-end). -use std::path::Path; -use std::process::{Command, Output}; +mod common; -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} +use std::path::Path; +use std::process::Output; /// Runs `devflow release ` with an ISOLATED `HOME` (a fresh -/// empty directory, no `.gitconfig`) and no inherited `SSH_AUTH_SOCK`/ +/// empty directory, no `.gitconfig`, set by `common::hermetic_devflow` along +/// with the XDG and DevFlow cache dirs) and no inherited `SSH_AUTH_SOCK`/ /// `SSH_AGENT_PID` — so the release check reads git config through the /// fixture's own repo-local config, never the OPERATOR's global /// `~/.gitconfig`. Without this isolation, these tests would be /// non-deterministic on any machine whose global config sets git signing /// or other release-relevant values. fn run_release(project: &Path, args: &[&str]) -> Output { - let isolated_home = tempfile::tempdir().unwrap(); - Command::new(devflow_bin()) + let isolated_home = common::HermeticHome::new(); + common::hermetic_devflow(&isolated_home) .arg("release") .args(args) .arg(project) - .env("HOME", isolated_home.path()) .env_remove("SSH_AUTH_SOCK") .env_remove("SSH_AGENT_PID") .output() diff --git a/crates/devflow-cli/tests/yes_ship_config.rs b/crates/devflow-cli/tests/yes_ship_config.rs index b9c5c3fb..fab07731 100644 --- a/crates/devflow-cli/tests/yes_ship_config.rs +++ b/crates/devflow-cli/tests/yes_ship_config.rs @@ -8,15 +8,15 @@ //! by `devflow-core`'s `config::tests::env_overrides_file_yes_ship`, and the //! binary here runs as a separate process so this suite stays out of the //! `ENV_MUTEX`-guarded unit-test convention. +//! +//! Every `devflow` child is built by `common::hermetic_devflow`, so it never +//! reads or writes the operator's real `~/.cache/devflow` (999.141). + +mod common; use devflow_core::phase_id::PhaseId; use std::fs; use std::path::Path; -use std::process::Command; - -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} fn git(root: &Path, args: &[&str]) { let output = devflow_core::test_support::git_command(root) @@ -66,7 +66,9 @@ fn run_dry_run(root: &Path, phase: PhaseId, extra_args: &[&str]) -> String { ]; args.extend(extra_args.iter().map(|s| s.to_string())); - let output = Command::new(devflow_bin()) + // `--dry-run` launches nothing, so the home only has to outlive `output()`. + let home = common::HermeticHome::new(); + let output = common::hermetic_devflow(&home) .args(&args) .arg(root) .current_dir(root) From 820956c4efb62a41a3328aa169094b1d60d9f242 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:17:52 -0400 Subject: [PATCH 021/112] test(50-21): finish hermetic e2e migration and pin binary naming - reap_strays_e2e and help_snapshot spawn devflow only through common::hermetic_devflow; reap_strays_e2e still runs no real stray reap - hermetic_spawn_sites.rs covers all five suites and adds only_common_names_the_devflow_binary: a recursive walk of tests/ that allows only common/mod.rs (and recover_clean_e2e.rs) to name the binary - help snapshot unchanged Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/tests/help_snapshot.rs | 13 +-- .../devflow-cli/tests/hermetic_spawn_sites.rs | 96 +++++++++++++++++++ crates/devflow-cli/tests/reap_strays_e2e.rs | 12 ++- 3 files changed, 110 insertions(+), 11 deletions(-) diff --git a/crates/devflow-cli/tests/help_snapshot.rs b/crates/devflow-cli/tests/help_snapshot.rs index 72722239..f1258e40 100644 --- a/crates/devflow-cli/tests/help_snapshot.rs +++ b/crates/devflow-cli/tests/help_snapshot.rs @@ -5,12 +5,12 @@ //! ```bash //! cargo run -q -p devflow -- --help > crates/devflow-cli/tests/snapshots/devflow-help.txt //! ``` +//! +//! The child is built by `common::hermetic_devflow`, so it never reads the +//! operator's real config or cache (999.141); the help text does not depend +//! on either. -use std::process::Command; - -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} +mod common; /// Normalize trailing whitespace per line so editor/clap cosmetic drift /// doesn't produce false diffs. @@ -25,7 +25,8 @@ fn normalized(text: &str) -> String { #[test] fn help_output_matches_committed_snapshot() { - let output = Command::new(devflow_bin()) + let home = common::HermeticHome::new(); + let output = common::hermetic_devflow(&home) .arg("--help") .output() .expect("run devflow --help"); diff --git a/crates/devflow-cli/tests/hermetic_spawn_sites.rs b/crates/devflow-cli/tests/hermetic_spawn_sites.rs index 9d5af90b..08f08233 100644 --- a/crates/devflow-cli/tests/hermetic_spawn_sites.rs +++ b/crates/devflow-cli/tests/hermetic_spawn_sites.rs @@ -19,8 +19,16 @@ const HERMETIC_SUITES: &[&str] = &[ "auto_chain_flag_e2e.rs", "yes_ship_config.rs", "release_check.rs", + "reap_strays_e2e.rs", + "help_snapshot.rs", ]; +/// The only files under `tests/` (paths relative to it, `/`-separated) whose +/// code may name the binary's env var. `common/mod.rs` is the one legitimate +/// site. `recover_clean_e2e.rs` is 50-04's migration and is tolerated so the +/// scan holds whichever of 50-04 and 50-21 lands first. +const BINARY_NAMERS: &[&str] = &["common/mod.rs", "recover_clean_e2e.rs"]; + /// `crates/devflow-cli/tests`. fn tests_dir() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("tests") @@ -68,6 +76,42 @@ fn read_suite(name: &str) -> String { .unwrap_or_else(|err| panic!("cannot read suite {}: {err}", path.display())) } +/// Whether any code line of `source` names the binary's env var. +fn names_binary(source: &str) -> bool { + code_lines(source) + .iter() + .any(|line| line.contains(binary_env_name())) +} + +/// Every `.rs` file below `dir`, recursively, as a path relative to `base` +/// joined with `/`. Panics on an unreadable directory or entry, so a walk +/// that cannot see a file fails instead of reading as clean. +fn rust_files_under(base: &Path, dir: &Path, files: &mut Vec<(String, PathBuf)>) { + let entries = fs::read_dir(dir) + .unwrap_or_else(|err| panic!("cannot read directory {}: {err}", dir.display())); + for entry in entries { + let entry = + entry.unwrap_or_else(|err| panic!("cannot read an entry of {}: {err}", dir.display())); + let path = entry.path(); + // `file_type` does not follow symlinks, so a link loop cannot recurse. + let file_type = entry + .file_type() + .unwrap_or_else(|err| panic!("cannot stat {}: {err}", path.display())); + if file_type.is_dir() { + rust_files_under(base, &path, files); + } else if path.extension().is_some_and(|ext| ext == "rs") { + let relative = path + .strip_prefix(base) + .expect("walked path is under the walk base") + .components() + .map(|part| part.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"); + files.push((relative, path)); + } + } +} + #[test] fn migrated_suites_spawn_devflow_only_through_the_hermetic_builder() { // Planted controls first, on inline strings: the matcher must flag each @@ -133,3 +177,55 @@ fn migrated_suites_spawn_devflow_only_through_the_hermetic_builder() { failures.join("\n") ); } + +#[test] +fn only_common_names_the_devflow_binary() { + // Planted controls first, on inline strings run through the same matcher. + let naming = format!(" let bin = env!(\"{}\");", binary_env_name()); + assert!( + names_binary(&naming), + "a code line naming the binary's env var must be detected" + ); + let commented = format!("// {naming}"); + assert!( + !names_binary(&commented), + "the same line behind `// ` must not be detected" + ); + + let base = tests_dir(); + let mut files = Vec::new(); + rust_files_under(&base, &base, &mut files); + let mut namers: Vec = files + .into_iter() + .filter(|(_, path)| { + let source = fs::read_to_string(path) + .unwrap_or_else(|err| panic!("cannot read {}: {err}", path.display())); + names_binary(&source) + }) + .map(|(relative, _)| relative) + .collect(); + namers.sort(); + + // Real-data control: the walk and the matcher find the one legitimate + // site. + assert!( + namers.iter().any(|path| path == "common/mod.rs"), + "common/mod.rs was not found naming the binary under {}; found {namers:?}", + base.display() + ); + let unexpected: Vec<&String> = namers + .iter() + .filter(|path| !BINARY_NAMERS.contains(&path.as_str())) + .collect(); + assert!( + unexpected.is_empty(), + "files under {} name the devflow binary outside common/mod.rs; spawn it through \ + common::hermetic_devflow instead:\n{}", + base.display(), + unexpected + .iter() + .map(|path| path.as_str()) + .collect::>() + .join("\n") + ); +} diff --git a/crates/devflow-cli/tests/reap_strays_e2e.rs b/crates/devflow-cli/tests/reap_strays_e2e.rs index f2ac81de..286fbeb5 100644 --- a/crates/devflow-cli/tests/reap_strays_e2e.rs +++ b/crates/devflow-cli/tests/reap_strays_e2e.rs @@ -43,16 +43,17 @@ //! `gate_sweep_e2e.rs` document — the one CLI-level assertion this file //! does make (`devflow stop` reporting the deleted root as unreachable) is //! driven via the real compiled binary, exactly like those files. +//! +//! That one `devflow` child is built by `common::hermetic_devflow`, so it +//! never reads or writes the operator's real `~/.cache/devflow` (999.141). + +mod common; use devflow_core::agent; use std::path::Path; use std::process::Command; use std::time::{Duration, Instant}; -fn devflow_bin() -> &'static str { - env!("CARGO_BIN_EXE_devflow") -} - /// Poll `predicate` until it's true, panicking with `what` if `timeout_secs` /// elapses first — mirrors `stop_e2e.rs`'s `wait_for`. fn wait_for(mut predicate: impl FnMut() -> bool, timeout_secs: u64, what: &str) { @@ -125,6 +126,7 @@ fn spawn_monitor_wrapper_fixture(cwd: &Path) -> std::process::Child { /// discovery) and clear it with verified death. #[test] fn reap_clears_a_process_whose_root_was_deleted_which_devflow_stop_cannot_see() { + let home = common::HermeticHome::new(); let (outer, root) = nested_root_fixture(); let mut child = spawn_monitor_wrapper_fixture(&root); let pid = child.id(); @@ -159,7 +161,7 @@ fn reap_clears_a_process_whose_root_was_deleted_which_devflow_stop_cannot_see() // deleted path, so it reports the process as unreachable -- neither // clearing it nor pretending the machine is otherwise clean by // signalling something else. - let stop_output = Command::new(devflow_bin()) + let stop_output = common::hermetic_devflow(&home) .args(["stop", "--phase", "900", "--root"]) .arg(&root) .output() From 71201d98a2b8b84dbf8e454a97eff54e2efe9ef6 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:33:14 -0400 Subject: [PATCH 022/112] test(50-07): add failing tracer tests for the supervisor runtime - Child-process supervisor: socket shutdown must kill the agent group and report Stopped 143; control: an agent exiting 7 completes with its code - Source scan for process exit / signal mask / signal wait calls with a planted-positive control - Runtime types, pure decision fns and a ping-only control thread; shutdown, stop-aware wait_agent and the scanner are stubs - Drop the dead_code expectations the runtime now satisfies Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor.rs | 10 +- .../devflow-core/src/supervisor/protocol.rs | 31 - crates/devflow-core/src/supervisor/runtime.rs | 859 ++++++++++++++++++ 3 files changed, 861 insertions(+), 39 deletions(-) create mode 100644 crates/devflow-core/src/supervisor/runtime.rs diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs index 1b3a6031..e2dbe27e 100644 --- a/crates/devflow-core/src/supervisor.rs +++ b/crates/devflow-core/src/supervisor.rs @@ -22,6 +22,7 @@ //! a stray file can never be mistaken for a dead supervisor (R-6). pub mod protocol; +pub mod runtime; use crate::phase_id::PhaseId; use crate::registry; @@ -407,7 +408,7 @@ pub(crate) fn rfc3339_from_unix(unix_secs: u64) -> String { not(test), expect( dead_code, - reason = "the launcher (50-07) is the first non-test writer of a sidecar" + reason = "the launcher (50-08) is the first non-test writer of a sidecar timestamp" ) )] pub(crate) fn rfc3339_now() -> String { @@ -423,13 +424,6 @@ pub(crate) fn rfc3339_now() -> String { /// is safe here; only sockets must never be renamed into place. The socket /// directory is created `0700` if absent and refused if it is unsafe, as /// [`publish`] does. -#[cfg_attr( - not(test), - expect( - dead_code, - reason = "the launcher (50-07) is the first non-test writer of a sidecar" - ) -)] pub(crate) fn write_sidecar(socket: &SocketPath, sidecar: &Sidecar) -> io::Result<()> { use std::io::Write; use std::os::unix::fs::OpenOptionsExt; diff --git a/crates/devflow-core/src/supervisor/protocol.rs b/crates/devflow-core/src/supervisor/protocol.rs index e2214d4f..2af20252 100644 --- a/crates/devflow-core/src/supervisor/protocol.rs +++ b/crates/devflow-core/src/supervisor/protocol.rs @@ -89,13 +89,6 @@ pub struct ShutdownStopped { /// Every reply a server writes, one variant per wire shape. #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr( - not(test), - expect( - dead_code, - reason = "server side of the protocol; the supervisor runtime (50-07) is its first non-test caller" - ) -)] pub(crate) enum Reply { /// `{"ok":true, ...}` answering a ping. Ping(PingReply), @@ -108,10 +101,6 @@ pub(crate) enum Reply { } impl Reply { - #[cfg_attr( - not(test), - expect(dead_code, reason = "used only by write_reply; see Reply") - )] fn to_json(&self) -> serde_json::Result { let mut object = match self { Self::Ping(reply) => serde_json::to_value(reply)?, @@ -171,10 +160,6 @@ fn set_timeouts(stream: &UnixStream) -> io::Result<()> { /// Server side: read the connection's single request. A request that is not /// valid JSON or names an unknown op is `InvalidData`; the caller answers it /// with `Reply::Error("unknown_op")`. -#[cfg_attr( - not(test), - expect(dead_code, reason = "server side of the protocol; see Reply") -)] pub(crate) fn read_request(stream: &UnixStream) -> io::Result { set_timeouts(stream)?; let line = read_line(&mut BufReader::new(stream))?; @@ -182,10 +167,6 @@ pub(crate) fn read_request(stream: &UnixStream) -> io::Result { } /// Server side: write one reply line. -#[cfg_attr( - not(test), - expect(dead_code, reason = "server side of the protocol; see Reply") -)] pub(crate) fn write_reply(mut stream: &UnixStream, reply: &Reply) -> io::Result<()> { set_timeouts(stream)?; let mut line = serde_json::to_string(&reply.to_json()?)?; @@ -211,10 +192,6 @@ pub(crate) fn euid() -> u32 { /// A failed read returns false, and so does every other target, so the uid /// check is never skipped (T-50-10, T-50-87). #[cfg(target_os = "linux")] -#[cfg_attr( - not(test), - expect(dead_code, reason = "server side of the protocol; see Reply") -)] pub(crate) fn peer_is_same_user(stream: &UnixStream) -> bool { use std::os::fd::AsRawFd; let mut cred = libc::ucred { @@ -242,10 +219,6 @@ pub(crate) fn peer_is_same_user(stream: &UnixStream) -> bool { /// uid, read with `getpeereid`. Compile-checked only: no macOS host has run it /// (backlog 999.160). #[cfg(target_os = "macos")] -#[cfg_attr( - not(test), - expect(dead_code, reason = "server side of the protocol; see Reply") -)] pub(crate) fn peer_is_same_user(stream: &UnixStream) -> bool { use std::os::fd::AsRawFd; let mut uid: libc::uid_t = 0; @@ -260,10 +233,6 @@ pub(crate) fn peer_is_same_user(stream: &UnixStream) -> bool { /// fail closed. A client then reads no reply (`NoReply`), never `Stale`, so /// the backstop cannot arm on this target. #[cfg(not(any(target_os = "linux", target_os = "macos")))] -#[cfg_attr( - not(test), - expect(dead_code, reason = "server side of the protocol; see Reply") -)] pub(crate) fn peer_is_same_user(_stream: &UnixStream) -> bool { false } diff --git a/crates/devflow-core/src/supervisor/runtime.rs b/crates/devflow-core/src/supervisor/runtime.rs new file mode 100644 index 00000000..c123bba1 --- /dev/null +++ b/crates/devflow-core/src/supervisor/runtime.rs @@ -0,0 +1,859 @@ +//! The supervisor's process body: one terminal-state owner, one control +//! thread, and the decision that a stop is never a completion (R-2). +//! +//! **Threads.** Two participants, plus a signal handler that does no work: +//! - the **main thread** spawns the agent, calls [`Supervisor::serve`], then +//! blocks in [`Supervisor::wait_agent`]. It observes the agent's exit with +//! `waitid(WNOWAIT)`, which does not reap, so the zombie leader still holds +//! the agent's process group while leftover members are killed. Only then +//! does it reap. +//! - one **control thread**, started by [`Supervisor::serve`], `poll`s the +//! listening socket and, when [`install_signal_pipe`] has run, the signal +//! pipe's read end. It answers `ping`, turns a `shutdown` request or a +//! signal byte into [`StopToken::request_stop`], and acts on the returned +//! [`StopAction`]. +//! - the SIGTERM/SIGINT **handler** only compares `getpid()` with the +//! supervisor's pid and `write`s one byte to the pipe; both calls are +//! async-signal-safe. The supervisor never blocks a signal: a blocked mask +//! is inherited by every agent it spawns (RESEARCH Finding 3, probe A). +//! +//! **The one invariant.** Every stop and exit decision is taken under the +//! [`StopToken`]'s owner mutex: a stop request, the main thread's "the agent +//! exited, do we advance?" decision, and every advance safe point. A stop and +//! the agent's natural exit are therefore totally ordered, and a stop that +//! wins the race always ends the run as stopped. The group signals the +//! control thread sends are sent under the same mutex, and only while the +//! exit has not been observed, so no group signal is sent after the leader +//! is reaped. + +use super::protocol::{self, PingReply, Reply, Request, ShutdownAccepted, ShutdownStopped}; +use super::{PublishError, Sidecar, SocketPath}; +use std::io; +use std::os::fd::AsRawFd; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError}; +use std::thread::JoinHandle; + +/// The exit code a stopped run reports: the shell convention for a death by +/// SIGTERM, `128 + 15`. +const STOPPED_EXIT_CODE: i32 = 128 + libc::SIGTERM; + +/// Who owns the run right now. Its [`as_str`](Self::as_str) label is the +/// `owner` field of a ping reply. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum OwnerPhase { + /// The agent is running under this supervisor. + AgentRunning, + /// The agent completed and the CLI is advancing the pipeline. + Advancing, + /// The CLI is waiting on a gate answer. + GateWait, + /// The run was handed to the next stage's supervisor. + HandedOff, + /// The run is over. + Finished, +} + +impl OwnerPhase { + /// The protocol's `owner` label. + pub fn as_str(self) -> &'static str { + match self { + Self::AgentRunning => "agent_running", + Self::Advancing => "advancing", + Self::GateWait => "gate_wait", + Self::HandedOff => "handed_off", + Self::Finished => "finished", + } + } +} + +/// Where a stop request came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum StopSource { + /// A `shutdown` request on the socket. + Socket, + /// A signal delivered to the supervisor, by number. + Signal(i32), +} + +/// How the agent's run ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum AgentOutcome { + /// The agent exited on its own with this code (`128 + n` for a death by + /// signal `n`), and no stop was requested first. Only this outcome may + /// advance the pipeline. + Completed(i32), + /// A stop was requested before the exit decision. Never advances. + Stopped, +} + +/// What a stop request does, decided by [`on_stop_request`] from the owner +/// phase at the moment the request is taken. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum StopAction { + /// The agent is running: end its process group. + KillAgentGroup, + /// The CLI owns the run: the stop is flagged and its next safe point + /// stops. + FlagForSafePoint, + /// The run was handed off: reply `handed_off` and act on nothing. + ReplyHandedOff, + /// The run is already over: act on nothing. + AlreadyFinished, +} + +/// What an advance safe point does. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum SafePoint { + /// No stop is requested: carry on. + Proceed, + /// A stop is requested: persist the stop and go no further. + StopHere, +} + +/// The decision for a stop request taken in `phase`. +pub fn on_stop_request(phase: OwnerPhase) -> StopAction { + match phase { + OwnerPhase::AgentRunning => StopAction::KillAgentGroup, + OwnerPhase::Advancing | OwnerPhase::GateWait => StopAction::FlagForSafePoint, + OwnerPhase::HandedOff => StopAction::ReplyHandedOff, + OwnerPhase::Finished => StopAction::AlreadyFinished, + } +} + +/// The decision when the agent exits with `code`: a requested stop always +/// wins over the exit code. +pub fn on_agent_exit(stop_requested: bool, code: i32) -> AgentOutcome { + if stop_requested { + AgentOutcome::Stopped + } else { + AgentOutcome::Completed(code) + } +} + +/// The decision at an advance safe point. +pub fn at_safe_point(stop_requested: bool) -> SafePoint { + if stop_requested { + SafePoint::StopHere + } else { + SafePoint::Proceed + } +} + +/// The state the owner mutex guards. +#[derive(Debug)] +struct OwnerState { + phase: OwnerPhase, + stop_requested: bool, + /// The first stop's source, for the log. + stop_source: Option, + /// The main thread has observed the agent's exit. Set under the lock + /// before the leader is reaped, so a group signal sent under the lock + /// while this is false always reaches an unreaped group. + agent_exited: bool, + /// The run ended as a stop: [`AgentOutcome::Stopped`] or a safe point + /// that returned [`SafePoint::StopHere`]. + ended_stopped: bool, +} + +/// The owner mutex and the condition variable the control thread waits on +/// for the agent's exit. +#[derive(Debug)] +struct Owner { + state: Mutex, + exited: Condvar, +} + +/// A handle to the run's single terminal-state owner. Cloning shares the +/// owner; every method takes the owner mutex, so their decisions are totally +/// ordered with each other and with the supervisor's. +#[derive(Debug, Clone)] +pub struct StopToken { + owner: Arc, +} + +impl StopToken { + fn new(phase: OwnerPhase) -> Self { + Self { + owner: Arc::new(Owner { + state: Mutex::new(OwnerState { + phase, + stop_requested: false, + stop_source: None, + agent_exited: false, + ended_stopped: false, + }), + exited: Condvar::new(), + }), + } + } + + fn lock(&self) -> MutexGuard<'_, OwnerState> { + self.owner + .state + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + /// Whether a stop has been requested. + pub fn stop_requested(&self) -> bool { + self.lock().stop_requested + } + + /// The current owner phase. + pub fn phase(&self) -> OwnerPhase { + self.lock().phase + } + + /// Move the run to `phase`, e.g. `GateWait` around a gate poll. + pub fn enter(&self, phase: OwnerPhase) { + self.lock().phase = phase; + } + + /// Take a stop request and return what it does. A request in + /// `AgentRunning`, `Advancing` or `GateWait` sets the stop flag; one in + /// `HandedOff` or `Finished` changes nothing. This only decides: the + /// caller acts on the returned [`StopAction`] (the supervisor's control + /// thread ends the agent group for [`StopAction::KillAgentGroup`]). + pub fn request_stop(&self, source: StopSource) -> StopAction { + request_stop_locked(&mut self.lock(), source) + } + + /// An advance safe point: [`SafePoint::StopHere`] when a stop has been + /// requested, which also records that the run ended as a stop. + pub fn safe_point(&self) -> SafePoint { + let mut state = self.lock(); + let decision = at_safe_point(state.stop_requested); + if decision == SafePoint::StopHere { + state.ended_stopped = true; + } + decision + } + + /// Whether the run ended as a stop, so a parked shutdown caller may be + /// told `stopped`. + fn ended_stopped(&self) -> bool { + self.lock().ended_stopped + } + + /// The main thread's exit decision. Under the owner lock: mark the exit + /// observed (waking the control thread), run `reap` with the stop flag, + /// decide with [`on_agent_exit`], and leave the phase at `Advancing` for + /// a completion or `Finished` for a stop. + fn observe_exit(&self, reap: impl FnOnce(bool) -> io::Result) -> io::Result { + let mut state = self.lock(); + state.agent_exited = true; + self.owner.exited.notify_all(); + let code = reap(state.stop_requested)?; + let outcome = on_agent_exit(state.stop_requested, code); + match outcome { + AgentOutcome::Completed(_) => state.phase = OwnerPhase::Advancing, + AgentOutcome::Stopped => { + state.phase = OwnerPhase::Finished; + state.ended_stopped = true; + } + } + Ok(outcome) + } +} + +/// [`StopToken::request_stop`] on an already locked owner, so the control +/// thread can act on the decision under the same lock. +fn request_stop_locked(state: &mut OwnerState, source: StopSource) -> StopAction { + let action = on_stop_request(state.phase); + if matches!( + action, + StopAction::KillAgentGroup | StopAction::FlagForSafePoint + ) { + state.stop_requested = true; + state.stop_source.get_or_insert(source); + } + action +} + +/// Signal the agent's process group. The one place a group signal is sent: +/// it refuses a pgid that does not fit `pid_t`, is `0` or `1`, or is this +/// process's own group, because `kill(0, …)` signals our own group and +/// `kill(-1, …)` signals every process we may signal (T-50-18). +fn signal_group(pgid: u32, signal: libc::c_int) -> bool { + let Ok(signed) = libc::pid_t::try_from(pgid) else { + tracing::warn!("agent pgid {pgid} does not fit pid_t; not signalling"); + return false; + }; + if signed <= 1 { + tracing::warn!("refusing to signal process group {signed}"); + return false; + } + // SAFETY: getpgrp has no preconditions and cannot fail. + if signed == unsafe { libc::getpgrp() } { + tracing::warn!("refusing to signal the supervisor's own process group {signed}"); + return false; + } + // SAFETY: `signed > 1`, so `-signed < -1`: neither our own group (`0`) + // nor every process (`-1`). + unsafe { libc::kill(-signed, signal) == 0 } +} + +/// An exit status as a code: the exit code, or `128 + n` for a death by +/// signal `n` (the `monitor` convention: SIGTERM is 143, SIGKILL 137). +fn exit_code(status: std::process::ExitStatus) -> i32 { + status.code().unwrap_or_else(|| { + use std::os::unix::process::ExitStatusExt; + status.signal().map_or(-1, |signal| 128 + signal) + }) +} + +/// A running supervisor: its published socket, its sidecar and its control +/// thread. Dropping it without [`finish`](Self::finish) still stops the +/// thread and removes the launch files. +#[derive(Debug)] +pub struct Supervisor { + socket: SocketPath, + agent_pgid: u32, + token: StopToken, + quit: Arc, + control: Option>>, + closed: bool, +} + +impl Supervisor { + /// Write `sidecar`, publish `socket` (so the sidecar exists before the + /// socket name appears, R-7), and start the control thread. The run + /// starts in [`OwnerPhase::AgentRunning`]: call this after the agent is + /// spawned, with the agent's identity in `sidecar`. + pub fn serve(socket: SocketPath, sidecar: Sidecar) -> Result { + super::write_sidecar(&socket, &sidecar)?; + let listener = match super::publish(&socket) { + Ok(listener) => listener, + // Another launch's live socket holds the name: its sidecar is + // not ours to remove. + Err(PublishError::AlreadyExists(path)) => { + return Err(PublishError::AlreadyExists(path)); + } + Err(error) => { + let _ = std::fs::remove_file(socket.sidecar_path()); + return Err(error); + } + }; + let started = listener.set_nonblocking(true).and_then(|()| { + let token = StopToken::new(OwnerPhase::AgentRunning); + let quit = Arc::new(AtomicBool::new(false)); + let control = Control { + listener, + sidecar: sidecar.clone(), + token: token.clone(), + quit: Arc::clone(&quit), + }; + let handle = std::thread::Builder::new() + .name("supervisor-control".to_string()) + .spawn(move || control.run())?; + Ok(Self { + socket: socket.clone(), + agent_pgid: sidecar.agent_pgid, + token, + quit, + control: Some(handle), + closed: false, + }) + }); + started.map_err(|error| { + let _ = super::remove_launch_files(&socket); + PublishError::Io(error) + }) + } + + /// Block until `child` (the agent this supervisor serves) exits, decide + /// under the owner lock whether the run completed or was stopped, and + /// reap it. On [`AgentOutcome::Stopped`] the group's leftover members + /// are SIGKILLed first, while the unreaped leader still holds the pgid + /// (RESEARCH Pitfall 9). + pub fn wait_agent(&self, child: &mut std::process::Child) -> io::Result { + // RED stub: no stop handling yet. + let status = child.wait()?; + self.token.observe_exit(|_| Ok(exit_code(status))) + } + + /// A handle to this run's owner, for the CLI's safe points. + pub fn token(&self) -> StopToken { + self.token.clone() + } + + /// Mark the run handed off to the next stage's supervisor: a later stop + /// request is answered `handed_off` and acts on nothing. + pub fn hand_off(&self) { + self.token.enter(OwnerPhase::HandedOff); + } + + /// End the supervisor: stop the control thread, remove the socket and + /// sidecar, then send each parked shutdown caller its final line. That + /// line is `stopped` only when the run ended as a stop + /// ([`AgentOutcome::Stopped`] or a safe point that stopped), otherwise + /// `{"ok":false,"error":"handed_off"}`, so no caller is told a stop + /// happened that did not. The files go before the line, so a caller that + /// reads `stopped` finds the socket already gone. + pub fn finish(mut self) { + let parked = self.close(); + let reply = if self.token.ended_stopped() { + Reply::Stopped(ShutdownStopped { + exit_code: STOPPED_EXIT_CODE, + advance_suppressed: true, + }) + } else { + Reply::Error("handed_off".to_string()) + }; + self.token.enter(OwnerPhase::Finished); + for stream in parked { + if let Err(error) = protocol::write_reply(&stream, &reply) { + tracing::warn!("could not send the final shutdown line: {error}"); + } + } + } + + /// Stop and join the control thread and remove the launch files, once. + /// Returns the parked shutdown connections. + fn close(&mut self) -> Vec { + if self.closed { + return Vec::new(); + } + self.closed = true; + self.quit.store(true, Ordering::Release); + let parked = self + .control + .take() + .and_then(|handle| handle.join().ok()) + .unwrap_or_default(); + if let Err(error) = super::remove_launch_files(&self.socket) { + tracing::warn!( + "could not remove the launch files for {}: {error}", + self.socket.as_path().display() + ); + } + parked + } +} + +impl Drop for Supervisor { + fn drop(&mut self) { + // A parked caller is dropped without a final line: it reads a closed + // connection, which is an error, never a stop. + drop(self.close()); + } +} + +/// The control thread's state. +struct Control { + listener: UnixListener, + sidecar: Sidecar, + token: StopToken, + quit: Arc, +} + +impl Control { + /// Poll the listener until told to quit; return the parked shutdown + /// connections. + fn run(self) -> Vec { + let mut parked = Vec::new(); + let tick = libc::c_int::try_from(crate::agent::TERMINATE_VERIFY_POLL.as_millis()) + .unwrap_or(libc::c_int::MAX); + while !self.quit.load(Ordering::Acquire) { + let mut fds = [libc::pollfd { + fd: self.listener.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }]; + // SAFETY: `fds` is a live array of initialized pollfd entries and + // its length is passed alongside it. + let ready = unsafe { libc::poll(fds.as_mut_ptr(), fds.len() as libc::nfds_t, tick) }; + if ready <= 0 { + continue; + } + if fds[0].revents & libc::POLLIN != 0 { + self.accept_all(&mut parked); + } + } + parked + } + + /// Serve every pending connection. + fn accept_all(&self, parked: &mut Vec) { + loop { + match self.listener.accept() { + Ok((stream, _)) => self.serve_connection(stream, parked), + Err(error) if error.kind() == io::ErrorKind::WouldBlock => return, + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => { + tracing::warn!("supervisor accept failed: {error}"); + return; + } + } + } + } + + /// Answer one connection's single request. + fn serve_connection(&self, stream: UnixStream, parked: &mut Vec) { + // An accepted socket may inherit the listener's non-blocking flag. + if stream.set_nonblocking(false).is_err() || !protocol::peer_is_same_user(&stream) { + return; + } + match protocol::read_request(&stream) { + Ok(Request::Ping) => { + let reply = Reply::Ping(self.ping_reply()); + if let Err(error) = protocol::write_reply(&stream, &reply) { + tracing::warn!("could not answer a ping: {error}"); + } + } + Ok(Request::Shutdown) => { + // RED stub: shutdown is not implemented yet. + let _ = parked; + let _ = protocol::write_reply(&stream, &Reply::Error("not_implemented".into())); + } + Err(error) if error.kind() == io::ErrorKind::InvalidData => { + let _ = protocol::write_reply(&stream, &Reply::Error("unknown_op".to_string())); + } + Err(_) => {} + } + } + + /// The ping reply: the sidecar's identity plus the current owner phase. + fn ping_reply(&self) -> PingReply { + let sidecar = &self.sidecar; + PingReply { + launch_id: sidecar.launch_id.clone(), + phase: sidecar.phase.to_string(), + stage: sidecar.stage.clone().unwrap_or_default(), + launch: sidecar.launch.clone(), + owner: self.token.phase().as_str().to_string(), + agent_pid: sidecar.agent_pid, + agent_pgid: sidecar.agent_pgid, + agent_start_time: sidecar.agent_start_time, + boot_id: (!sidecar.boot_id.is_empty()).then(|| sidecar.boot_id.clone()), + supervisor_pid: sidecar.supervisor_pid, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::phase_id::PhaseId; + use crate::supervisor::{Liveness, ShutdownOutcome, new_launch_id, probe, request_shutdown}; + use crate::test_support::{assert_child_ran_exactly_one_passing_test, spawn_test_in_child}; + use std::os::unix::process::CommandExt; + use std::path::{Path, PathBuf}; + use std::process::{Child, Command, Output, Stdio}; + use std::time::{Duration, Instant}; + + /// The socket path the child supervisor serves at. + const SOCKET_ENV: &str = "DEVFLOW_TEST_RUNTIME_SOCKET"; + /// Where the child supervisor writes its `AgentOutcome`. + const OUTCOME_ENV: &str = "DEVFLOW_TEST_RUNTIME_OUTCOME"; + /// The `sh -c` script the child supervisor runs as its agent. + const AGENT_ENV: &str = "DEVFLOW_TEST_RUNTIME_AGENT"; + + /// How long the parent waits for a child supervisor to answer a ping or + /// to exit. + const CHILD_WAIT: Duration = Duration::from_secs(30); + + /// A short cache dir under `/tmp`, so socket paths stay far below + /// `sun_path`. + fn short_cache() -> tempfile::TempDir { + tempfile::Builder::new() + .prefix("dfs") + .tempdir_in("/tmp") + .expect("create a short cache dir under /tmp") + } + + fn wait_until(within: Duration, mut done: impl FnMut() -> bool) -> bool { + let deadline = Instant::now() + within; + loop { + if done() { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(20)); + } + } + + /// Whether any member of process group `pgid` still exists (zombies + /// included). `EPERM` means a member exists that we may not signal. + fn group_exists(pgid: u32) -> bool { + let signed = libc::pid_t::try_from(pgid).expect("pgid fits pid_t"); + assert!(signed > 1, "refusing to probe process group {signed}"); + // SAFETY: `signed > 1`; signal 0 only checks for existence. + let rc = unsafe { libc::kill(-signed, 0) }; + rc == 0 || io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) + } + + /// Kills and reaps a child process on every exit path. + struct ChildGuard(Option); + + impl ChildGuard { + fn child(&mut self) -> &mut Child { + self.0.as_mut().expect("the child is still owned") + } + + /// Wait for the child to exit on its own, then collect its output. + fn wait_exit(&mut self) -> Output { + let exited = wait_until(CHILD_WAIT, || { + self.child().try_wait().expect("poll the child").is_some() + }); + let child = self.0.take().expect("the child is still owned"); + if !exited { + let mut child = ChildGuard(Some(child)); + let _ = child.child().kill(); + let output = child.0.take().expect("owned").wait_with_output(); + panic!("the child supervisor did not exit within {CHILD_WAIT:?}: {output:?}"); + } + child + .wait_with_output() + .expect("collect the child's output") + } + } + + impl Drop for ChildGuard { + fn drop(&mut self) { + if let Some(mut child) = self.0.take() { + // `Child::kill` is SIGKILL to the pid this test spawned and + // still owns, never a pid read from disk. + let _ = child.kill(); + let _ = child.wait(); + } + } + } + + /// SIGKILLs whatever is left of a process group on drop. + struct GroupGuard(u32); + + impl Drop for GroupGuard { + fn drop(&mut self) { + if group_exists(self.0) { + signal_group(self.0, libc::SIGKILL); + } + } + } + + /// A sidecar for an agent this test process spawned. + fn test_sidecar(socket: &SocketPath, agent_pid: u32) -> Sidecar { + let stem = socket + .as_path() + .file_stem() + .and_then(std::ffi::OsStr::to_str) + .expect("socket stem"); + Sidecar { + launch_id: stem.rsplit('-').next().expect("launch id").to_string(), + project_root: PathBuf::from("/project/runtime"), + phase: PhaseId::new(50), + stage: Some("code".to_string()), + agent: "test".to_string(), + launch: "capture".to_string(), + agent_pid, + agent_pgid: agent_pid, + agent_start_time: crate::agent::process_start_time(agent_pid).unwrap_or(0), + boot_id: crate::supervisor::boot_id().unwrap_or_default(), + supervisor_pid: std::process::id(), + supervisor_start_time: crate::agent::process_start_time(std::process::id()) + .unwrap_or(0), + created_at: crate::supervisor::rfc3339_now(), + } + } + + /// The child half shared by every runtime process test: spawn the agent script in its own process + /// group, serve, wait, finish, and record the outcome. + fn supervise_in_child() { + let socket = std::env::var_os(SOCKET_ENV).expect("the parent names the socket"); + let socket = SocketPath::new(PathBuf::from(socket)).expect("socket path fits"); + let outcome_path = std::env::var_os(OUTCOME_ENV).expect("the parent names the outcome"); + let script = std::env::var_os(AGENT_ENV).expect("the parent names the agent script"); + let mut agent = Command::new("/bin/sh") + .arg("-c") + .arg(script) + .env("PATH", "/usr/bin:/bin") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .process_group(0) + .spawn() + .expect("spawn the agent"); + let _group = GroupGuard(agent.id()); + let sidecar = test_sidecar(&socket, agent.id()); + let supervisor = Supervisor::serve(socket, sidecar).expect("serve the socket"); + let outcome = supervisor + .wait_agent(&mut agent) + .expect("wait for the agent"); + supervisor.finish(); + std::fs::write(outcome_path, format!("{outcome:?}")).expect("record the outcome"); + } + + /// A child supervisor the parent launched, once it answers a ping. + struct Launched { + child: ChildGuard, + group: GroupGuard, + socket: PathBuf, + outcome: PathBuf, + reply: PingReply, + } + + impl Launched { + fn outcome(&self) -> String { + std::fs::read_to_string(&self.outcome).expect("the child recorded its outcome") + } + } + + /// Re-run `test_name` as a child supervisor for `script` and wait until + /// its socket answers a ping. + fn launch(test_name: &str, cache: &Path, script: &str) -> Launched { + let socket = SocketPath::for_launch( + cache, + Path::new("/project/runtime"), + PhaseId::new(50), + &new_launch_id(), + ) + .expect("socket path fits sun_path"); + let outcome = cache.join("outcome"); + let mut child = ChildGuard(Some(spawn_test_in_child( + test_name, + cache, + &[ + (SOCKET_ENV, socket.as_path().as_os_str()), + (OUTCOME_ENV, outcome.as_os_str()), + (AGENT_ENV, std::ffi::OsStr::new(script)), + ], + ))); + let deadline = Instant::now() + CHILD_WAIT; + let reply = loop { + if let Ok(Liveness::Alive(reply)) = probe(socket.as_path()) { + break reply; + } + let exited = child.child().try_wait().expect("poll the child").is_some(); + if exited || Instant::now() >= deadline { + let output = child.0.take().expect("owned").wait_with_output(); + panic!("the child supervisor never answered a ping (exited: {exited}): {output:?}"); + } + std::thread::sleep(Duration::from_millis(20)); + }; + Launched { + group: GroupGuard(reply.agent_pgid), + child, + socket: socket.as_path().to_path_buf(), + outcome, + reply, + } + } + + const SHUTDOWN_TEST: &str = "supervisor::runtime::tests::shutdown_over_the_socket_kills_the_agent_group_and_reports_stopped"; + + #[test] + fn shutdown_over_the_socket_kills_the_agent_group_and_reports_stopped() { + if crate::test_support::in_child_test(SHUTDOWN_TEST) { + supervise_in_child(); + return; + } + let cache = short_cache(); + let mut run = launch(SHUTDOWN_TEST, cache.path(), "sleep 60 & sleep 60 & wait"); + assert_eq!(run.reply.owner, "agent_running"); + let pgid = run.reply.agent_pgid; + assert_eq!(pgid, run.reply.agent_pid, "the agent leads its own group"); + assert!( + group_exists(pgid), + "control: the agent group runs before the stop" + ); + + let answer = request_shutdown(&run.socket, CHILD_WAIT); + assert_eq!( + answer.as_ref().ok(), + Some(&ShutdownOutcome::Stopped { + exit_code: 143, + advance_suppressed: true + }), + "a socket shutdown must report the stop, got {answer:?}" + ); + let output = run.child.wait_exit(); + assert_child_ran_exactly_one_passing_test(&output, SHUTDOWN_TEST); + assert!( + wait_until(Duration::from_secs(5), || !group_exists(pgid)), + "every member of the agent group must be gone" + ); + assert!( + matches!(probe(&run.socket), Ok(Liveness::Gone)), + "the socket must be gone after the stop" + ); + assert_eq!(run.outcome(), "Stopped"); + drop(run.group); + } + + const COMPLETES_TEST: &str = + "supervisor::runtime::tests::an_agent_that_exits_on_its_own_completes_with_its_code"; + + #[test] + fn an_agent_that_exits_on_its_own_completes_with_its_code() { + if crate::test_support::in_child_test(COMPLETES_TEST) { + supervise_in_child(); + return; + } + let cache = short_cache(); + let go = cache.path().join("go"); + let script = format!( + "while [ ! -e '{}' ]; do sleep 0.05; done; exit 7", + go.display() + ); + let mut run = launch(COMPLETES_TEST, cache.path(), &script); + assert_eq!(run.reply.owner, "agent_running"); + std::fs::write(&go, "").expect("let the agent exit"); + let output = run.child.wait_exit(); + assert_child_ran_exactly_one_passing_test(&output, COMPLETES_TEST); + assert_eq!(run.outcome(), "Completed(7)"); + assert!( + matches!(probe(&run.socket), Ok(Liveness::Gone)), + "finish must remove the socket" + ); + } + + /// One `line N: ` entry per needle on each non-comment line of + /// `source` that holds one: a process exit or a signal-mask or + /// signal-wait call. + fn forbidden_calls(source: &str) -> Vec { + // RED stub. + let _ = source; + Vec::new() + } + + /// The needles, each joined from two fragments at run time so this test + /// module never holds one verbatim. + fn needles() -> [String; 4] { + [ + ["process", "::exit"].concat(), + ["sigproc", "mask"].concat(), + ["pthread_sig", "mask"].concat(), + ["sig", "wait"].concat(), + ] + } + + #[test] + fn runtime_source_has_none_of_the_forbidden_signal_or_exit_calls() { + assert_eq!( + forbidden_calls(include_str!("runtime.rs")), + Vec::::new(), + "runtime.rs must not exit the process, block a signal or wait for one" + ); + // Planted-positive control: each needle on a code line is flagged, + // and the same needle after `//` is not. + let planted: String = needles() + .iter() + .map(|needle| format!(" let _ = {needle}(0);\n // {needle}\n")) + .collect(); + let found = forbidden_calls(&planted); + assert_eq!(found.len(), 4, "every planted call is flagged: {found:?}"); + for (index, needle) in needles().iter().enumerate() { + let line = index * 2 + 1; + assert_eq!(found[index], format!("line {line}: {needle}")); + } + } +} From 12396cb7ae91b310f894ba897b4bc91f57c05d56 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:34:18 -0400 Subject: [PATCH 023/112] feat(50-07): stop the agent group on a socket shutdown and report it - Control thread takes shutdown under the owner lock: SIGTERM the agent group while the exit is unobserved, park the caller, SIGKILL after TERMINATE_VERIFY_WAIT if the main thread has not seen the exit - wait_agent observes the exit with waitid WNOWAIT, decides under the lock, SIGKILLs leftover members while the zombie leader holds the pgid, then reaps - forbidden_calls source scan implemented Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor/runtime.rs | 138 ++++++++++++++++-- 1 file changed, 127 insertions(+), 11 deletions(-) diff --git a/crates/devflow-core/src/supervisor/runtime.rs b/crates/devflow-core/src/supervisor/runtime.rs index c123bba1..74e68d29 100644 --- a/crates/devflow-core/src/supervisor/runtime.rs +++ b/crates/devflow-core/src/supervisor/runtime.rs @@ -300,6 +300,34 @@ fn signal_group(pgid: u32, signal: libc::c_int) -> bool { unsafe { libc::kill(-signed, signal) == 0 } } +/// Block until `pid` (a child of this process) has exited, without reaping +/// it: `waitid(P_PID, pid, WEXITED | WNOWAIT)`. The unreaped zombie keeps its +/// pid and process group, so a group signal sent afterwards still reaches +/// only that group. +fn wait_for_exit_without_reaping(pid: u32) -> io::Result<()> { + let id = libc::id_t::from(pid); + loop { + // SAFETY: an all-zero siginfo_t is a valid out-parameter for waitid. + let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; + // SAFETY: `info` is a live, correctly sized out-parameter. + let rc = unsafe { + libc::waitid( + libc::P_PID, + id, + &raw mut info, + libc::WEXITED | libc::WNOWAIT, + ) + }; + if rc == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } + } +} + /// An exit status as a code: the exit code, or `128 + n` for a death by /// signal `n` (the `monitor` convention: SIGTERM is 143, SIGKILL 137). fn exit_code(status: std::process::ExitStatus) -> i32 { @@ -374,9 +402,16 @@ impl Supervisor { /// are SIGKILLed first, while the unreaped leader still holds the pgid /// (RESEARCH Pitfall 9). pub fn wait_agent(&self, child: &mut std::process::Child) -> io::Result { - // RED stub: no stop handling yet. - let status = child.wait()?; - self.token.observe_exit(|_| Ok(exit_code(status))) + wait_for_exit_without_reaping(child.id())?; + let pgid = self.agent_pgid; + self.token.observe_exit(|stop_requested| { + if stop_requested { + // The zombie leader still holds the pgid, so this reaches + // only the agent's own leftover members. + signal_group(pgid, libc::SIGKILL); + } + child.wait().map(exit_code) + }) } /// A handle to this run's owner, for the CLI's safe points. @@ -508,11 +543,7 @@ impl Control { tracing::warn!("could not answer a ping: {error}"); } } - Ok(Request::Shutdown) => { - // RED stub: shutdown is not implemented yet. - let _ = parked; - let _ = protocol::write_reply(&stream, &Reply::Error("not_implemented".into())); - } + Ok(Request::Shutdown) => self.stop(StopSource::Socket, Some(stream), parked), Err(error) if error.kind() == io::ErrorKind::InvalidData => { let _ = protocol::write_reply(&stream, &Reply::Error("unknown_op".to_string())); } @@ -520,6 +551,81 @@ impl Control { } } + /// Take a stop request (with the requesting connection, for a socket + /// `shutdown`) and act on [`StopToken::request_stop`]'s decision. + /// + /// - `KillAgentGroup`: under the owner lock, and only while the exit has + /// not been observed, SIGTERM the agent's group. Reply `accepted` and + /// park the connection for [`Supervisor::finish`]'s final line. Then + /// wait up to [`crate::agent::TERMINATE_VERIFY_WAIT`] for the main + /// thread to observe the exit, and SIGKILL the group if it has not. + /// - `FlagForSafePoint`: reply `accepted` and park; the CLI's next safe + /// point stops the run. + /// - `ReplyHandedOff`: reply `handed_off`. + /// - `AlreadyFinished`: report the stop if the run ended as one, + /// otherwise reply `already_finished`. + fn stop(&self, source: StopSource, stream: Option, parked: &mut Vec) { + let pgid = self.sidecar.agent_pgid; + let mut state = self.token.lock(); + let action = request_stop_locked(&mut state, source); + let owner = state.phase.as_str().to_string(); + let ended_stopped = state.ended_stopped; + if action == StopAction::KillAgentGroup && !state.agent_exited { + tracing::info!("stop requested by {source:?}: terminating agent group {pgid}"); + signal_group(pgid, libc::SIGTERM); + } + drop(state); + + let accepted = Reply::Accepted(ShutdownAccepted { owner }); + let replies: &[Reply] = match action { + StopAction::KillAgentGroup | StopAction::FlagForSafePoint => { + std::slice::from_ref(&accepted) + } + StopAction::ReplyHandedOff => &[Reply::Error("handed_off".to_string())], + StopAction::AlreadyFinished if ended_stopped => &[ + accepted.clone(), + Reply::Stopped(ShutdownStopped { + exit_code: STOPPED_EXIT_CODE, + advance_suppressed: true, + }), + ], + StopAction::AlreadyFinished => &[Reply::Error("already_finished".to_string())], + }; + if let Some(stream) = stream { + let sent = replies + .iter() + .try_for_each(|reply| protocol::write_reply(&stream, reply)); + match sent { + Ok(()) + if matches!( + action, + StopAction::KillAgentGroup | StopAction::FlagForSafePoint + ) => + { + parked.push(stream); + } + Ok(()) => {} + Err(error) => tracing::warn!("could not answer a shutdown request: {error}"), + } + } + + if action == StopAction::KillAgentGroup { + let state = self.token.lock(); + let (state, _) = self + .token + .owner + .exited + .wait_timeout_while(state, crate::agent::TERMINATE_VERIFY_WAIT, |state| { + !state.agent_exited + }) + .unwrap_or_else(PoisonError::into_inner); + if !state.agent_exited { + tracing::warn!("agent group {pgid} outlived SIGTERM; sending SIGKILL"); + signal_group(pgid, libc::SIGKILL); + } + } + } + /// The ping reply: the sidecar's identity plus the current owner phase. fn ping_reply(&self) -> PingReply { let sidecar = &self.sidecar; @@ -820,9 +926,19 @@ mod tests { /// `source` that holds one: a process exit or a signal-mask or /// signal-wait call. fn forbidden_calls(source: &str) -> Vec { - // RED stub. - let _ = source; - Vec::new() + let needles = needles(); + let mut found = Vec::new(); + for (index, line) in source.lines().enumerate() { + if line.trim_start().starts_with("//") { + continue; + } + for needle in &needles { + if line.contains(needle.as_str()) { + found.push(format!("line {}: {needle}", index + 1)); + } + } + } + found } /// The needles, each joined from two fragments at run time so this test From ff46146403a547b70da4e6d7a087102ec29e0951 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:36:54 -0400 Subject: [PATCH 024/112] test(50-07): prove a stop racing natural completion never advances (V-14) - Five interleavings against the real StopToken and a real agent process, each window held open with barriers: stop before exit, exit observed then stop before the decision, stop during a gate wait, stop before the next spawn, stop between spawn and publish - Control: the same machine with no stop advances - Table tests over on_stop_request, on_agent_exit, at_safe_point and the token's request_stop and safe_point for every phase Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor/runtime.rs | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) diff --git a/crates/devflow-core/src/supervisor/runtime.rs b/crates/devflow-core/src/supervisor/runtime.rs index 74e68d29..06480952 100644 --- a/crates/devflow-core/src/supervisor/runtime.rs +++ b/crates/devflow-core/src/supervisor/runtime.rs @@ -972,4 +972,225 @@ mod tests { assert_eq!(found[index], format!("line {line}: {needle}")); } } + + /// The windows of one stage's life in which a stop can land (V-14). + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum Window { + /// The agent is still running. + BeforeExit, + /// `waitid` has seen the exit, the leader is unreaped, and the exit + /// decision has not been taken. + ExitObservedBeforeDecision, + /// The CLI is waiting on a gate. + DuringGateWait, + /// Safe point 1 has passed and the run is `Advancing` again, before + /// safe point 2 (right before the next stage's spawn). + AfterAdvancingBeforeSpawn, + /// The next stage is spawned and answers ping, before it is + /// published (safe point 3). + BetweenSpawnAndPublish, + } + + /// How one stage's life ended. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum Verdict { + /// The next stage was published: the pipeline advanced. + Advanced, + /// The run stopped at this decision and launched nothing further. + Stopped(&'static str), + } + + /// One stage's life against a real [`StopToken`]: a real agent process + /// (`cat`, which exits when its stdin closes) is observed with `waitid` + /// without reaping, the exit is decided under the owner lock, and the + /// advance passes its gate wait and three safe points. `pause` runs at + /// `window` and holds it open. + fn run_stage(token: &StopToken, window: Option, pause: &dyn Fn()) -> Verdict { + let at = |here: Window| { + if window == Some(here) { + pause(); + } + }; + let mut agent = ChildGuard(Some( + Command::new("/bin/cat") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .spawn() + .expect("spawn the agent"), + )); + at(Window::BeforeExit); + drop(agent.child().stdin.take()); + wait_for_exit_without_reaping(agent.child().id()).expect("observe the exit"); + at(Window::ExitObservedBeforeDecision); + let outcome = token + .observe_exit(|_| agent.child().wait().map(exit_code)) + .expect("decide the exit"); + if outcome == AgentOutcome::Stopped { + return Verdict::Stopped("exit decision"); + } + assert_eq!(outcome, AgentOutcome::Completed(0)); + assert_eq!(token.phase(), OwnerPhase::Advancing); + if token.safe_point() == SafePoint::StopHere { + return Verdict::Stopped("safe point 1"); + } + token.enter(OwnerPhase::GateWait); + at(Window::DuringGateWait); + if token.safe_point() == SafePoint::StopHere { + return Verdict::Stopped("gate wait"); + } + token.enter(OwnerPhase::Advancing); + at(Window::AfterAdvancingBeforeSpawn); + if token.safe_point() == SafePoint::StopHere { + return Verdict::Stopped("safe point 2"); + } + at(Window::BetweenSpawnAndPublish); + if token.safe_point() == SafePoint::StopHere { + return Verdict::Stopped("safe point 3"); + } + Verdict::Advanced + } + + #[test] + fn a_stop_racing_natural_completion_never_advances() { + let cases = [ + ( + Window::BeforeExit, + StopAction::KillAgentGroup, + "exit decision", + ), + ( + Window::ExitObservedBeforeDecision, + StopAction::KillAgentGroup, + "exit decision", + ), + ( + Window::DuringGateWait, + StopAction::FlagForSafePoint, + "gate wait", + ), + ( + Window::AfterAdvancingBeforeSpawn, + StopAction::FlagForSafePoint, + "safe point 2", + ), + ( + Window::BetweenSpawnAndPublish, + StopAction::FlagForSafePoint, + "safe point 3", + ), + ]; + for (window, expected_action, stopped_at) in cases { + let token = StopToken::new(OwnerPhase::AgentRunning); + let ready = Arc::new(std::sync::Barrier::new(2)); + let done = Arc::new(std::sync::Barrier::new(2)); + let stopper = { + let (token, ready, done) = (token.clone(), Arc::clone(&ready), Arc::clone(&done)); + std::thread::spawn(move || { + ready.wait(); + let action = token.request_stop(StopSource::Socket); + done.wait(); + action + }) + }; + let verdict = run_stage(&token, Some(window), &|| { + ready.wait(); + done.wait(); + }); + let action = stopper.join().expect("the stopper thread"); + assert_eq!(action, expected_action, "{window:?}: the stop's action"); + assert_eq!( + verdict, + Verdict::Stopped(stopped_at), + "{window:?}: a stop must end the run, never advance it" + ); + assert!(token.stop_requested(), "{window:?}: the stop is recorded"); + assert!(token.ended_stopped(), "{window:?}: the run ended as a stop"); + } + } + + #[test] + fn natural_completion_without_a_stop_advances() { + let token = StopToken::new(OwnerPhase::AgentRunning); + assert_eq!(run_stage(&token, None, &|| {}), Verdict::Advanced); + assert!(!token.stop_requested()); + assert!(!token.ended_stopped(), "no stop, so no stopped line"); + assert_eq!(token.phase(), OwnerPhase::Advancing); + } + + #[test] + fn every_owner_phase_has_exactly_one_stop_decision() { + use OwnerPhase::{Advancing, AgentRunning, Finished, GateWait, HandedOff}; + let table = [ + ( + AgentRunning, + StopAction::KillAgentGroup, + true, + "agent_running", + ), + (Advancing, StopAction::FlagForSafePoint, true, "advancing"), + (GateWait, StopAction::FlagForSafePoint, true, "gate_wait"), + (HandedOff, StopAction::ReplyHandedOff, false, "handed_off"), + (Finished, StopAction::AlreadyFinished, false, "finished"), + ]; + for (phase, action, flags, label) in table { + assert_eq!(on_stop_request(phase), action, "{phase:?}"); + assert_eq!(phase.as_str(), label); + let token = StopToken::new(phase); + assert_eq!( + token.safe_point(), + SafePoint::Proceed, + "{phase:?}: before a stop" + ); + assert!(!token.ended_stopped(), "{phase:?}: before a stop"); + assert_eq!( + token.request_stop(StopSource::Signal(libc::SIGTERM)), + action + ); + assert_eq!(token.stop_requested(), flags, "{phase:?}: the stop flag"); + assert_eq!( + token.phase(), + phase, + "{phase:?}: a stop never moves the phase" + ); + let expected = if flags { + SafePoint::StopHere + } else { + SafePoint::Proceed + }; + assert_eq!( + token.safe_point(), + expected, + "{phase:?}: the next safe point" + ); + assert_eq!(token.ended_stopped(), flags, "{phase:?}: ended as a stop"); + } + } + + #[test] + fn exit_and_safe_point_decisions_cover_every_input() { + for code in [0, 1, 7, 137, STOPPED_EXIT_CODE] { + assert_eq!(on_agent_exit(false, code), AgentOutcome::Completed(code)); + assert_eq!(on_agent_exit(true, code), AgentOutcome::Stopped); + } + assert_eq!(at_safe_point(false), SafePoint::Proceed); + assert_eq!(at_safe_point(true), SafePoint::StopHere); + // The exit decision leaves the phase where the CLI expects it. + let completed = StopToken::new(OwnerPhase::AgentRunning); + assert_eq!( + completed.observe_exit(|_| Ok(7)).expect("decide"), + AgentOutcome::Completed(7) + ); + assert_eq!(completed.phase(), OwnerPhase::Advancing); + assert!(!completed.ended_stopped()); + let stopped = StopToken::new(OwnerPhase::AgentRunning); + stopped.request_stop(StopSource::Socket); + assert_eq!( + stopped + .observe_exit(|stop| Ok(if stop { 143 } else { 0 })) + .expect("decide"), + AgentOutcome::Stopped + ); + assert_eq!(stopped.phase(), OwnerPhase::Finished); + assert!(stopped.ended_stopped()); + } } From 7f7305c94f74a8365048cf072ba384517596caaf Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:49:04 -0400 Subject: [PATCH 025/112] test(50-07): fail V-14 instead of hanging when a window is unreached A run that ended before its stop window left the stopper thread blocked on the barrier and the test hung on the join (seen with a hand mutant that made every exit a stop). Assert the window was reached first. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor/runtime.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/devflow-core/src/supervisor/runtime.rs b/crates/devflow-core/src/supervisor/runtime.rs index 06480952..90f59548 100644 --- a/crates/devflow-core/src/supervisor/runtime.rs +++ b/crates/devflow-core/src/supervisor/runtime.rs @@ -1092,10 +1092,18 @@ mod tests { action }) }; + let reached = std::cell::Cell::new(false); let verdict = run_stage(&token, Some(window), &|| { + reached.set(true); ready.wait(); done.wait(); }); + // A run that ended before its window would leave the stopper + // blocked on the barrier: fail here rather than hang on the join. + assert!( + reached.get(), + "{window:?}: the run ended before its window was reached: {verdict:?}" + ); let action = stopper.join().expect("the stopper thread"); assert_eq!(action, expected_action, "{window:?}: the stop's action"); assert_eq!( From d234bb7807bb269f7130f0551b9641a87a12de45 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:50:01 -0400 Subject: [PATCH 026/112] test(50-07): add failing signal-pipe and TERM-ignoring member tests - SIGTERM and SIGINT to a child supervisor must end as a stop with the agent group and socket gone; the agent's SigBlk has no TERM/INT bit and its SigCgt no TERM bit; control: without the pipe the supervisor dies on TERM and its agent group lives on - A member that ignores TERM must be gone within the grace window after a socket shutdown - install_signal_pipe is a no-op stub Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor/runtime.rs | 230 +++++++++++++++++- 1 file changed, 226 insertions(+), 4 deletions(-) diff --git a/crates/devflow-core/src/supervisor/runtime.rs b/crates/devflow-core/src/supervisor/runtime.rs index 90f59548..d08a4bb2 100644 --- a/crates/devflow-core/src/supervisor/runtime.rs +++ b/crates/devflow-core/src/supervisor/runtime.rs @@ -337,6 +337,12 @@ fn exit_code(status: std::process::ExitStatus) -> i32 { }) } +/// Turn SIGTERM and SIGINT into stop events for this process's supervisor. +/// RED stub: installs nothing yet. +pub fn install_signal_pipe() -> io::Result<()> { + Ok(()) +} + /// A running supervisor: its published socket, its sidecar and its control /// thread. Dropping it without [`finish`](Self::finish) still stops the /// thread and removes the launch files. @@ -661,6 +667,8 @@ mod tests { const OUTCOME_ENV: &str = "DEVFLOW_TEST_RUNTIME_OUTCOME"; /// The `sh -c` script the child supervisor runs as its agent. const AGENT_ENV: &str = "DEVFLOW_TEST_RUNTIME_AGENT"; + /// `1` when the child supervisor installs the signal pipe first. + const PIPE_ENV: &str = "DEVFLOW_TEST_RUNTIME_PIPE"; /// How long the parent waits for a child supervisor to answer a ping or /// to exit. @@ -771,13 +779,17 @@ mod tests { } } - /// The child half shared by every runtime process test: spawn the agent script in its own process + /// The child half shared by every runtime process test: optionally + /// install the signal pipe, spawn the agent script in its own process /// group, serve, wait, finish, and record the outcome. fn supervise_in_child() { let socket = std::env::var_os(SOCKET_ENV).expect("the parent names the socket"); let socket = SocketPath::new(PathBuf::from(socket)).expect("socket path fits"); let outcome_path = std::env::var_os(OUTCOME_ENV).expect("the parent names the outcome"); let script = std::env::var_os(AGENT_ENV).expect("the parent names the agent script"); + if std::env::var_os(PIPE_ENV).is_some_and(|value| value == "1") { + install_signal_pipe().expect("install the signal pipe"); + } let mut agent = Command::new("/bin/sh") .arg("-c") .arg(script) @@ -815,7 +827,7 @@ mod tests { /// Re-run `test_name` as a child supervisor for `script` and wait until /// its socket answers a ping. - fn launch(test_name: &str, cache: &Path, script: &str) -> Launched { + fn launch(test_name: &str, cache: &Path, script: &str, install_pipe: bool) -> Launched { let socket = SocketPath::for_launch( cache, Path::new("/project/runtime"), @@ -824,6 +836,7 @@ mod tests { ) .expect("socket path fits sun_path"); let outcome = cache.join("outcome"); + let pipe = if install_pipe { "1" } else { "0" }; let mut child = ChildGuard(Some(spawn_test_in_child( test_name, cache, @@ -831,6 +844,7 @@ mod tests { (SOCKET_ENV, socket.as_path().as_os_str()), (OUTCOME_ENV, outcome.as_os_str()), (AGENT_ENV, std::ffi::OsStr::new(script)), + (PIPE_ENV, std::ffi::OsStr::new(pipe)), ], ))); let deadline = Instant::now() + CHILD_WAIT; @@ -863,7 +877,12 @@ mod tests { return; } let cache = short_cache(); - let mut run = launch(SHUTDOWN_TEST, cache.path(), "sleep 60 & sleep 60 & wait"); + let mut run = launch( + SHUTDOWN_TEST, + cache.path(), + "sleep 60 & sleep 60 & wait", + false, + ); assert_eq!(run.reply.owner, "agent_running"); let pgid = run.reply.agent_pgid; assert_eq!(pgid, run.reply.agent_pid, "the agent leads its own group"); @@ -910,7 +929,7 @@ mod tests { "while [ ! -e '{}' ]; do sleep 0.05; done; exit 7", go.display() ); - let mut run = launch(COMPLETES_TEST, cache.path(), &script); + let mut run = launch(COMPLETES_TEST, cache.path(), &script, false); assert_eq!(run.reply.owner, "agent_running"); std::fs::write(&go, "").expect("let the agent exit"); let output = run.child.wait_exit(); @@ -1201,4 +1220,207 @@ mod tests { assert_eq!(stopped.phase(), OwnerPhase::Finished); assert!(stopped.ended_stopped()); } + + /// The named mask field of `/proc//status` (`SigBlk`, `SigCgt`, + /// `SigIgn`, ...). Panics when the field is absent, so an unreadable + /// mask never reads as clear. + #[cfg(target_os = "linux")] + fn signal_mask_field(pid: u32, field: &str) -> u64 { + let status = std::fs::read_to_string(format!("/proc/{pid}/status")) + .unwrap_or_else(|error| panic!("/proc/{pid}/status must be readable: {error}")); + let hex = status + .lines() + .find_map(|line| line.strip_prefix(field)?.strip_prefix(':')) + .unwrap_or_else(|| panic!("/proc/{pid}/status has no {field} line")) + .trim(); + u64::from_str_radix(hex, 16).expect("signal masks are hex") + } + + /// `signal`'s bit in a `/proc` signal mask. + #[cfg(target_os = "linux")] + fn mask_bit(signal: libc::c_int) -> u64 { + 1 << (signal - 1) + } + + /// Whether `pid` is gone or a zombie: it can no longer run. + #[cfg(target_os = "linux")] + fn process_is_dead(pid: u32) -> bool { + match std::fs::read_to_string(format!("/proc/{pid}/stat")) { + Err(_) => true, + Ok(stat) => stat + .rfind(')') + .and_then(|end| stat[end + 1..].split_whitespace().next()) + .is_some_and(|state| state == "Z"), + } + } + + /// Send `signal` to a child this test spawned and still owns. + fn signal_child(child: &mut ChildGuard, signal: libc::c_int) { + let pid = libc::pid_t::try_from(child.child().id()).expect("pid fits pid_t"); + assert!(pid > 1, "refusing to signal pid {pid}"); + // SAFETY: `pid > 1` is a single, unreaped child of this process. + assert_eq!(unsafe { libc::kill(pid, signal) }, 0, "signal the child"); + } + + #[cfg(target_os = "linux")] + const SIGNAL_TEST: &str = "supervisor::runtime::tests::sigterm_is_delivered_as_a_stop_event_not_handled_in_the_handler"; + + #[cfg(target_os = "linux")] + #[test] + fn sigterm_is_delivered_as_a_stop_event_not_handled_in_the_handler() { + use crate::test_support::{EXEC_VISIBILITY_POLL, EXEC_VISIBILITY_WAIT}; + use std::os::unix::process::ExitStatusExt; + if crate::test_support::in_child_test(SIGNAL_TEST) { + supervise_in_child(); + return; + } + let script = "sleep 60 & exec sleep 60"; + for signal in [libc::SIGTERM, libc::SIGINT] { + let cache = short_cache(); + let mut run = launch(SIGNAL_TEST, cache.path(), script, true); + let (agent, pgid) = (run.reply.agent_pid, run.reply.agent_pgid); + // Read the masks only once the agent runs `sleep`: before exec it + // is a copy of the supervisor and carries its handler. + assert!( + crate::test_support::wait_for_exec_visibility( + agent, + "sleep", + EXEC_VISIBILITY_WAIT, + EXEC_VISIBILITY_POLL + ), + "the agent must exec sleep" + ); + let supervisor = run.child.child().id(); + assert_ne!( + signal_mask_field(supervisor, "SigCgt") & mask_bit(libc::SIGTERM), + 0, + "control: the mask reader sees the supervisor's own TERM handler" + ); + let term_and_int = mask_bit(libc::SIGTERM) | mask_bit(libc::SIGINT); + assert_eq!( + signal_mask_field(agent, "SigBlk") & term_and_int, + 0, + "the agent must not inherit a blocked TERM or INT" + ); + assert_eq!( + signal_mask_field(agent, "SigCgt") & mask_bit(libc::SIGTERM), + 0, + "the agent's TERM handler must be the default" + ); + + signal_child(&mut run.child, signal); + let output = run.child.wait_exit(); + assert_child_ran_exactly_one_passing_test(&output, SIGNAL_TEST); + assert_eq!(run.outcome(), "Stopped", "signal {signal} is a stop"); + assert!( + wait_until(Duration::from_secs(5), || !group_exists(pgid)), + "signal {signal}: the agent group must be gone" + ); + assert!( + matches!(probe(&run.socket), Ok(Liveness::Gone)), + "signal {signal}: the socket must be gone" + ); + } + + // Control: a supervisor that did not install the pipe dies on the + // same SIGTERM and leaves its agent group running. + let cache = short_cache(); + let mut run = launch(SIGNAL_TEST, cache.path(), script, false); + let pgid = run.reply.agent_pgid; + signal_child(&mut run.child, libc::SIGTERM); + let output = run.child.wait_exit(); + assert_eq!( + output.status.signal(), + Some(libc::SIGTERM), + "with no handler the supervisor dies on TERM: {output:?}" + ); + assert!( + group_exists(pgid), + "with no handler the agent group outlives its supervisor" + ); + // Reap that group by the pgid its ping reply named. + assert!( + signal_group(pgid, libc::SIGKILL), + "SIGKILL the orphaned group" + ); + assert!( + wait_until(Duration::from_secs(5), || !group_exists(pgid)), + "the orphaned group must be gone after SIGKILL" + ); + } + + #[cfg(target_os = "linux")] + const IGNORER_TEST: &str = + "supervisor::runtime::tests::a_term_ignoring_group_member_is_killed_after_the_leader_exits"; + + #[cfg(target_os = "linux")] + #[test] + fn a_term_ignoring_group_member_is_killed_after_the_leader_exits() { + use crate::test_support::{EXEC_VISIBILITY_POLL, EXEC_VISIBILITY_WAIT}; + if crate::test_support::in_child_test(IGNORER_TEST) { + supervise_in_child(); + return; + } + let cache = short_cache(); + let member_file = cache.path().join("member"); + let script = format!( + "(trap '' TERM; exec sleep 60) & echo $! > '{}'; wait", + member_file.display() + ); + let mut run = launch(IGNORER_TEST, cache.path(), &script, false); + let pgid = run.reply.agent_pgid; + let mut member = None; + assert!( + wait_until(Duration::from_secs(5), || { + member = std::fs::read_to_string(&member_file) + .ok() + .and_then(|text| text.trim().parse::().ok()); + member.is_some() + }), + "the agent must record its member's pid" + ); + let member = member.expect("member pid"); + assert!( + crate::test_support::wait_for_exec_visibility( + member, + "sleep", + EXEC_VISIBILITY_WAIT, + EXEC_VISIBILITY_POLL + ), + "the member must exec sleep" + ); + // The fixture is what it claims: the member is in the agent's group + // and ignores TERM. + // SAFETY: getpgid only reads; `member` is a live pid we just saw. + let member_pgid = unsafe { libc::getpgid(libc::pid_t::try_from(member).expect("pid")) }; + assert_eq!(u32::try_from(member_pgid).ok(), Some(pgid)); + assert_ne!( + signal_mask_field(member, "SigIgn") & mask_bit(libc::SIGTERM), + 0, + "the member must ignore TERM" + ); + + let answer = request_shutdown(&run.socket, CHILD_WAIT); + assert_eq!( + answer.as_ref().ok(), + Some(&ShutdownOutcome::Stopped { + exit_code: 143, + advance_suppressed: true + }), + "the stop is reported, got {answer:?}" + ); + assert!( + wait_until(crate::agent::TERMINATE_VERIFY_WAIT, || process_is_dead( + member + )), + "the TERM-ignoring member must be gone within the grace window" + ); + let output = run.child.wait_exit(); + assert_child_ran_exactly_one_passing_test(&output, IGNORER_TEST); + assert_eq!(run.outcome(), "Stopped"); + assert!( + wait_until(Duration::from_secs(5), || !group_exists(pgid)), + "the agent group must be gone" + ); + } } From 629c55bfc0783531eb38829f0af8b2dc5797225e Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:53:45 -0400 Subject: [PATCH 027/112] feat(50-07): deliver SIGTERM and SIGINT as stop events via a self-pipe - install_signal_pipe: once per process, a close-on-exec non-blocking pipe (pipe2 on Linux, pipe plus fcntl elsewhere) and an SA_RESTART handler that only checks getpid and writes one byte - The control thread polls the pipe and turns each byte into a stop with StopSource::Signal, acted on like a socket shutdown - The test child disarms its own group guard once wait_agent returns, so a member the supervisor failed to kill is no longer hidden Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor/runtime.rs | 158 +++++++++++++++++- 1 file changed, 150 insertions(+), 8 deletions(-) diff --git a/crates/devflow-core/src/supervisor/runtime.rs b/crates/devflow-core/src/supervisor/runtime.rs index d08a4bb2..137d909e 100644 --- a/crates/devflow-core/src/supervisor/runtime.rs +++ b/crates/devflow-core/src/supervisor/runtime.rs @@ -31,8 +31,8 @@ use super::{PublishError, Sidecar, SocketPath}; use std::io; use std::os::fd::AsRawFd; use std::os::unix::net::{UnixListener, UnixStream}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError}; +use std::sync::atomic::{AtomicBool, AtomicI32, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock, PoisonError}; use std::thread::JoinHandle; /// The exit code a stopped run reports: the shell convention for a death by @@ -337,12 +337,116 @@ fn exit_code(status: std::process::ExitStatus) -> i32 { }) } -/// Turn SIGTERM and SIGINT into stop events for this process's supervisor. -/// RED stub: installs nothing yet. +/// The signal pipe's read end, or `-1` before [`install_signal_pipe`]. +static SIGNAL_READ_FD: AtomicI32 = AtomicI32::new(-1); +/// The signal pipe's write end, the handler's only output. +static SIGNAL_WRITE_FD: AtomicI32 = AtomicI32::new(-1); +/// The pid that installed the handler. A forked child that has not exec'd +/// yet runs the inherited handler under another pid and writes nothing. +static SUPERVISOR_PID: AtomicI32 = AtomicI32::new(0); + +/// The SIGTERM/SIGINT handler: the pid check and a one-byte `write`, both +/// async-signal-safe, and nothing else. The control thread turns the byte +/// into a stop. +extern "C" fn on_stop_signal(signo: libc::c_int) { + // SAFETY: getpid is async-signal-safe and has no preconditions. + if unsafe { libc::getpid() } == SUPERVISOR_PID.load(Ordering::Relaxed) { + let byte = signo as u8; + // SAFETY: write is async-signal-safe; `byte` is a live one-byte + // buffer. A full pipe drops the byte, which is harmless: the first + // byte already requested the stop. + unsafe { + libc::write( + SIGNAL_WRITE_FD.load(Ordering::Relaxed), + (&raw const byte).cast(), + 1, + ); + } + } +} + +/// Turn SIGTERM and SIGINT into stop events for this process's supervisor: +/// once per process, create a close-on-exec, non-blocking pipe and install +/// [`on_stop_signal`] with `SA_RESTART` for both signals. The control thread +/// [`Supervisor::serve`] starts reads the pipe and requests the stop. +/// +/// Call it at supervisor-process entry, before the agent is spawned. It +/// installs process-global signal state, so only a supervisor process calls +/// it, and it never blocks a signal: a blocked mask would be inherited by +/// every agent (RESEARCH Finding 3). pub fn install_signal_pipe() -> io::Result<()> { + static INSTALLED: OnceLock> = OnceLock::new(); + INSTALLED + .get_or_init(|| install_once().map_err(|error| (error.kind(), error.to_string()))) + .clone() + .map_err(|(kind, message)| io::Error::new(kind, message)) +} + +fn install_once() -> io::Result<()> { + let [read, write] = cloexec_nonblocking_pipe()?; + SIGNAL_WRITE_FD.store(write, Ordering::SeqCst); + // SAFETY: getpid has no preconditions. + SUPERVISOR_PID.store(unsafe { libc::getpid() }, Ordering::SeqCst); + SIGNAL_READ_FD.store(read, Ordering::SeqCst); + for signal in [libc::SIGTERM, libc::SIGINT] { + // SAFETY: an all-zero sigaction is valid; the fields that matter + // are set below. + let mut action: libc::sigaction = unsafe { std::mem::zeroed() }; + action.sa_sigaction = on_stop_signal as extern "C" fn(libc::c_int) as libc::sighandler_t; + action.sa_flags = libc::SA_RESTART; + // SAFETY: `action.sa_mask` is a live sigset_t. + unsafe { libc::sigemptyset(&raw mut action.sa_mask) }; + // SAFETY: `action` is a fully initialized sigaction; the old action + // is not wanted. + if unsafe { libc::sigaction(signal, &raw const action, std::ptr::null_mut()) } != 0 { + return Err(io::Error::last_os_error()); + } + } Ok(()) } +/// A pipe whose two ends are close-on-exec and non-blocking, so no agent +/// inherits it and a full pipe never blocks the handler. +#[cfg(target_os = "linux")] +fn cloexec_nonblocking_pipe() -> io::Result<[libc::c_int; 2]> { + let mut fds = [-1; 2]; + // SAFETY: `fds` is a live two-element out-parameter. + if unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC | libc::O_NONBLOCK) } != 0 { + return Err(io::Error::last_os_error()); + } + Ok(fds) +} + +/// [`cloexec_nonblocking_pipe`] where libc has no `pipe2` (Apple targets): +/// `pipe`, then `FD_CLOEXEC` and `O_NONBLOCK` on each end. +#[cfg(not(target_os = "linux"))] +fn cloexec_nonblocking_pipe() -> io::Result<[libc::c_int; 2]> { + let mut fds = [-1; 2]; + // SAFETY: `fds` is a live two-element out-parameter. + if unsafe { libc::pipe(fds.as_mut_ptr()) } != 0 { + return Err(io::Error::last_os_error()); + } + for fd in fds { + // SAFETY: `fd` is an open descriptor this call created. + let configured = unsafe { + libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC) == 0 && { + let flags = libc::fcntl(fd, libc::F_GETFL); + flags >= 0 && libc::fcntl(fd, libc::F_SETFL, flags | libc::O_NONBLOCK) == 0 + } + }; + if !configured { + let error = io::Error::last_os_error(); + for fd in fds { + // SAFETY: both descriptors were created above and are not + // shared yet. + unsafe { libc::close(fd) }; + } + return Err(error); + } + } + Ok(fds) +} + /// A running supervisor: its published socket, its sidecar and its control /// thread. Dropping it without [`finish`](Self::finish) still stops the /// thread and removes the launch files. @@ -487,6 +591,28 @@ impl Drop for Supervisor { } } +/// Read every pending byte from the non-blocking signal pipe; each is one +/// delivered signal's number. +fn drain_signal_pipe(fd: libc::c_int) -> Vec { + let mut signals = Vec::new(); + let mut buf = [0_u8; 64]; + loop { + // SAFETY: `buf` is a live buffer of the length passed. + let read = unsafe { libc::read(fd, buf.as_mut_ptr().cast(), buf.len()) }; + let Ok(read) = usize::try_from(read) else { + if io::Error::last_os_error().kind() == io::ErrorKind::Interrupted { + continue; + } + // WouldBlock: the pipe is drained. + return signals; + }; + if read == 0 { + return signals; + } + signals.extend(buf[..read].iter().map(|&byte| i32::from(byte))); + } +} + /// The control thread's state. struct Control { listener: UnixListener, @@ -503,17 +629,27 @@ impl Control { let tick = libc::c_int::try_from(crate::agent::TERMINATE_VERIFY_POLL.as_millis()) .unwrap_or(libc::c_int::MAX); while !self.quit.load(Ordering::Acquire) { - let mut fds = [libc::pollfd { - fd: self.listener.as_raw_fd(), + // A negative fd (no pipe installed) is ignored by poll. + let mut fds = [ + self.listener.as_raw_fd(), + SIGNAL_READ_FD.load(Ordering::SeqCst), + ] + .map(|fd| libc::pollfd { + fd, events: libc::POLLIN, revents: 0, - }]; + }); // SAFETY: `fds` is a live array of initialized pollfd entries and // its length is passed alongside it. let ready = unsafe { libc::poll(fds.as_mut_ptr(), fds.len() as libc::nfds_t, tick) }; if ready <= 0 { continue; } + if fds[1].revents & libc::POLLIN != 0 { + for signal in drain_signal_pipe(fds[1].fd) { + self.stop(StopSource::Signal(signal), None, &mut parked); + } + } if fds[0].revents & libc::POLLIN != 0 { self.accept_all(&mut parked); } @@ -800,12 +936,17 @@ mod tests { .process_group(0) .spawn() .expect("spawn the agent"); - let _group = GroupGuard(agent.id()); + // Covers a panic before the supervisor has decided the run. Once + // `wait_agent` returns, ending the group is the supervisor's job, so + // the guard is disarmed: a cleanup here would hide a member the + // supervisor failed to kill. The parent's guard still reaps it. + let group = GroupGuard(agent.id()); let sidecar = test_sidecar(&socket, agent.id()); let supervisor = Supervisor::serve(socket, sidecar).expect("serve the socket"); let outcome = supervisor .wait_agent(&mut agent) .expect("wait for the agent"); + std::mem::forget(group); supervisor.finish(); std::fs::write(outcome_path, format!("{outcome:?}")).expect("record the outcome"); } @@ -1255,6 +1396,7 @@ mod tests { } /// Send `signal` to a child this test spawned and still owns. + #[cfg(target_os = "linux")] fn signal_child(child: &mut ChildGuard, signal: libc::c_int) { let pid = libc::pid_t::try_from(child.child().id()).expect("pid fits pid_t"); assert!(pid > 1, "refusing to signal pid {pid}"); From c484f8d6cf5ac777127d03bf8f5a07ccaddd30cd Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 13:55:24 -0400 Subject: [PATCH 028/112] test(50-07): pin the final line a parked shutdown caller hears - Stopped at a safe point: stopped, socket already gone - Taken but never stopped before finish: handed_off, never stopped - Handed off before the request: handed_off at once, nothing flagged Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor/runtime.rs | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/crates/devflow-core/src/supervisor/runtime.rs b/crates/devflow-core/src/supervisor/runtime.rs index 137d909e..a3ccb3ab 100644 --- a/crates/devflow-core/src/supervisor/runtime.rs +++ b/crates/devflow-core/src/supervisor/runtime.rs @@ -1565,4 +1565,84 @@ mod tests { "the agent group must be gone" ); } + + /// A supervisor served from this test process for an agent that is + /// never signalled: its sidecar pgid is `0`, which `signal_group` + /// refuses, and no test here stops it in `AgentRunning`. + fn in_process_supervisor(cache: &Path) -> (Supervisor, PathBuf) { + let socket = SocketPath::for_launch( + cache, + Path::new("/project/runtime"), + PhaseId::new(50), + &new_launch_id(), + ) + .expect("socket path fits sun_path"); + let mut sidecar = test_sidecar(&socket, std::process::id()); + sidecar.agent_pgid = 0; + let path = socket.as_path().to_path_buf(); + let supervisor = Supervisor::serve(socket, sidecar).expect("serve the socket"); + (supervisor, path) + } + + #[test] + fn a_parked_shutdown_hears_stopped_only_when_the_run_stopped() { + let cache = short_cache(); + let stopped = ShutdownOutcome::Stopped { + exit_code: 143, + advance_suppressed: true, + }; + + // Taken while the CLI advances, then stopped at a safe point: the + // parked caller hears `stopped`, and the socket is already gone. + let (supervisor, path) = in_process_supervisor(cache.path()); + let token = supervisor.token(); + token.enter(OwnerPhase::Advancing); + let caller = { + let path = path.clone(); + std::thread::spawn(move || request_shutdown(&path, CHILD_WAIT).ok()) + }; + assert!( + wait_until(CHILD_WAIT, || token.stop_requested()), + "the request is taken" + ); + assert_eq!(token.safe_point(), SafePoint::StopHere); + supervisor.finish(); + assert_eq!(caller.join().expect("caller"), Some(stopped)); + assert!(matches!(probe(&path), Ok(Liveness::Gone))); + + // Control: taken while advancing, but no safe point stopped the run + // before it ended: the caller hears `handed_off`, never `stopped`. + let (supervisor, path) = in_process_supervisor(cache.path()); + let token = supervisor.token(); + token.enter(OwnerPhase::Advancing); + let caller = { + let path = path.clone(); + std::thread::spawn(move || request_shutdown(&path, CHILD_WAIT).ok()) + }; + assert!( + wait_until(CHILD_WAIT, || token.stop_requested()), + "the request is taken" + ); + supervisor.finish(); + assert_eq!( + caller.join().expect("caller"), + Some(ShutdownOutcome::HandedOff) + ); + + // Handed off before the request: answered at once, nothing flagged, + // and a ping names the owner. + let (supervisor, path) = in_process_supervisor(cache.path()); + supervisor.hand_off(); + match probe(&path) { + Ok(Liveness::Alive(reply)) => assert_eq!(reply.owner, "handed_off"), + other => panic!("a served socket answers ping, got {other:?}"), + } + assert_eq!( + request_shutdown(&path, CHILD_WAIT).ok(), + Some(ShutdownOutcome::HandedOff) + ); + assert!(!supervisor.token().stop_requested()); + supervisor.finish(); + assert!(matches!(probe(&path), Ok(Liveness::Gone))); + } } From 98209aa56fb1ae915954838f1c087c090cadc89a Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 14:09:38 -0400 Subject: [PATCH 029/112] test(50-08): add failing capture-shape tracer for run_supervised_stage - MonitorError is #[non_exhaustive] with SocketPath, Publish, PgidMismatch and NotReady (named R-9 break; semver gate --through 50-08 verdict=match) - StageIo, StageSpec, StageRun and a stub run_supervised_stage - capture_arm_runs_the_agent_under_an_addressed_supervisor runs the supervisor in a child test process and fails on the stub Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/monitor.rs | 539 ++++++++++++++++++++++++++++- 1 file changed, 538 insertions(+), 1 deletion(-) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index a5e34138..76cf7d12 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -14,10 +14,13 @@ use crate::agent_result::{IdleTimeoutCommit, IdleTimeoutRecord}; use crate::git::hermetic_command; use crate::phase_id::PhaseId; +use crate::stage::Stage; use crate::state::{AgentKind, State}; +use crate::supervisor::SocketPath; +use crate::supervisor::runtime::{AgentOutcome, Supervisor}; use std::io::{BufRead, BufReader, Write}; use std::os::unix::process::CommandExt; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Stdio; use std::sync::mpsc; use std::time::Duration; @@ -25,6 +28,7 @@ use tracing::{debug, info, warn}; /// Errors produced by monitor operations. #[derive(Debug, thiserror::Error)] +#[non_exhaustive] pub enum MonitorError { /// Spawning the monitor process failed. #[error("failed to spawn monitor: {0}")] @@ -38,6 +42,25 @@ pub enum MonitorError { /// A child spawned with piped stdio did not expose one of its pipes. #[error("supervised child exposed no {0} pipe")] NoChildPipe(&'static str), + /// The supervisor socket path was refused before anything was bound. + #[error("supervisor socket path refused: {0}")] + SocketPath(#[from] crate::supervisor::SocketPathError), + /// The supervisor socket or its sidecar could not be published. + #[error("could not publish the supervisor socket: {0}")] + Publish(#[from] crate::supervisor::PublishError), + /// The agent did not lead its own process group, so a group signal + /// could reach a process that is not the agent's (R-3). The agent was + /// killed and reaped. + #[error("agent {pid} is not its own process-group leader (pgid {pgid})")] + PgidMismatch { + /// The agent's pid. + pid: u32, + /// The process group the agent was found in (`-1` when unreadable). + pgid: i32, + }, + /// A supervision shape that this build cannot run yet. + #[error("supervision shape not ready: {0}")] + NotReady(String), } /// Idle-timeout default in seconds (D-02): the measured constraint-8 floor. @@ -1207,6 +1230,116 @@ pub fn run_pipe_owning_monitor( Ok(code) } +/// How [`run_supervised_stage`] wires the agent's stdio: the two supervision +/// shapes of one supervisor body. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum StageIo { + /// The capture shape (the `MonitorLaunch::Legacy` script's replacement): + /// stdin is `/dev/null`, stdout and stderr go straight to the phase's + /// capture and stderr files. + Capture, + /// The stream shape (`MonitorLaunch::PipeOwning`): the supervisor owns + /// both pipes, delivers `prompt` as the first turn on stdin, tees stdout + /// to the capture file, applies the close rule and fires the idle + /// timeout after `idle_timeout` of silence. + Stream { + /// The stage prompt, delivered on the agent's stdin. + prompt: String, + /// How long the stream may be silent before the idle timeout fires. + idle_timeout: Duration, + }, +} + +impl StageIo { + /// The launch label a ping reply and the sidecar carry: `capture` or + /// `stream`. + pub fn label(&self) -> &'static str { + match self { + Self::Capture => "capture", + Self::Stream { .. } => "stream", + } + } +} + +/// Everything [`run_supervised_stage`] needs to supervise one stage's agent. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct StageSpec { + /// The project root the capture, stderr, exit and pid files live under. + pub project_root: PathBuf, + /// The phase the stage runs for. + pub phase: PhaseId, + /// The stage the agent runs, when one is known. + pub stage: Option, + /// The agent's working directory (the phase worktree, or the project + /// root when worktree mode is off). + pub workdir: PathBuf, + /// The agent program. + pub program: String, + /// The agent's argv after the program, passed literally. + pub args: Vec, + /// The adapter environment, applied after `hermetic_command`'s scrub so + /// deliberate configuration survives it (process env, never argv). + pub envs: Vec<(String, String)>, + /// Which coding agent this is. + pub agent: AgentKind, + /// The supervision shape. + pub io: StageIo, + /// Where the supervisor publishes its socket. + pub socket: SocketPath, + /// The per-launch id the sidecar records. + pub launch_id: String, +} + +impl StageSpec { + /// A spec with no stage, no extra argv and no adapter env, whose agent + /// runs in `project_root`. Set the public fields for the rest. + pub fn new( + project_root: PathBuf, + phase: PhaseId, + agent: AgentKind, + program: String, + io: StageIo, + socket: SocketPath, + launch_id: String, + ) -> Self { + Self { + workdir: project_root.clone(), + project_root, + phase, + stage: None, + program, + args: Vec::new(), + envs: Vec::new(), + agent, + io, + socket, + launch_id, + } + } +} + +/// A supervised stage whose agent has exited. +#[derive(Debug)] +pub struct StageRun { + /// How the agent's run ended. Only [`AgentOutcome::Completed`] may + /// advance the pipeline. + pub outcome: AgentOutcome, + /// The still-published supervisor. The caller owns + /// [`Supervisor::hand_off`] and [`Supervisor::finish`]. + pub supervisor: Supervisor, +} + +/// Supervise one stage's agent under an addressed [`Supervisor`], in either +/// supervision shape. +pub fn run_supervised_stage(spec: StageSpec) -> Result { + Err(MonitorError::NotReady(format!( + "{} shape: RED stub", + spec.io.label() + ))) +} + /// The idle-timeout firing sequence, in the ONE order it may run (D-05). /// /// 1. Enumerate the commits the agent made. @@ -4284,4 +4417,408 @@ exit 0 let parsed = crate::agent_result::parse_antigravity_event_result(&capture).unwrap(); assert_eq!(parsed.status, crate::agent_result::AgentStatus::Success); } + + // ---- 50-08: both supervision shapes on the 50-07 runtime ---------- + // + // Each supervisor runs in a child test process (`spawn_test_in_child`), + // because `run_supervised_stage` installs process-wide signal state. + // The parent probes the child's socket, signals the child by the pid of + // the `Child` it owns, and reads the files the child wrote. + + /// The socket the child supervisor publishes. + const STAGE_SOCKET_ENV: &str = "DEVFLOW_TEST_STAGE_SOCKET"; + /// Where the child records ` `. + const STAGE_OUTCOME_ENV: &str = "DEVFLOW_TEST_STAGE_OUTCOME"; + /// `capture` or `stream`. + const STAGE_SHAPE_ENV: &str = "DEVFLOW_TEST_STAGE_SHAPE"; + /// The fake agent's absolute path. + const STAGE_PROGRAM_ENV: &str = "DEVFLOW_TEST_STAGE_PROGRAM"; + /// The project root the phase files live under. + const STAGE_ROOT_ENV: &str = "DEVFLOW_TEST_STAGE_ROOT"; + /// The agent's working directory. + const STAGE_WORKDIR_ENV: &str = "DEVFLOW_TEST_STAGE_WORKDIR"; + /// The agent kind's label. + const STAGE_AGENT_ENV: &str = "DEVFLOW_TEST_STAGE_AGENT"; + /// One adapter env pair as `NAME=VALUE`, or empty for none. + const STAGE_ADAPTER_ENV: &str = "DEVFLOW_TEST_STAGE_ADAPTER_ENV"; + + /// The phase every supervised-stage test runs. + const STAGE_PHASE: PhaseId = PhaseId::new(50); + /// The stream shape's idle timeout in these tests: far above every wait + /// below, so it never fires unless a test asks it to. + const STAGE_IDLE_TIMEOUT: Duration = Duration::from_secs(120); + /// How long the parent waits for a child supervisor to answer or exit. + const STAGE_CHILD_WAIT: Duration = Duration::from_secs(30); + /// The first turn the stream shape delivers. + const STAGE_PROMPT: &str = "SUPERVISED-STAGE-PROMPT"; + + /// One supervision shape, as the tests name it. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum Shape { + Capture, + Stream, + } + + impl Shape { + fn label(self) -> &'static str { + self.io().label() + } + + fn io(self) -> StageIo { + match self { + Shape::Capture => StageIo::Capture, + Shape::Stream => StageIo::Stream { + prompt: STAGE_PROMPT.to_string(), + idle_timeout: STAGE_IDLE_TIMEOUT, + }, + } + } + + fn parse(label: &str) -> Self { + match label { + "capture" => Shape::Capture, + "stream" => Shape::Stream, + other => panic!("unknown shape {other}"), + } + } + } + + /// The child half of every supervised-stage test: build the spec from + /// the parent's env, run the stage, record ` `, finish. + fn supervise_stage_in_child() { + let var = + |name: &str| std::env::var_os(name).unwrap_or_else(|| panic!("the parent sets {name}")); + let socket = + SocketPath::new(PathBuf::from(var(STAGE_SOCKET_ENV))).expect("socket path fits"); + let launch_id = socket + .as_path() + .file_stem() + .and_then(std::ffi::OsStr::to_str) + .and_then(|stem| stem.rsplit('-').next()) + .expect("the socket name ends in its launch id") + .to_string(); + let shape = Shape::parse(var(STAGE_SHAPE_ENV).to_str().expect("utf-8 shape")); + let agent: AgentKind = var(STAGE_AGENT_ENV) + .to_str() + .expect("utf-8 agent") + .parse() + .expect("a known agent label"); + let program = var(STAGE_PROGRAM_ENV) + .into_string() + .expect("utf-8 program path"); + let mut spec = StageSpec::new( + PathBuf::from(var(STAGE_ROOT_ENV)), + STAGE_PHASE, + agent, + program, + shape.io(), + socket, + launch_id, + ); + spec.workdir = PathBuf::from(var(STAGE_WORKDIR_ENV)); + spec.stage = Some(Stage::Code); + let adapter_env = var(STAGE_ADAPTER_ENV) + .into_string() + .expect("utf-8 adapter env"); + if let Some((name, value)) = adapter_env.split_once('=') { + spec.envs.push((name.to_string(), value.to_string())); + } + let run = run_supervised_stage(spec).expect("supervise the stage"); + let record = format!( + "{:?} {}", + run.outcome, + run.supervisor.token().phase().as_str() + ); + run.supervisor.finish(); + std::fs::write(var(STAGE_OUTCOME_ENV), record).expect("record the outcome"); + } + + /// A short tempdir under `/tmp` holding the cache, the project root and + /// the agent's separate working directory (`/wt`). + struct StageFixture { + cache: tempfile::TempDir, + root: PathBuf, + workdir: PathBuf, + } + + impl StageFixture { + fn new() -> Self { + let cache = tempfile::Builder::new() + .prefix("dfs") + .tempdir_in("/tmp") + .expect("create a short cache dir under /tmp"); + let root = cache.path().join("proj"); + let workdir = root.join("wt"); + std::fs::create_dir_all(&workdir).expect("create the project and its worktree"); + Self { + cache, + root, + workdir, + } + } + + fn path(&self, name: &str) -> PathBuf { + self.cache.path().join(name) + } + + /// Write an executable `#!/bin/sh` fake agent named `name`. + fn agent(&self, name: &str, body: &str) -> PathBuf { + use std::os::unix::fs::PermissionsExt; + let path = self.path(name); + std::fs::write( + &path, + format!("#!/bin/sh\nPATH=/usr/bin:/bin\nexport PATH\n{body}\n"), + ) + .expect("write the fake agent"); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)) + .expect("make the fake agent executable"); + path + } + + fn read(&self, path: PathBuf) -> String { + std::fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())) + } + + fn stdout(&self) -> String { + self.read(crate::agent_result::stdout_path(&self.root, STAGE_PHASE)) + } + + fn stderr(&self) -> String { + self.read(crate::agent_result::stderr_path(&self.root, STAGE_PHASE)) + } + + fn exit_code(&self) -> String { + self.read(crate::agent_result::exit_code_path(&self.root, STAGE_PHASE)) + .trim() + .to_string() + } + + fn pid_file(&self) -> String { + self.read(crate::agent_result::agent_pid_path(&self.root, STAGE_PHASE)) + .trim() + .to_string() + } + } + + /// Polls `done` every 20 ms until it holds or `within` elapses. + fn stage_wait_until(within: Duration, mut done: impl FnMut() -> bool) -> bool { + let deadline = std::time::Instant::now() + within; + loop { + if done() { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(20)); + } + } + + /// Kills and reaps the child test process on every exit path. Only the + /// pid of the `Child` this test spawned is ever signalled. + struct StageChild(Option); + + impl StageChild { + fn try_exited(&mut self) -> bool { + self.0 + .as_mut() + .expect("the child is still owned") + .try_wait() + .expect("poll the child") + .is_some() + } + + /// Wait for the child to exit on its own and collect its output. + fn wait_exit(&mut self) -> std::process::Output { + let exited = stage_wait_until(STAGE_CHILD_WAIT, || self.try_exited()); + let child = self.0.take().expect("the child is still owned"); + if !exited { + let mut child = StageChild(Some(child)); + let _ = child.0.as_mut().expect("owned").kill(); + let output = child.0.take().expect("owned").wait_with_output(); + panic!("the child supervisor did not exit within {STAGE_CHILD_WAIT:?}: {output:?}"); + } + child + .wait_with_output() + .expect("collect the child's output") + } + } + + impl Drop for StageChild { + fn drop(&mut self) { + if let Some(mut child) = self.0.take() { + let _ = child.kill(); + let _ = child.wait(); + } + } + } + + /// SIGKILLs whatever is left of the agent's process group on drop. + struct StageGroup(u32); + + impl Drop for StageGroup { + fn drop(&mut self) { + let Ok(signed) = libc::pid_t::try_from(self.0) else { + return; + }; + // SAFETY: getpgrp has no preconditions. + if signed > 1 && signed != unsafe { libc::getpgrp() } { + // SAFETY: `signed > 1` and not our own group. + unsafe { libc::kill(-signed, libc::SIGKILL) }; + } + } + } + + /// A child supervisor the parent launched, once its socket answered. + struct StageLaunch { + child: StageChild, + group: StageGroup, + socket: SocketPath, + outcome: PathBuf, + reply: crate::supervisor::protocol::PingReply, + sidecar: crate::supervisor::Sidecar, + } + + impl StageLaunch { + fn agent_pid(&self) -> u32 { + self.reply.agent_pid + } + + fn liveness(&self) -> crate::supervisor::Liveness { + crate::supervisor::probe(self.socket.as_path()).expect("probe the socket") + } + + /// Wait for the child to exit, require that it ran exactly its one + /// test, and return the outcome it recorded. + fn finish(&mut self, test_name: &str) -> String { + let output = self.child.wait_exit(); + crate::test_support::assert_child_ran_exactly_one_passing_test(&output, test_name); + std::fs::read_to_string(&self.outcome).expect("the child recorded its outcome") + } + } + + /// Re-run `test_name` as a child supervisor of `program` in `shape`, and + /// wait until its socket answers a ping. At that first Alive the sidecar + /// must already be readable (R-7: the sidecar precedes the socket name). + fn launch_stage( + test_name: &str, + fixture: &StageFixture, + shape: Shape, + agent: AgentKind, + program: &Path, + adapter_env: &str, + ) -> StageLaunch { + let socket = SocketPath::for_launch( + fixture.cache.path(), + &fixture.root, + STAGE_PHASE, + &crate::supervisor::new_launch_id(), + ) + .expect("socket path fits sun_path"); + let outcome = fixture.path("outcome"); + let agent_label = agent.to_string(); + let mut child = StageChild(Some(crate::test_support::spawn_test_in_child( + test_name, + fixture.cache.path(), + &[ + (STAGE_SOCKET_ENV, socket.as_path().as_os_str()), + (STAGE_OUTCOME_ENV, outcome.as_os_str()), + (STAGE_SHAPE_ENV, std::ffi::OsStr::new(shape.label())), + (STAGE_PROGRAM_ENV, program.as_os_str()), + (STAGE_ROOT_ENV, fixture.root.as_os_str()), + (STAGE_WORKDIR_ENV, fixture.workdir.as_os_str()), + (STAGE_AGENT_ENV, std::ffi::OsStr::new(&agent_label)), + (STAGE_ADAPTER_ENV, std::ffi::OsStr::new(adapter_env)), + ], + ))); + let deadline = std::time::Instant::now() + STAGE_CHILD_WAIT; + let reply = loop { + if let Ok(crate::supervisor::Liveness::Alive(reply)) = + crate::supervisor::probe(socket.as_path()) + { + break reply; + } + let exited = child.try_exited(); + if exited || std::time::Instant::now() >= deadline { + let output = child.0.take().expect("owned").wait_with_output(); + panic!("the child supervisor never answered a ping (exited: {exited}): {output:?}"); + } + std::thread::sleep(Duration::from_millis(20)); + }; + let group = StageGroup(reply.agent_pgid); + let sidecar = crate::supervisor::read_sidecar(&socket.sidecar_path()) + .expect("the sidecar is readable at the first moment the socket answers (R-7)"); + StageLaunch { + child, + group, + socket, + outcome, + reply, + sidecar, + } + } + + const CAPTURE_TRACER_TEST: &str = + "monitor::tests::capture_arm_runs_the_agent_under_an_addressed_supervisor"; + + #[test] + fn capture_arm_runs_the_agent_under_an_addressed_supervisor() { + if crate::test_support::in_child_test(CAPTURE_TRACER_TEST) { + supervise_stage_in_child(); + return; + } + let fixture = StageFixture::new(); + let go = fixture.path("go"); + let program = fixture.agent( + "codex", + &format!( + "echo 'DEVFLOW_RESULT: {{\"status\":\"success\"}}'\n\ + echo 'CAPTURE-STDERR-LINE' >&2\n\ + while [ ! -e '{go}' ]; do sleep 0.05; done\n\ + exit 7", + go = go.display() + ), + ); + let mut run = launch_stage( + CAPTURE_TRACER_TEST, + &fixture, + Shape::Capture, + AgentKind::Codex, + &program, + "", + ); + assert_eq!(run.reply.launch, "capture"); + assert_eq!(run.reply.owner, "agent_running"); + assert_eq!( + run.reply.agent_pgid, + run.agent_pid(), + "the agent leads its group" + ); + assert_eq!(run.sidecar.launch, "capture"); + assert_eq!(run.sidecar.agent, "codex"); + assert_eq!(run.sidecar.agent_pid, run.agent_pid()); + assert_eq!(run.sidecar.stage.as_deref(), Some("code")); + + std::fs::write(&go, "").expect("let the agent exit"); + assert_eq!(run.finish(CAPTURE_TRACER_TEST), "Completed(7) advancing"); + + let stdout = fixture.stdout(); + let stderr = fixture.stderr(); + assert!(stdout.contains("DEVFLOW_RESULT:"), "stdout: {stdout:?}"); + assert!( + !stdout.contains("CAPTURE-STDERR-LINE"), + "stdout: {stdout:?}" + ); + assert!(stderr.contains("CAPTURE-STDERR-LINE"), "stderr: {stderr:?}"); + assert!(!stderr.contains("DEVFLOW_RESULT"), "stderr: {stderr:?}"); + assert_eq!(fixture.pid_file(), run.agent_pid().to_string()); + assert_eq!(fixture.exit_code(), "7"); + assert_eq!(run.liveness(), crate::supervisor::Liveness::Gone); + assert!( + !run.socket.sidecar_path().exists(), + "finish removes the sidecar with the socket" + ); + drop(run.group); + } } From 5c5c86d3c62dd35dc9758049c9c443920eb04225 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 14:11:37 -0400 Subject: [PATCH 030/112] feat(50-08): run the capture shape under an addressed supervisor - run_supervised_stage installs the signal pipe, spawns the agent with process_group(0) via hermetic_command, requires getpgid == pid (PgidMismatch kills and reaps otherwise), writes the pid file, serves the socket (sidecar first), waits through wait_agent and writes the exit file (143 on Stopped) - capture stdio: stdin null, stdout and stderr to separate files - rfc3339_now drops its expect(dead_code): the sidecar builder is its first non-test caller; STOPPED_EXIT_CODE is now pub(crate) - the stream shape returns NotReady until task 2 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/monitor.rs | 161 +++++++++++++++++- crates/devflow-core/src/supervisor.rs | 7 - crates/devflow-core/src/supervisor/runtime.rs | 5 +- 3 files changed, 160 insertions(+), 13 deletions(-) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index 76cf7d12..3b0abfae 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -1331,13 +1331,166 @@ pub struct StageRun { pub supervisor: Supervisor, } +/// How long [`confirm_own_group`] waits for a just-spawned agent's +/// `setpgid` to become visible before it declares a mismatch. +const PGID_SETTLE_WAIT: Duration = Duration::from_millis(100); + /// Supervise one stage's agent under an addressed [`Supervisor`], in either -/// supervision shape. +/// supervision shape, and return once the agent has exited. +/// +/// The order is the contract: +/// 1. [`install_signal_pipe`](crate::supervisor::runtime::install_signal_pipe), +/// so SIGTERM and SIGINT to this process become stop events, never a +/// death that orphans the agent (D-05, 999.134). +/// 2. Spawn the agent through `hermetic_command` (the adapter env re-applied +/// after its scrub, the gate notify command stripped) with +/// `process_group(0)`, then require that it leads its own process group +/// before anything is recorded (R-3). A mismatch kills and reaps it and +/// returns [`MonitorError::PgidMismatch`]. +/// 3. Write the agent pid file, then [`Supervisor::serve`]: the sidecar is +/// written before the socket name appears (R-7). +/// 4. [`Supervisor::wait_agent`], then the exit file: the agent's code for +/// [`AgentOutcome::Completed`], `143` for [`AgentOutcome::Stopped`]. +/// +/// The caller owns [`Supervisor::hand_off`] and [`Supervisor::finish`], and +/// advances only on [`AgentOutcome::Completed`]. pub fn run_supervised_stage(spec: StageSpec) -> Result { - Err(MonitorError::NotReady(format!( - "{} shape: RED stub", + if let StageIo::Stream { .. } = spec.io { + return Err(MonitorError::NotReady( + "stream shape lands in task 2".to_string(), + )); + } + crate::supervisor::runtime::install_signal_pipe()?; + + let stdout_file = crate::agent_result::stdout_path(&spec.project_root, spec.phase); + let stderr_file = crate::agent_result::stderr_path(&spec.project_root, spec.phase); + let exit_file = crate::agent_result::exit_code_path(&spec.project_root, spec.phase); + let pid_file = crate::agent_result::agent_pid_path(&spec.project_root, spec.phase); + if let Some(parent) = stdout_file.parent() { + crate::workflow::ensure_devflow_dir(parent)?; + } + + // Both files truncate at open, as the Legacy script's `>` and `2>` do, + // and stay separate so stderr never corrupts the parsed stdout capture. + let mut agent_cmd = hermetic_command(&spec.program, &spec.workdir); + strip_notify_env(&mut agent_cmd); + agent_cmd + .args(&spec.args) + .envs(spec.envs.iter().map(|(k, v)| (k.as_str(), v.as_str()))) + .stdin(Stdio::null()) + .stdout(Stdio::from(std::fs::File::create(&stdout_file)?)) + .stderr(Stdio::from(std::fs::File::create(&stderr_file)?)) + .process_group(0); + let mut child = agent_cmd.spawn()?; + let pid = child.id(); + + if let Err(pgid) = confirm_own_group(pid) { + abandon_agent(&mut child); + return Err(MonitorError::PgidMismatch { pid, pgid }); + } + if let Err(err) = std::fs::write(&pid_file, format!("{pid}\n")) { + abandon_agent(&mut child); + return Err(err.into()); + } + let supervisor = match Supervisor::serve(spec.socket.clone(), stage_sidecar(&spec, pid)) { + Ok(supervisor) => supervisor, + Err(err) => { + abandon_agent(&mut child); + return Err(err.into()); + } + }; + + let outcome = match supervisor.wait_agent(&mut child) { + Ok(outcome) => outcome, + Err(err) => { + abandon_agent(&mut child); + return Err(err.into()); + } + }; + let code = match outcome { + AgentOutcome::Completed(code) => code, + AgentOutcome::Stopped => crate::supervisor::runtime::STOPPED_EXIT_CODE, + }; + std::fs::write(&exit_file, format!("{code}\n"))?; + info!( + "supervised {} agent {pid} ended: {outcome:?}", spec.io.label() - ))) + ); + Ok(StageRun { + outcome, + supervisor, + }) +} + +/// Require that `pid` leads its own process group: `getpgid(pid) == pid`. +/// A just-spawned child's `setpgid` has normally landed by the time `spawn` +/// returns; if it has not, re-read `/proc//stat` field 5 (falling back +/// to `getpgid` where `/proc` is unreadable) for at most +/// [`PGID_SETTLE_WAIT`]. Returns the pgid last seen on a mismatch (`-1` when +/// it could not be read). +fn confirm_own_group(pid: u32) -> Result<(), i32> { + let Ok(signed) = libc::pid_t::try_from(pid) else { + return Err(-1); + }; + // SAFETY: getpgid has no memory preconditions; a bad pid returns -1. + let read_pgid = || proc_stat_pgrp(pid).unwrap_or_else(|| unsafe { libc::getpgid(signed) }); + // SAFETY: as above. + let mut pgid = unsafe { libc::getpgid(signed) }; + let deadline = std::time::Instant::now() + PGID_SETTLE_WAIT; + while pgid != signed && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + pgid = read_pgid(); + } + if pgid == signed { Ok(()) } else { Err(pgid) } +} + +/// Field 5 (`pgrp`) of `/proc//stat`, or `None` where it cannot be read. +fn proc_stat_pgrp(pid: u32) -> Option { + let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + // `comm` (field 2) may hold spaces or parentheses: split after the last + // `)`. Field 3 is then index 0, so field 5 is index 2. + let rest = &stat[stat.rfind(')')? + 1..]; + rest.split_whitespace().nth(2)?.parse().ok() +} + +/// SIGKILL an agent this function's caller spawned (its group, when it leads +/// one) and reap it, on a path that gives up before the supervisor owns it. +/// The pid comes from the in-memory `Child`, never from disk. +fn abandon_agent(child: &mut std::process::Child) { + // SAFETY: getpgrp has no preconditions and cannot fail. + let own_group = unsafe { libc::getpgrp() }; + if let Ok(signed) = libc::pid_t::try_from(child.id()) + && signed > 1 + && signed != own_group + { + // SAFETY: `signed > 1` and not our own group. A child that does not + // lead a group makes this ESRCH: no group can be numbered by a pid + // that is still alive as a non-leader. + unsafe { libc::kill(-signed, libc::SIGKILL) }; + } + let _ = child.kill(); + let _ = child.wait(); +} + +/// The sidecar for `spec`'s agent, spawned as `agent_pid` (its own group +/// leader), served by this process. +fn stage_sidecar(spec: &StageSpec, agent_pid: u32) -> crate::supervisor::Sidecar { + let supervisor_pid = std::process::id(); + crate::supervisor::Sidecar { + launch_id: spec.launch_id.clone(), + project_root: spec.project_root.clone(), + phase: spec.phase, + stage: spec.stage.as_ref().map(ToString::to_string), + agent: spec.agent.to_string(), + launch: spec.io.label().to_string(), + agent_pid, + agent_pgid: agent_pid, + agent_start_time: crate::agent::process_start_time(agent_pid).unwrap_or(0), + boot_id: crate::supervisor::boot_id().unwrap_or_default(), + supervisor_pid, + supervisor_start_time: crate::agent::process_start_time(supervisor_pid).unwrap_or(0), + created_at: crate::supervisor::rfc3339_now(), + } } /// The idle-timeout firing sequence, in the ONE order it may run (D-05). diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs index e2dbe27e..6b018503 100644 --- a/crates/devflow-core/src/supervisor.rs +++ b/crates/devflow-core/src/supervisor.rs @@ -404,13 +404,6 @@ pub(crate) fn rfc3339_from_unix(unix_secs: u64) -> String { /// The current time as [`rfc3339_from_unix`] formats it, for /// [`Sidecar::created_at`]. -#[cfg_attr( - not(test), - expect( - dead_code, - reason = "the launcher (50-08) is the first non-test writer of a sidecar timestamp" - ) -)] pub(crate) fn rfc3339_now() -> String { let secs = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/crates/devflow-core/src/supervisor/runtime.rs b/crates/devflow-core/src/supervisor/runtime.rs index a3ccb3ab..8ccf9f1e 100644 --- a/crates/devflow-core/src/supervisor/runtime.rs +++ b/crates/devflow-core/src/supervisor/runtime.rs @@ -36,8 +36,9 @@ use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock, PoisonError}; use std::thread::JoinHandle; /// The exit code a stopped run reports: the shell convention for a death by -/// SIGTERM, `128 + 15`. -const STOPPED_EXIT_CODE: i32 = 128 + libc::SIGTERM; +/// SIGTERM, `128 + 15`. The monitor writes it to the exit file on +/// [`AgentOutcome::Stopped`]. +pub(crate) const STOPPED_EXIT_CODE: i32 = 128 + libc::SIGTERM; /// Who owns the run right now. Its [`as_str`](Self::as_str) label is the /// `owner` field of a ping reply. From c5480fda9f5e59c90d891e02fbb2ed8ed329cd68 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 14:14:00 -0400 Subject: [PATCH 031/112] refactor(50-08): share the stream I/O loop with run_supervised_stage - drive_stream_io owns the writer and reader threads, the close rule and the idle-timeout extension loop; run_pipe_owning_monitor calls it with fire_idle_timeout as its callback and keeps its order (pid file, drive, release stdin, wait, exit file, join) - spawn_stage_agent builds both shapes' agent command (capture: stdin null, stdout to the capture file; stream: both pipes) - the stream shape of run_supervised_stage still returns NotReady Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/monitor.rs | 181 ++++++++++++++++++++++------- 1 file changed, 139 insertions(+), 42 deletions(-) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index 3b0abfae..ce81bd37 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -1009,7 +1009,7 @@ pub fn run_pipe_owning_monitor( // a capture from a previous attempt can never be mixed into this one's // (the launch path archives the prior capture first, but relying on that // to make an append-mode open safe would be an unstated coupling). - let mut capture = std::fs::File::create(&stdout_file)?; + let capture = std::fs::File::create(&stdout_file)?; let mut agent_cmd = hermetic_command(program, workdir); // The coding agent never needs the gate notify command (WR-07). @@ -1037,6 +1037,70 @@ pub fn run_pipe_owning_monitor( let child_pid = child.id(); std::fs::write(&pid_file, format!("{child_pid}\n"))?; + let threads = drive_stream_io(&mut child, capture, agent, prompt, idle_timeout, || { + fire_idle_timeout(project_root, phase, workdir, child_pid, idle_timeout); + })?; + + let status = child.wait()?; + // A signal-killed child has NO exit code — `status.code()` is `None`, and + // the previous `unwrap_or(-1)` threw the signal away (peer review + // 2026-08-03, found independently by both reviewers and by the 31-04 plan + // review as W1). That silently defeated the classification 31-04 took care + // to preserve: `evaluate_layer2` and + // `reconcile_stream_success_against_exit_code` map **137** to + // `ResourceKilled` (routed to `GateInfra` — an infrastructure fault) and + // **127** to `AgentUnavailable`. Recording `-1` matched neither, so a real + // OOM kill arrived as a generic `Failed` and routed to `GateReview`, asking + // an operator to code-review a stage that was killed by the kernel. + // + // `128 + signal` is the shell convention those constants already encode: + // SIGKILL(9) -> 137, SIGTERM(15) -> 143. `-1` is now reachable only when a + // status is neither exited nor signalled, which POSIX does not define. + let code = status.code().unwrap_or_else(|| { + use std::os::unix::process::ExitStatusExt; + status.signal().map_or(-1, |signal| 128 + signal) + }); + std::fs::write(&exit_file, format!("{code}\n"))?; + + threads.join(); + + info!("supervised child {child_pid} exited with code {code}"); + Ok(code) +} + +/// The stream shape's writer and reader threads, joined once the agent has +/// been reaped. +struct StreamThreads { + writer: std::thread::JoinHandle<()>, + reader: std::thread::JoinHandle<()>, +} + +impl StreamThreads { + fn join(self) { + let _ = self.writer.join(); + let _ = self.reader.join(); + } +} + +/// Drive a `stream-json` child's two pipes until its stream is over, with +/// the threading model [`run_pipe_owning_monitor`]'s doc describes: a writer +/// thread that delivers the first turn and holds stdin open, a reader thread +/// that tees stdout into `capture`, and this thread applying the close rule +/// and the idle timeout. +/// +/// Returns once stdout reaches EOF, the idle timeout has fired (after +/// `on_idle_timeout` ran), or the stream stayed quiet after the close rule +/// released stdin. Stdin is released before it returns, so the caller may +/// then wait for the child. Shared by [`run_pipe_owning_monitor`] and +/// [`run_supervised_stage`]'s stream shape, so the two cannot drift apart. +fn drive_stream_io( + child: &mut std::process::Child, + mut capture: std::fs::File, + agent: AgentKind, + prompt: &str, + idle_timeout: Duration, + on_idle_timeout: impl FnOnce(), +) -> Result { let mut child_stdin = child .stdin .take() @@ -1192,7 +1256,7 @@ pub fn run_pipe_owning_monitor( ({idle_timeout:?} each) — treating as a hang, not as progress" ); } - fire_idle_timeout(project_root, phase, workdir, child_pid, idle_timeout); + on_idle_timeout(); break; } } @@ -1202,32 +1266,7 @@ pub fn run_pipe_owning_monitor( // open stdin may never exit, and `child.wait()` would then block forever. drop(close_tx); - let status = child.wait()?; - // A signal-killed child has NO exit code — `status.code()` is `None`, and - // the previous `unwrap_or(-1)` threw the signal away (peer review - // 2026-08-03, found independently by both reviewers and by the 31-04 plan - // review as W1). That silently defeated the classification 31-04 took care - // to preserve: `evaluate_layer2` and - // `reconcile_stream_success_against_exit_code` map **137** to - // `ResourceKilled` (routed to `GateInfra` — an infrastructure fault) and - // **127** to `AgentUnavailable`. Recording `-1` matched neither, so a real - // OOM kill arrived as a generic `Failed` and routed to `GateReview`, asking - // an operator to code-review a stage that was killed by the kernel. - // - // `128 + signal` is the shell convention those constants already encode: - // SIGKILL(9) -> 137, SIGTERM(15) -> 143. `-1` is now reachable only when a - // status is neither exited nor signalled, which POSIX does not define. - let code = status.code().unwrap_or_else(|| { - use std::os::unix::process::ExitStatusExt; - status.signal().map_or(-1, |signal| 128 + signal) - }); - std::fs::write(&exit_file, format!("{code}\n"))?; - - let _ = writer.join(); - let _ = reader.join(); - - info!("supervised child {child_pid} exited with code {code}"); - Ok(code) + Ok(StreamThreads { writer, reader }) } /// How [`run_supervised_stage`] wires the agent's stdio: the two supervision @@ -1349,7 +1388,11 @@ const PGID_SETTLE_WAIT: Duration = Duration::from_millis(100); /// returns [`MonitorError::PgidMismatch`]. /// 3. Write the agent pid file, then [`Supervisor::serve`]: the sidecar is /// written before the socket name appears (R-7). -/// 4. [`Supervisor::wait_agent`], then the exit file: the agent's code for +/// 4. The stream shape only: drive the agent's pipes through the same +/// helper `run_pipe_owning_monitor` uses (first turn, capture tee, close +/// rule, idle timeout). An idle-timeout group kill is a natural outcome, +/// `Completed(128 + n)` with the idle-timeout verdict file, never a stop. +/// 5. [`Supervisor::wait_agent`], then the exit file: the agent's code for /// [`AgentOutcome::Completed`], `143` for [`AgentOutcome::Stopped`]. /// /// The caller owns [`Supervisor::hand_off`] and [`Supervisor::finish`], and @@ -1363,25 +1406,13 @@ pub fn run_supervised_stage(spec: StageSpec) -> Result { crate::supervisor::runtime::install_signal_pipe()?; let stdout_file = crate::agent_result::stdout_path(&spec.project_root, spec.phase); - let stderr_file = crate::agent_result::stderr_path(&spec.project_root, spec.phase); let exit_file = crate::agent_result::exit_code_path(&spec.project_root, spec.phase); let pid_file = crate::agent_result::agent_pid_path(&spec.project_root, spec.phase); if let Some(parent) = stdout_file.parent() { crate::workflow::ensure_devflow_dir(parent)?; } - // Both files truncate at open, as the Legacy script's `>` and `2>` do, - // and stay separate so stderr never corrupts the parsed stdout capture. - let mut agent_cmd = hermetic_command(&spec.program, &spec.workdir); - strip_notify_env(&mut agent_cmd); - agent_cmd - .args(&spec.args) - .envs(spec.envs.iter().map(|(k, v)| (k.as_str(), v.as_str()))) - .stdin(Stdio::null()) - .stdout(Stdio::from(std::fs::File::create(&stdout_file)?)) - .stderr(Stdio::from(std::fs::File::create(&stderr_file)?)) - .process_group(0); - let mut child = agent_cmd.spawn()?; + let (mut child, stream_capture) = spawn_stage_agent(&spec)?; let pid = child.id(); if let Err(pgid) = confirm_own_group(pid) { @@ -1400,6 +1431,29 @@ pub fn run_supervised_stage(spec: StageSpec) -> Result { } }; + let threads = match (&spec.io, stream_capture) { + ( + StageIo::Stream { + prompt, + idle_timeout, + }, + Some(capture), + ) => { + let idle = *idle_timeout; + let driven = drive_stream_io(&mut child, capture, spec.agent, prompt, idle, || { + fire_idle_timeout(&spec.project_root, spec.phase, &spec.workdir, pid, idle); + }); + match driven { + Ok(threads) => Some(threads), + Err(err) => { + abandon_agent(&mut child); + return Err(err); + } + } + } + _ => None, + }; + let outcome = match supervisor.wait_agent(&mut child) { Ok(outcome) => outcome, Err(err) => { @@ -1412,6 +1466,9 @@ pub fn run_supervised_stage(spec: StageSpec) -> Result { AgentOutcome::Stopped => crate::supervisor::runtime::STOPPED_EXIT_CODE, }; std::fs::write(&exit_file, format!("{code}\n"))?; + if let Some(threads) = threads { + threads.join(); + } info!( "supervised {} agent {pid} ended: {outcome:?}", spec.io.label() @@ -1422,6 +1479,46 @@ pub fn run_supervised_stage(spec: StageSpec) -> Result { }) } +/// Spawn `spec`'s agent in its own process group (`process_group(0)`, R-3), +/// through `hermetic_command` with the adapter env re-applied after its +/// scrub and the gate notify command stripped. Returns the child and, for +/// the stream shape, the capture file the reader thread tees into. +/// +/// The capture and stderr files truncate at open, as the Legacy script's +/// `>` and `2>` do, and stay separate so stderr never corrupts the parsed +/// stdout capture. The capture shape hands the capture file to the agent as +/// its stdout, with stdin `/dev/null`; the stream shape pipes both. +fn spawn_stage_agent( + spec: &StageSpec, +) -> Result<(std::process::Child, Option), MonitorError> { + let capture = std::fs::File::create(crate::agent_result::stdout_path( + &spec.project_root, + spec.phase, + ))?; + let stderr = std::fs::File::create(crate::agent_result::stderr_path( + &spec.project_root, + spec.phase, + ))?; + let mut agent_cmd = hermetic_command(&spec.program, &spec.workdir); + strip_notify_env(&mut agent_cmd); + agent_cmd + .args(&spec.args) + .envs(spec.envs.iter().map(|(k, v)| (k.as_str(), v.as_str()))) + .stderr(Stdio::from(stderr)) + .process_group(0); + let stream_capture = match spec.io { + StageIo::Capture => { + agent_cmd.stdin(Stdio::null()).stdout(Stdio::from(capture)); + None + } + StageIo::Stream { .. } => { + agent_cmd.stdin(Stdio::piped()).stdout(Stdio::piped()); + Some(capture) + } + }; + Ok((agent_cmd.spawn()?, stream_capture)) +} + /// Require that `pid` leads its own process group: `getpgid(pid) == pid`. /// A just-spawned child's `setpgid` has normally landed by the time `spawn` /// returns; if it has not, re-read `/proc//stat` field 5 (falling back From 805a7593823325d78007f5639f64f2eabce882d6 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 14:15:42 -0400 Subject: [PATCH 032/112] test(50-08): add failing stream-shape, own-group and liveness tests - stream_arm_runs_the_agent_under_an_addressed_supervisor: an Antigravity fake reads its first turn, prints its result line and completes with 0 under an addressed supervisor - both_arms_put_the_agent_in_its_own_group (V-10) with a control that a process left in the launcher's group fails the same check - both_arms_answer_liveness_identically_for_each_condition (V-09): running, supervisor SIGKILLed and finished, per shape, with a label control - all three fail on the stream shape's NotReady Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/monitor.rs | 300 +++++++++++++++++++++++++++++ 1 file changed, 300 insertions(+) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index ce81bd37..60172d37 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -4710,6 +4710,8 @@ exit 0 } impl Shape { + const BOTH: [Shape; 2] = [Shape::Capture, Shape::Stream]; + fn label(self) -> &'static str { self.io().label() } @@ -4731,6 +4733,65 @@ exit 0 other => panic!("unknown shape {other}"), } } + + /// The other shape, for the label control. + fn other(self) -> Self { + match self { + Shape::Capture => Shape::Stream, + Shape::Stream => Shape::Capture, + } + } + + /// The non-Claude adapter each shape runs in these tests (R-11): + /// Codex under capture, Antigravity under stream. + fn adapter(self) -> AgentKind { + match self { + Shape::Capture => AgentKind::Codex, + Shape::Stream => AgentKind::Antigravity, + } + } + + /// The agent script's first lines: the stream shape reads its first + /// turn from stdin before anything else, as a real stream agent does, + /// and exits 91 or 92 when the turn is missing or lacks the prompt. + fn preamble(self) -> &'static str { + match self { + Shape::Capture => "", + Shape::Stream => { + "IFS= read -r turn || { echo NO_FIRST_TURN >&2; exit 91; }\n\ + case \"$turn\" in *SUPERVISED-STAGE-PROMPT*) ;; *) echo BAD_TURN >&2; exit 92 ;; esac\n" + } + } + } + + /// The result line the shape's adapter prints on stdout. + fn result_line(self) -> &'static str { + match self { + Shape::Capture => r#"DEVFLOW_RESULT: {"status":"success"}"#, + Shape::Stream => ANTG_RESULT_MARKER_LINE, + } + } + } + + /// The stderr line every waiting agent prints. + const STAGE_STDERR_LINE: &str = "STAGE-STDERR-LINE"; + + /// A fake agent for `shape`, named after its adapter: it reads its first + /// turn (stream), prints the shape's result line on stdout and + /// [`STAGE_STDERR_LINE`] on stderr, then waits for `go` and exits `code`. + fn waiting_agent(fixture: &StageFixture, shape: Shape, go: &Path, code: i32) -> PathBuf { + fixture.agent( + &shape.adapter().to_string(), + &format!( + "{preamble}printf '%s\\n' '{result}'\n\ + echo '{STAGE_STDERR_LINE}' >&2\n\ + while [ ! -e '{go}' ]; do sleep 0.05; done\n\ + exit {code}", + preamble = shape.preamble(), + result = shape.result_line(), + go = go.display(), + ), + ) } /// The child half of every supervised-stage test: build the spec from @@ -4870,6 +4931,18 @@ exit 0 struct StageChild(Option); impl StageChild { + fn id(&self) -> u32 { + self.0.as_ref().expect("the child is still owned").id() + } + + /// SIGKILL the child (by the pid of the `Child` this test owns) and + /// reap it. + fn kill_now(&mut self) { + let mut child = self.0.take().expect("the child is still owned"); + child.kill().expect("SIGKILL the child supervisor"); + child.wait().expect("reap the child supervisor"); + } + fn try_exited(&mut self) -> bool { self.0 .as_mut() @@ -5071,4 +5144,231 @@ exit 0 ); drop(run.group); } + + const STREAM_TRACER_TEST: &str = + "monitor::tests::stream_arm_runs_the_agent_under_an_addressed_supervisor"; + + #[test] + fn stream_arm_runs_the_agent_under_an_addressed_supervisor() { + if crate::test_support::in_child_test(STREAM_TRACER_TEST) { + supervise_stage_in_child(); + return; + } + let fixture = StageFixture::new(); + let go = fixture.path("go"); + let program = waiting_agent(&fixture, Shape::Stream, &go, 0); + let mut run = launch_stage( + STREAM_TRACER_TEST, + &fixture, + Shape::Stream, + AgentKind::Antigravity, + &program, + "", + ); + assert_eq!(run.reply.launch, "stream"); + assert_eq!(run.reply.owner, "agent_running"); + assert_eq!(run.sidecar.launch, "stream"); + assert_eq!(run.sidecar.agent, "antigravity"); + assert_eq!(run.sidecar.agent_pid, run.agent_pid()); + + std::fs::write(&go, "").expect("let the agent exit"); + let outcome = run.finish(STREAM_TRACER_TEST); + let stderr = fixture.stderr(); + assert_eq!( + outcome, "Completed(0) advancing", + "91/92 mean the first turn was missing or lacked the prompt; stderr: {stderr:?}" + ); + let stdout = fixture.stdout(); + assert!( + stdout.lines().any(|line| line == ANTG_RESULT_MARKER_LINE), + "the capture holds the stream's result line verbatim: {stdout:?}" + ); + assert!(!stdout.contains(STAGE_STDERR_LINE), "stdout: {stdout:?}"); + assert!(stderr.contains(STAGE_STDERR_LINE), "stderr: {stderr:?}"); + assert_eq!(fixture.pid_file(), run.agent_pid().to_string()); + assert_eq!(fixture.exit_code(), "0"); + assert_eq!(run.liveness(), crate::supervisor::Liveness::Gone); + drop(run.group); + } + + /// `Ok` when `agent_pid` leads its own process group and that group is + /// not `supervisor_pid`'s; the reason otherwise. Uses the production + /// [`confirm_own_group`] check for the first half. + fn check_own_group(agent_pid: u32, supervisor_pid: u32) -> Result<(), String> { + confirm_own_group(agent_pid) + .map_err(|pgid| format!("agent {agent_pid} is in group {pgid}, not its own"))?; + let supervisor = libc::pid_t::try_from(supervisor_pid).expect("pid fits pid_t"); + // SAFETY: getpgid has no memory preconditions. + let supervisor_group = unsafe { libc::getpgid(supervisor) }; + if i64::from(supervisor_group) == i64::from(agent_pid) { + return Err(format!( + "agent {agent_pid} shares its group with supervisor {supervisor_pid}" + )); + } + Ok(()) + } + + const OWN_GROUP_TEST: &str = "monitor::tests::both_arms_put_the_agent_in_its_own_group"; + + #[test] + fn both_arms_put_the_agent_in_its_own_group() { + if crate::test_support::in_child_test(OWN_GROUP_TEST) { + supervise_stage_in_child(); + return; + } + for shape in Shape::BOTH { + let fixture = StageFixture::new(); + let go = fixture.path("go"); + let program = waiting_agent(&fixture, shape, &go, 0); + let mut run = launch_stage( + OWN_GROUP_TEST, + &fixture, + shape, + shape.adapter(), + &program, + "", + ); + let agent = run.agent_pid(); + check_own_group(agent, run.child.id()) + .unwrap_or_else(|reason| panic!("{shape:?}: {reason}")); + let agent_signed = libc::pid_t::try_from(agent).expect("pid fits pid_t"); + let child_signed = libc::pid_t::try_from(run.child.id()).expect("pid fits pid_t"); + // SAFETY: getpgid has no memory preconditions. + let (agent_group, child_group) = + unsafe { (libc::getpgid(agent_signed), libc::getpgid(child_signed)) }; + assert_eq!(agent_group, agent_signed, "{shape:?}: pgid == pid"); + assert_ne!( + agent_group, child_group, + "{shape:?}: the agent's group is not the supervisor's" + ); + std::fs::write(&go, "").expect("let the agent exit"); + assert_eq!(run.finish(OWN_GROUP_TEST), "Completed(0) advancing"); + drop(run.group); + } + + // Control: the same check refuses a process spawned without + // `process_group(0)`, which stays in this test process's group. + let mut stray = StageChild(Some( + std::process::Command::new("/bin/sleep") + .arg("30") + .stdin(std::process::Stdio::null()) + .spawn() + .expect("spawn a process in this test's group"), + )); + let verdict = check_own_group(stray.id(), std::process::id()); + stray.kill_now(); + assert!( + verdict.is_err(), + "control: a process left in the launcher's group must fail the check" + ); + } + + /// What one (shape, condition) row of V-09 must probe as. + #[derive(Debug, Clone, Copy)] + enum ExpectedLiveness { + Alive(&'static str), + Stale, + Gone, + } + + /// `Ok` when `got` is exactly `expected` (an Alive reply must carry the + /// expected launch label); the mismatch otherwise. + fn liveness_row( + expected: ExpectedLiveness, + got: &crate::supervisor::Liveness, + ) -> Result<(), String> { + use crate::supervisor::Liveness; + match (expected, got) { + (ExpectedLiveness::Alive(label), Liveness::Alive(reply)) if reply.launch == label => { + Ok(()) + } + (ExpectedLiveness::Stale, Liveness::Stale) + | (ExpectedLiveness::Gone, Liveness::Gone) => Ok(()), + _ => Err(format!("expected {expected:?}, got {got:?}")), + } + } + + /// The condition a V-09 row puts the supervisor in before probing. + #[derive(Debug, Clone, Copy)] + enum Condition { + /// The agent is still running. + Running, + /// The supervisor child was SIGKILLed; its agent may live on. + SupervisorKilled, + /// The agent exited and the supervisor finished. + Finished, + } + + const LIVENESS_TEST: &str = + "monitor::tests::both_arms_answer_liveness_identically_for_each_condition"; + + /// Launch `shape`, put it in `condition`, and probe its socket. + fn probe_under(shape: Shape, condition: Condition) -> crate::supervisor::Liveness { + let fixture = StageFixture::new(); + let go = fixture.path("go"); + let program = waiting_agent(&fixture, shape, &go, 0); + let mut run = launch_stage( + LIVENESS_TEST, + &fixture, + shape, + shape.adapter(), + &program, + "", + ); + match condition { + Condition::Running => { + let got = run.liveness(); + std::fs::write(&go, "").expect("let the agent exit"); + assert_eq!(run.finish(LIVENESS_TEST), "Completed(0) advancing"); + got + } + Condition::SupervisorKilled => { + // The `Child` the parent spawned, never a pid from disk. The + // agent outlives it in its own group; `run.group` kills it. + run.child.kill_now(); + run.liveness() + } + Condition::Finished => { + std::fs::write(&go, "").expect("let the agent exit"); + assert_eq!(run.finish(LIVENESS_TEST), "Completed(0) advancing"); + run.liveness() + } + } + } + + #[test] + fn both_arms_answer_liveness_identically_for_each_condition() { + if crate::test_support::in_child_test(LIVENESS_TEST) { + supervise_stage_in_child(); + return; + } + let mut failures = Vec::new(); + for shape in Shape::BOTH { + for condition in [ + Condition::Running, + Condition::SupervisorKilled, + Condition::Finished, + ] { + let expected = match condition { + Condition::Running => ExpectedLiveness::Alive(shape.label()), + Condition::SupervisorKilled => ExpectedLiveness::Stale, + Condition::Finished => ExpectedLiveness::Gone, + }; + let got = probe_under(shape, condition); + if let Err(reason) = liveness_row(expected, &got) { + failures.push(format!("{shape:?} {condition:?}: {reason}")); + } + // Control: an Alive reply does not satisfy the other shape's + // label, so the label column is really checked. + if let Condition::Running = condition { + assert!( + liveness_row(ExpectedLiveness::Alive(shape.other().label()), &got).is_err(), + "control: {shape:?}'s Alive reply must fail the {:?} label", + shape.other() + ); + } + } + } + assert!(failures.is_empty(), "rows that probed wrong: {failures:#?}"); + } } From 223550aeb698fce1a1945a3623312339de1d5a99 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 14:17:36 -0400 Subject: [PATCH 033/112] feat(50-08): run the stream shape on the same supervisor body - run_supervised_stage drives the stream shape's pipes through drive_stream_io (first turn, capture tee, close rule, idle timeout) and then waits through Supervisor::wait_agent instead of child.wait() - an idle-timeout group kill stays a natural Completed(128 + n) - both shapes now put the agent in its own group and answer liveness identically (V-10, V-09) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/monitor.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index 60172d37..32c76831 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -1398,11 +1398,6 @@ const PGID_SETTLE_WAIT: Duration = Duration::from_millis(100); /// The caller owns [`Supervisor::hand_off`] and [`Supervisor::finish`], and /// advances only on [`AgentOutcome::Completed`]. pub fn run_supervised_stage(spec: StageSpec) -> Result { - if let StageIo::Stream { .. } = spec.io { - return Err(MonitorError::NotReady( - "stream shape lands in task 2".to_string(), - )); - } crate::supervisor::runtime::install_signal_pipe()?; let stdout_file = crate::agent_result::stdout_path(&spec.project_root, spec.phase); From 1956ebfac48c2a5e9956c55d71865694bb69ce9e Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 14:23:48 -0400 Subject: [PATCH 034/112] test(50-08): prove stop semantics and the R-A..R-M table for both shapes - V-16: SIGTERM and SIGINT to the supervisor, per shape, end the agent group, write 143, report Stopped and leave the socket Gone; control: a TERM straight to the agent is a natural Completed(143) - V-08: one checker per responsibility (R-A cwd, R-B streams, R-C pid, R-D exit and self-signal, R-E completion, R-F adapter env in a grandchild, R-J no-state sweep, R-K, R-L modes, R-M, SigBlk) under a fake codex (capture) and a fake antigravity (stream); control: the R-B checker rejects a capture whose agent merged stderr into stdout - sigterm_in_mask's parser is now signal_mask, shared with the SigBlk row Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/monitor.rs | 471 ++++++++++++++++++++++++++++- 1 file changed, 469 insertions(+), 2 deletions(-) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index 32c76831..de7d5192 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -3500,6 +3500,14 @@ pid_t fork(void) { /// the field cannot be read: an absent field must not read as "clear". #[cfg(target_os = "linux")] fn sigterm_in_mask(pid: u32, field: &str) -> bool { + signal_mask(pid, field) & (1 << (libc::SIGTERM - 1)) != 0 + } + + /// The named signal mask of `pid`'s `/proc//status` (`SigCgt`, + /// `SigBlk`, ...). Panics when the field cannot be read: an absent field + /// must not read as "clear". + #[cfg(target_os = "linux")] + fn signal_mask(pid: u32, field: &str) -> u64 { let status = std::fs::read_to_string(format!("/proc/{pid}/status")) .expect("/proc//status must be readable for a live child"); let hex = status @@ -3507,8 +3515,7 @@ pid_t fork(void) { .find_map(|line| line.strip_prefix(field)?.strip_prefix(':')) .unwrap_or_else(|| panic!("/proc/{pid}/status has no {field} line")) .trim(); - let mask = u64::from_str_radix(hex, 16).expect("signal masks are hex"); - mask & (1 << (libc::SIGTERM - 1)) != 0 + u64::from_str_radix(hex, 16).expect("signal masks are hex") } #[cfg(target_os = "linux")] @@ -4930,6 +4937,17 @@ exit 0 self.0.as_ref().expect("the child is still owned").id() } + /// Send `signal` to the child, by the pid of the `Child` this test + /// owns. + fn signal(&self, signal: libc::c_int) { + let pid = libc::pid_t::try_from(self.id()).expect("pid fits pid_t"); + assert!(pid > 1, "refusing to signal pid {pid}"); + // SAFETY: `pid > 1` names one process this test spawned and still + // owns unreaped. + let rc = unsafe { libc::kill(pid, signal) }; + assert_eq!(rc, 0, "signal {signal} to the child supervisor {pid}"); + } + /// SIGKILL the child (by the pid of the `Child` this test owns) and /// reap it. fn kill_now(&mut self) { @@ -5366,4 +5384,453 @@ exit 0 } assert!(failures.is_empty(), "rows that probed wrong: {failures:#?}"); } + + /// Whether any member of process group `pgid` still exists (zombies + /// included). `EPERM` means a member exists that we may not signal. + fn stage_group_exists(pgid: u32) -> bool { + let signed = libc::pid_t::try_from(pgid).expect("pgid fits pid_t"); + assert!(signed > 1, "refusing to probe process group {signed}"); + // SAFETY: `signed > 1`; signal 0 only checks for existence. + let rc = unsafe { libc::kill(-signed, 0) }; + rc == 0 || std::io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) + } + + /// A fake agent for `shape` that leaves a second group member running + /// (`sleep 60`, its stdio detached so it holds no capture pipe), then + /// waits for a file that never appears. Only a signal ends it. + fn group_agent(fixture: &StageFixture, shape: Shape) -> PathBuf { + fixture.agent( + &shape.adapter().to_string(), + &format!( + "{preamble}sleep 60 /dev/null 2>&1 &\n\ + printf '%s\\n' '{result}'\n\ + while [ ! -e '{never}' ]; do sleep 0.05; done\n\ + exit 0", + preamble = shape.preamble(), + result = shape.result_line(), + never = fixture.path("never").display(), + ), + ) + } + + const STOP_SIGNAL_TEST: &str = "monitor::tests::sigterm_to_the_supervisor_kills_the_agent_group_records_143_and_does_not_advance"; + + /// V-16. Do NOT widen the 5 s windows below to make this pass: a missed + /// TERM or INT is a defect, not slowness. + #[test] + fn sigterm_to_the_supervisor_kills_the_agent_group_records_143_and_does_not_advance() { + if crate::test_support::in_child_test(STOP_SIGNAL_TEST) { + supervise_stage_in_child(); + return; + } + for shape in Shape::BOTH { + for (name, signal) in [("SIGTERM", libc::SIGTERM), ("SIGINT", libc::SIGINT)] { + let fixture = StageFixture::new(); + let program = group_agent(&fixture, shape); + let mut run = launch_stage( + STOP_SIGNAL_TEST, + &fixture, + shape, + shape.adapter(), + &program, + "", + ); + let pgid = run.reply.agent_pgid; + assert!( + stage_group_exists(pgid), + "{shape:?} {name}: control: the agent group runs before the signal" + ); + + // The supervisor child's own pid, from the `Child` we own. + run.child.signal(signal); + let outcome = run.finish(STOP_SIGNAL_TEST); + assert_eq!( + outcome, "Stopped finished", + "{shape:?} {name}: a signal to the supervisor is a stop, never a completion" + ); + assert!( + stage_wait_until(Duration::from_secs(5), || !stage_group_exists(pgid)), + "{shape:?} {name}: every member of the agent group must be gone: {}", + proc_snapshot(run.agent_pid()) + ); + assert_eq!(fixture.exit_code(), "143", "{shape:?} {name}"); + assert_eq!( + run.liveness(), + crate::supervisor::Liveness::Gone, + "{shape:?} {name}: the socket is gone after the stop" + ); + drop(run.group); + } + + // Control: SIGTERM straight to the agent, not the supervisor. The + // agent dies of it (the test observes a real death), and the + // supervisor reports that as a natural completion that advances. + let fixture = StageFixture::new(); + let program = group_agent(&fixture, shape); + let mut run = launch_stage( + STOP_SIGNAL_TEST, + &fixture, + shape, + shape.adapter(), + &program, + "", + ); + let agent = libc::pid_t::try_from(run.agent_pid()).expect("pid fits pid_t"); + assert!(agent > 1, "refusing to signal pid {agent}"); + // SAFETY: `agent > 1` is the live agent the child supervisor + // still holds unreaped (it is waiting on it). + assert_eq!(unsafe { libc::kill(agent, libc::SIGTERM) }, 0); + assert_eq!( + run.finish(STOP_SIGNAL_TEST), + "Completed(143) advancing", + "{shape:?} control: a TERM to the agent alone is not a supervisor stop" + ); + assert_eq!(fixture.exit_code(), "143", "{shape:?} control"); + drop(run.group); + } + } + + /// One checker per supervisor responsibility of the spike's R-A..R-M + /// table (V-08), each returning the violation it found. Linux-only: + /// they read `/proc`. + #[cfg(target_os = "linux")] + mod responsibilities { + use std::os::unix::fs::PermissionsExt; + use std::path::Path; + + /// R-A: the agent's cwd is the worktree, not the project root. + pub(super) fn r_a_cwd(agent_pid: u32, workdir: &Path) -> Result<(), String> { + let cwd = std::fs::read_link(format!("/proc/{agent_pid}/cwd")) + .map_err(|err| format!("agent cwd unreadable: {err}"))?; + let want = workdir.canonicalize().map_err(|err| err.to_string())?; + (cwd == want) + .then_some(()) + .ok_or_else(|| format!("agent cwd {} is not {}", cwd.display(), want.display())) + } + + /// R-B: stdout and stderr land in separate files, and stderr never + /// reaches the stdout capture. + pub(super) fn r_b_streams( + stdout: &str, + stderr: &str, + result_line: &str, + stderr_line: &str, + ) -> Result<(), String> { + if !stdout.lines().any(|line| line == result_line) { + return Err(format!("stdout lacks the result line: {stdout:?}")); + } + if stdout.contains(stderr_line) { + return Err(format!("stderr text reached stdout: {stdout:?}")); + } + if !stderr.contains(stderr_line) { + return Err(format!("stderr lacks its line: {stderr:?}")); + } + Ok(()) + } + + /// R-C: the pid file names the agent, and that pid runs the adapter + /// program, not the supervisor. + pub(super) fn r_c_pid( + pid_file: &str, + agent_pid: u32, + program: &Path, + ) -> Result<(), String> { + if pid_file != agent_pid.to_string() { + return Err(format!("pid file {pid_file:?} is not agent {agent_pid}")); + } + let cmdline = std::fs::read(format!("/proc/{agent_pid}/cmdline")) + .map_err(|err| format!("agent cmdline unreadable: {err}"))?; + let cmdline = String::from_utf8_lossy(&cmdline).replace('\0', " "); + cmdline + .contains(&*program.to_string_lossy()) + .then_some(()) + .ok_or_else(|| format!("agent {agent_pid} runs {cmdline:?}, not the adapter")) + } + + /// R-D: the exit file holds the exact code (`128 + n` for signal n). + pub(super) fn r_d_exit(exit_file: &str, code: i32) -> Result<(), String> { + (exit_file == code.to_string()) + .then_some(()) + .ok_or_else(|| format!("exit file {exit_file:?}, want {code}")) + } + + /// R-E: natural completion is `Completed(code)` and the run is left + /// advancing. + pub(super) fn r_e_completes(outcome: &str, code: i32) -> Result<(), String> { + (outcome == format!("Completed({code}) advancing")) + .then_some(()) + .ok_or_else(|| format!("outcome {outcome:?} is not a completion with {code}")) + } + + /// R-F: the adapter variable reached a grandchild of the agent. + pub(super) fn r_f_env(recorded: Option<&str>, value: &str) -> Result<(), String> { + (recorded == Some(value)) + .then_some(()) + .ok_or_else(|| format!("grandchild saw {recorded:?}, want {value:?}")) + } + + /// R-J: a sweep for the root and phase finds the live launch with no + /// state file on disk. + pub(super) fn r_j_sweep( + cache: &Path, + root: &Path, + phase: crate::phase_id::PhaseId, + launch_id: &str, + ) -> Result<(), String> { + if crate::workflow::state_path(root, phase).exists() { + return Err("a state file exists, so this does not test the no-state sweep".into()); + } + let found = crate::supervisor::sweep_for_in(cache, root, phase); + let ours: Vec<_> = found + .iter() + .filter(|entry| { + entry + .sidecar + .as_ref() + .is_some_and(|sidecar| sidecar.launch_id == launch_id) + }) + .collect(); + match ours.as_slice() { + [entry] if matches!(entry.liveness, Ok(crate::supervisor::Liveness::Alive(_))) => { + Ok(()) + } + _ => Err(format!("sweep found {found:?}, want our one Alive launch")), + } + } + + /// R-K: a stopped run still records an exit code, 143. + pub(super) fn r_k_stop_exit(exit_file: &str) -> Result<(), String> { + r_d_exit(exit_file, 143) + } + + /// R-L: the socket is 0600 and the socket directory 0700. + pub(super) fn r_l_modes(socket: &Path) -> Result<(), String> { + let mode = |path: &Path| { + std::fs::metadata(path) + .map(|meta| meta.permissions().mode() & 0o777) + .map_err(|err| format!("{}: {err}", path.display())) + }; + let socket_mode = mode(socket)?; + let dir_mode = mode(socket.parent().ok_or("socket has no parent")?)?; + (socket_mode == 0o600 && dir_mode == 0o700) + .then_some(()) + .ok_or_else(|| format!("socket {socket_mode:o}, sock/ {dir_mode:o}")) + } + + /// R-M: a stop is not a completion: the run ended `Stopped` and + /// finished, so nothing advances. + pub(super) fn r_m_stop_is_not_completion(outcome: &str) -> Result<(), String> { + (outcome == "Stopped finished") + .then_some(()) + .ok_or_else(|| format!("a stopped run reported {outcome:?}")) + } + + /// The agent was handed no blocked signal: its leader (a shell, which + /// blocks SIGCHLD itself while it waits) has no TERM or INT bit in + /// `SigBlk`, and a child the shell runs, which inherits the mask the + /// agent was handed, has an empty `SigBlk`. + pub(super) fn sigblk_clear(agent_pid: u32, grandchild_pid: u32) -> Result<(), String> { + let stop_bits = (1_u64 << (libc::SIGTERM - 1)) | (1_u64 << (libc::SIGINT - 1)); + let leader = super::signal_mask(agent_pid, "SigBlk"); + let inherited = super::signal_mask(grandchild_pid, "SigBlk"); + (leader & stop_bits == 0 && inherited == 0) + .then_some(()) + .ok_or_else(|| format!("SigBlk leader {leader:016x}, inherited {inherited:016x}")) + } + } + + const RESPONSIBILITIES_TEST: &str = + "monitor::tests::both_arms_meet_every_supervisor_responsibility_with_a_non_claude_adapter"; + + /// The adapter variable R-F threads: one `hermetic_command` scrubs, so + /// it only reaches the agent when `envs` is applied after the scrub. + const R_F_VARIABLE: &str = "GIT_CONFIG_COUNT"; + const R_F_VALUE: &str = "1"; + + /// V-08 (R-11): the spike's responsibilities under both shapes, with a + /// fake `codex` under capture and a fake `antigravity` under stream. + /// R-G (detached from the spawning CLI) is proved in 50-11, where + /// `spawn_monitor` gives the supervisor its own group. + #[cfg(target_os = "linux")] + #[test] + fn both_arms_meet_every_supervisor_responsibility_with_a_non_claude_adapter() { + use responsibilities as r; + if crate::test_support::in_child_test(RESPONSIBILITIES_TEST) { + supervise_stage_in_child(); + return; + } + let mut rows: Vec<(Shape, &str, Result<(), String>)> = Vec::new(); + let adapter_env = format!("{R_F_VARIABLE}={R_F_VALUE}"); + for shape in Shape::BOTH { + // Run 1: a natural exit 7, observed alive first. + let fixture = StageFixture::new(); + let (go, env_out, gc_pid) = ( + fixture.path("go"), + fixture.path("grandchild-env"), + fixture.path("grandchild-pid"), + ); + let program = fixture.agent( + &shape.adapter().to_string(), + &format!( + "{preamble}sh -c 'printf %s \"${R_F_VARIABLE}\" > \"$1\"; exec sleep 60' sh '{env_out}' /dev/null 2>&1 &\n\ + echo $! > '{gc_pid}'\n\ + printf '%s\\n' '{result}'\n\ + echo '{STAGE_STDERR_LINE}' >&2\n\ + while [ ! -e '{go}' ]; do sleep 0.05; done\n\ + kill \"$(cat '{gc_pid}')\"\n\ + exit 7", + preamble = shape.preamble(), + result = shape.result_line(), + env_out = env_out.display(), + gc_pid = gc_pid.display(), + go = go.display(), + ), + ); + let mut run = launch_stage( + RESPONSIBILITIES_TEST, + &fixture, + shape, + shape.adapter(), + &program, + &adapter_env, + ); + let agent = run.agent_pid(); + assert!( + stage_wait_until(Duration::from_secs(5), || env_out.exists() + && gc_pid.exists()), + "{shape:?}: the agent's grandchild started" + ); + let grandchild: u32 = fixture + .read(gc_pid.clone()) + .trim() + .parse() + .expect("grandchild pid"); + rows.push((shape, "R-A", r::r_a_cwd(agent, &fixture.workdir))); + rows.push(( + shape, + "R-C", + r::r_c_pid(&fixture.pid_file(), agent, &program), + )); + rows.push(( + shape, + "R-F", + r::r_f_env(std::fs::read_to_string(&env_out).ok().as_deref(), R_F_VALUE), + )); + rows.push(( + shape, + "R-J", + r::r_j_sweep( + fixture.cache.path(), + &fixture.root, + STAGE_PHASE, + &run.sidecar.launch_id, + ), + )); + rows.push((shape, "R-L", r::r_l_modes(run.socket.as_path()))); + rows.push((shape, "SigBlk", r::sigblk_clear(agent, grandchild))); + std::fs::write(&go, "").expect("let the agent exit"); + let outcome = run.finish(RESPONSIBILITIES_TEST); + rows.push(( + shape, + "R-B", + r::r_b_streams( + &fixture.stdout(), + &fixture.stderr(), + shape.result_line(), + STAGE_STDERR_LINE, + ), + )); + rows.push((shape, "R-D exit", r::r_d_exit(&fixture.exit_code(), 7))); + rows.push((shape, "R-E", r::r_e_completes(&outcome, 7))); + drop(run.group); + + // Run 2: the agent kills itself with SIGUSR1 (128 + 10). Under + // capture it first merges stderr into stdout: the R-B control. + let fixture = StageFixture::new(); + let go = fixture.path("go"); + let merge = if shape == Shape::Capture { + "exec 2>&1\n" + } else { + "" + }; + let program = fixture.agent( + &shape.adapter().to_string(), + &format!( + "{preamble}{merge}printf '%s\\n' '{result}'\n\ + echo '{STAGE_STDERR_LINE}' >&2\n\ + while [ ! -e '{go}' ]; do sleep 0.05; done\n\ + kill -USR1 $$\n\ + sleep 5\n\ + exit 0", + preamble = shape.preamble(), + result = shape.result_line(), + go = go.display(), + ), + ); + let mut run = launch_stage( + RESPONSIBILITIES_TEST, + &fixture, + shape, + shape.adapter(), + &program, + "", + ); + std::fs::write(&go, "").expect("let the agent signal itself"); + let outcome = run.finish(RESPONSIBILITIES_TEST); + let by_signal = 128 + libc::SIGUSR1; + rows.push(( + shape, + "R-D signal", + r::r_d_exit(&fixture.exit_code(), by_signal), + )); + rows.push((shape, "R-E signal", r::r_e_completes(&outcome, by_signal))); + if shape == Shape::Capture { + assert!( + r::r_b_streams( + &fixture.stdout(), + &fixture.stderr(), + shape.result_line(), + STAGE_STDERR_LINE + ) + .is_err(), + "control: the R-B checker must reject a capture whose agent merged stderr \ + into stdout: {:?}", + fixture.stdout() + ); + } + drop(run.group); + + // Run 3: a stop (SIGTERM to the supervisor) still records 143 and + // is not a completion. + let fixture = StageFixture::new(); + let program = group_agent(&fixture, shape); + let mut run = launch_stage( + RESPONSIBILITIES_TEST, + &fixture, + shape, + shape.adapter(), + &program, + "", + ); + run.child.signal(libc::SIGTERM); + let outcome = run.finish(RESPONSIBILITIES_TEST); + rows.push((shape, "R-K", r::r_k_stop_exit(&fixture.exit_code()))); + rows.push((shape, "R-M", r::r_m_stop_is_not_completion(&outcome))); + drop(run.group); + } + let failures: Vec = rows + .iter() + .filter_map(|(shape, row, result)| { + result + .as_ref() + .err() + .map(|reason| format!("{shape:?} {row}: {reason}")) + }) + .collect(); + assert_eq!(rows.len(), 26, "every row ran: {rows:?}"); + assert!( + failures.is_empty(), + "responsibilities not met: {failures:#?}" + ); + } } From 60ba2a76a81b6b3dc46203f08d261445f0a43151 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 14:24:38 -0400 Subject: [PATCH 035/112] test(50-08): gate the V-08 constants to Linux with their test - RESPONSIBILITIES_TEST and the R-F variable were unused on aarch64-apple-darwin, where V-08 is cfg-gated off; the macOS compile check now has no warnings Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/monitor.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index de7d5192..d29ffa8f 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -5639,12 +5639,15 @@ exit 0 } } + #[cfg(target_os = "linux")] const RESPONSIBILITIES_TEST: &str = "monitor::tests::both_arms_meet_every_supervisor_responsibility_with_a_non_claude_adapter"; /// The adapter variable R-F threads: one `hermetic_command` scrubs, so /// it only reaches the agent when `envs` is applied after the scrub. + #[cfg(target_os = "linux")] const R_F_VARIABLE: &str = "GIT_CONFIG_COUNT"; + #[cfg(target_os = "linux")] const R_F_VALUE: &str = "1"; /// V-08 (R-11): the spike's responsibilities under both shapes, with a From 69f2a06adf3d7ef3e6278b0d17a1a87a2d002527 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 14:38:42 -0400 Subject: [PATCH 036/112] test(50-09): add failing tracer test for the stale-socket backstop - V-18: a validated leader-present group behind a STALE socket is killed; control: a start time one tick off is LeaderRecycled - teardown_stale acts only on a STALE socket (Alive, Gone, ProbeError) - backstop public types with stubbed assess/teardown_stale - agent::process_stat_fields (state, pgrp, session, start time) and its parser test, used by the test fixtures Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/agent.rs | 83 +++ crates/devflow-core/src/supervisor.rs | 1 + .../devflow-core/src/supervisor/backstop.rs | 533 ++++++++++++++++++ 3 files changed, 617 insertions(+) create mode 100644 crates/devflow-core/src/supervisor/backstop.rs diff --git a/crates/devflow-core/src/agent.rs b/crates/devflow-core/src/agent.rs index 55f1a5e9..49fe0906 100644 --- a/crates/devflow-core/src/agent.rs +++ b/crates/devflow-core/src/agent.rs @@ -220,6 +220,60 @@ pub fn process_start_time(pid: u32) -> Option { rest.split_whitespace().nth(19)?.parse::().ok() } +/// The `/proc//stat` fields the stale-socket backstop decides on +/// ([`crate::supervisor::backstop`]): one read gives state, process group, +/// session and start time together, so they describe the same instant. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + not(target_os = "linux"), + allow(dead_code, reason = "read only by the Linux backstop (MAC-2)") +)] +pub(crate) struct ProcStat { + /// Field 3, the one-letter state (`R`, `S`, `Z`, ...). + pub(crate) state: char, + /// Field 5, the process group id. + pub(crate) pgrp: i32, + /// Field 6, the session id. + pub(crate) session: i32, + /// Field 22, the start time in clock ticks since boot, as + /// [`process_start_time`] reads it. + pub(crate) start_time: u64, +} + +/// Read [`ProcStat`] for `pid` from one read of `/proc//stat`. `None` +/// when the file cannot be read or parsed (the process is gone, or this is +/// not Linux): the fail-closed direction, as for [`process_start_time`]. +#[cfg_attr( + not(target_os = "linux"), + allow(dead_code, reason = "read only by the Linux backstop (MAC-2)") +)] +pub(crate) fn process_stat_fields(pid: u32) -> Option { + parse_stat_fields(&std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?) +} + +/// [`process_stat_fields`]'s parser. Splits after the FINAL `)`, because +/// field 2 (comm) may contain spaces and parentheses; after that point field +/// 3 is index 0, so field 5 is 2, field 6 is 3 and field 22 is 19. +#[cfg_attr( + not(target_os = "linux"), + allow(dead_code, reason = "read only by the Linux backstop (MAC-2)") +)] +fn parse_stat_fields(stat: &str) -> Option { + let rest = &stat[stat.rfind(')')? + 1..]; + let fields: Vec<&str> = rest.split_whitespace().collect(); + let mut state_chars = fields.first()?.chars(); + let state = state_chars.next()?; + if state_chars.next().is_some() { + return None; + } + Some(ProcStat { + state, + pgrp: fields.get(2)?.parse().ok()?, + session: fields.get(3)?.parse().ok()?, + start_time: fields.get(19)?.parse().ok()?, + }) +} + /// Whether `pid` is the same process instance that recorded `expected_start`. /// /// The identity check `devflow stop` and friends should use. See @@ -1024,4 +1078,33 @@ mod tests { "age must grow monotonically across a sleep, never shrink" ); } + + #[test] + fn stat_fields_parse_after_the_last_paren_and_reject_a_short_line() { + // comm carries spaces and a ") (" so a naive split would misread + // every field after it. Fields after comm: state R, ppid 1, pgrp + // 4242, session 4100, then 15 filler fields, then start time 98765. + let filler = "0 ".repeat(15); + let line = format!("4242 (a) (b c) R 1 4242 4100 {filler}98765 0 0"); + assert_eq!( + parse_stat_fields(&line), + Some(ProcStat { + state: 'R', + pgrp: 4242, + session: 4100, + start_time: 98_765, + }) + ); + // Control: a line cut before field 22 is not a stat. + assert_eq!(parse_stat_fields("4242 (a) R 1 4242 4100 0 0"), None); + // And the live file agrees with process_start_time for this process. + #[cfg(target_os = "linux")] + { + let me = std::process::id(); + let stat = process_stat_fields(me).expect("read this process's own stat"); + assert_eq!(Some(stat.start_time), process_start_time(me)); + // SAFETY: getpgrp has no preconditions and cannot fail. + assert_eq!(stat.pgrp, unsafe { libc::getpgrp() }); + } + } } diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs index 6b018503..a51cf169 100644 --- a/crates/devflow-core/src/supervisor.rs +++ b/crates/devflow-core/src/supervisor.rs @@ -21,6 +21,7 @@ //! refused is `Stale`; any other surprise at the path is a [`ProbeError`], so //! a stray file can never be mistaken for a dead supervisor (R-6). +pub mod backstop; pub mod protocol; pub mod runtime; diff --git a/crates/devflow-core/src/supervisor/backstop.rs b/crates/devflow-core/src/supervisor/backstop.rs new file mode 100644 index 00000000..bd642f04 --- /dev/null +++ b/crates/devflow-core/src/supervisor/backstop.rs @@ -0,0 +1,533 @@ +//! The stale-socket backstop (D-04, R-4, R-8): what `devflow stop` does +//! when a launch's socket probes [`Liveness::Stale`]. The supervisor is +//! gone, but the agent's process group may still be running. +//! +//! This code signals a process group by a value read from disk, so it is +//! split in two. [`classify`] is a pure decision over gathered facts +//! ([`BackstopFacts`]). [`teardown_stale`] is a thin shell that acts on it. +//! The rule, in order: +//! +//! 0. A recorded pgid that does not fit `pid_t`, is `<= 1`, or is the +//! caller's own process group is [`BackstopDecision::RefuseUnsafeTarget`], +//! decided before any `/proc` read. +//! 1. Where process identity cannot be proved (off Linux, or a boot id or +//! process table that cannot be read), the answer is +//! [`BackstopDecision::RefuseIdentityUnavailable`] (MAC-2). +//! 2. The sidecar and the state record (when given) must agree on launch id, +//! pgid, agent start time and boot id, or it is +//! [`BackstopDecision::RefuseMismatch`]. +//! 3. A sidecar from another boot is [`BackstopDecision::PreviousBoot`]. +//! 4. No live (non-zombie) process of this uid in the group is +//! [`BackstopDecision::EmptyGroup`]. +//! 5. A leader (pid == pgid) with the recorded start time that still leads +//! the group is [`BackstopDecision::LeaderPresentSignal`]. +//! 6. A leader pid held by a process with another start time is +//! [`BackstopDecision::LeaderRecycled`]: the kernel does not reuse a pid +//! while it names a process group (RESEARCH A1), so the original group is +//! gone. +//! 7. Otherwise the leader is gone (or has left the group) while members +//! live: [`BackstopDecision::RefuseLeaderGone`]. They are reported, never +//! signalled, because nothing proves they are the agent's (the residual +//! F1 limit). +//! +//! Only `LeaderPresentSignal` signals. On Linux the signal goes through a +//! pidfd opened on the leader, with the start time re-read after opening, and +//! `PIDFD_SIGNAL_PROCESS_GROUP`. A kernel without that flag (`EINVAL`) falls +//! back to validate-then-`killpg` ([`SignalPath::ValidatedKillpg`]). Off Linux +//! no signalling path exists. + +#[cfg(test)] +use super::Liveness; +use super::{Sidecar, SocketPath}; +use crate::state::SupervisorRecord; +use std::io; + +/// One live process of the recorded process group, as gathered from +/// `/proc`. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct GroupMember { + /// The member's pid. + pub pid: u32, + /// Its start time in clock ticks since boot. + pub start_time: u64, + /// Its one-letter `/proc` state. + pub state: char, + /// Its session id. + pub session: i32, + /// Its command line, arguments joined by spaces (empty when unreadable). + pub cmdline: String, +} + +/// What the backstop decides for a STALE socket. See the module doc for the +/// order. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum BackstopDecision { + /// The recorded pgid does not fit `pid_t`, is 0 or 1, or is the + /// caller's own process group. Nothing is signalled or removed. + RefuseUnsafeTarget, + /// This platform gives DevFlow no process start time, group census or + /// boot id (off Linux; on Linux, an unreadable boot id or process table), + /// so the group's identity cannot be proved. Callers treat it like the + /// other `Refuse*` decisions: signal nothing, keep the socket and + /// sidecar, and count the run as possibly live (fail closed). + RefuseIdentityUnavailable, + /// The sidecar and the state record disagree; the text names the field. + /// Nothing is signalled or removed. + RefuseMismatch(String), + /// The sidecar was written in a previous boot, so nothing it names can + /// be running. The socket and sidecar are removed; nothing is signalled. + PreviousBoot, + /// No live process of this user is in the recorded group. The socket and + /// sidecar are removed; nothing is signalled. + EmptyGroup, + /// The leader is the recorded agent and still leads the group: the group + /// is signalled (SIGTERM, then SIGKILL). + LeaderPresentSignal, + /// The leader pid now names a different process, so the recorded group + /// is gone. The socket and sidecar are removed; nothing is signalled. + LeaderRecycled, + /// The leader is gone (or has left the group) but these members live. + /// They are reported, never signalled, and the socket and sidecar are + /// kept. + RefuseLeaderGone(Vec), +} + +/// How a group was signalled. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum SignalPath { + /// `pidfd_send_signal` with `PIDFD_SIGNAL_PROCESS_GROUP` on a pidfd of + /// the validated leader. + PidfdGroup, + /// The kernel lacks `PIDFD_SIGNAL_PROCESS_GROUP` (`EINVAL`): the leader + /// was re-validated through its pidfd and `/proc`, then `killpg`. + ValidatedKillpg, +} + +/// What [`teardown_stale`] did. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum BackstopOutcome { + /// The group was signalled and is empty; the socket and sidecar are + /// removed. `members` is the group as gathered before signalling. + Killed { + /// How the group was signalled. + via: SignalPath, + /// The group's live members before the signal. + members: Vec, + }, + /// Nothing was signalled and the socket and sidecar were removed + /// (`PreviousBoot`, `EmptyGroup` or `LeaderRecycled`). + RemovedNothingSignalled(BackstopDecision), + /// The socket and sidecar were kept. Nothing was signalled, except in + /// one case: after the SIGTERM, the leader exited and members survived + /// it, which reads `RefuseLeaderGone` and those members got that one + /// SIGTERM. + Refused(BackstopDecision), +} + +/// Why [`teardown_stale`] did not reach a decision. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum BackstopError { + /// The socket did not probe STALE (it is alive, gone, or the probe + /// failed); the backstop only acts on a STALE socket. + #[error("the socket is not stale: {0}")] + NotStale(String), + /// Reading, signalling or removing failed. + #[error("the stale-socket backstop failed: {0}")] + Io(#[from] io::Error), +} + +/// Decide what the backstop would do for this launch, without acting: for +/// read-only callers (doctor, the launch guard) that classify a STALE socket. +pub fn assess(_sidecar: &Sidecar, _state_record: Option<&SupervisorRecord>) -> BackstopDecision { + // RED stub (50-09 Task 1): the decision rule lands in GREEN. + BackstopDecision::RefuseIdentityUnavailable +} + +/// Tear down the launch behind a STALE socket. +pub fn teardown_stale( + _socket: &SocketPath, + _sidecar: &Sidecar, + _state_record: Option<&SupervisorRecord>, +) -> Result { + // RED stub (50-09 Task 1): probing, the decision and acting land in GREEN. + Ok(BackstopOutcome::Refused( + BackstopDecision::RefuseIdentityUnavailable, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::phase_id::PhaseId; + use crate::supervisor::{new_launch_id, protocol, publish, write_sidecar}; + use std::path::Path; + #[cfg(target_os = "linux")] + use std::process::{Child, Command, Stdio}; + #[cfg(target_os = "linux")] + use std::time::{Duration, Instant}; + + const ROOT: &str = "/project/root"; + /// A group whose leader stays: it waits on its two sleeping members. + #[cfg(target_os = "linux")] + const LEADER_WAITS: &str = "sleep 30 & sleep 30 & wait"; + #[cfg(target_os = "linux")] + /// How long a fixture may take to reach the state a test needs. + const SETTLE: Duration = Duration::from_secs(5); + #[cfg(target_os = "linux")] + const POLL: Duration = Duration::from_millis(20); + + /// A short cache dir under `/tmp`, so socket paths stay far below + /// `sun_path` however long `$TMPDIR` is. + fn short_cache() -> tempfile::TempDir { + tempfile::Builder::new() + .prefix("dfs") + .tempdir_in("/tmp") + .expect("create a short cache dir under /tmp") + } + + fn socket_for(cache: &Path, launch_id: &str) -> SocketPath { + SocketPath::for_launch(cache, Path::new(ROOT), PhaseId::new(50), launch_id) + .expect("socket path fits sun_path") + } + + /// A STALE socket: published, then its listener dropped. + fn stale_socket(cache: &Path) -> (SocketPath, String) { + let launch_id = new_launch_id(); + let socket = socket_for(cache, &launch_id); + drop(publish(&socket).expect("publish the socket")); + assert!( + matches!(super::super::probe(socket.as_path()), Ok(Liveness::Stale)), + "the fixture socket must probe STALE" + ); + (socket, launch_id) + } + + fn sidecar_for(launch_id: &str, pgid: u32, start_time: u64, boot_id: &str) -> Sidecar { + Sidecar { + launch_id: launch_id.to_string(), + project_root: Path::new(ROOT).to_path_buf(), + phase: PhaseId::new(50), + stage: Some("code".to_string()), + agent: "claude".to_string(), + launch: "capture".to_string(), + agent_pid: pgid, + agent_pgid: pgid, + agent_start_time: start_time, + boot_id: boot_id.to_string(), + supervisor_pid: std::process::id(), + supervisor_start_time: 0, + created_at: super::super::rfc3339_from_unix(1_700_000_000), + } + } + + /// A STALE socket whose sidecar records `pgid`, `start_time` and + /// `boot_id`. + fn stale_launch( + cache: &Path, + pgid: u32, + start_time: u64, + boot_id: &str, + ) -> (SocketPath, Sidecar) { + let (socket, launch_id) = stale_socket(cache); + let sidecar = sidecar_for(&launch_id, pgid, start_time, boot_id); + write_sidecar(&socket, &sidecar).expect("write the sidecar"); + (socket, sidecar) + } + + fn files_exist(socket: &SocketPath) -> (bool, bool) { + (socket.as_path().exists(), socket.sidecar_path().exists()) + } + + #[cfg(target_os = "linux")] + /// Whether the process `(pid, start_time)` is alive (not a zombie). + fn alive(pid: u32, start_time: u64) -> bool { + crate::agent::process_stat_fields(pid) + .is_some_and(|stat| stat.start_time == start_time && stat.state != 'Z') + } + + #[cfg(target_os = "linux")] + /// Live processes in `pgid`, read independently of the code under test. + fn live_in_group(pgid: u32) -> Vec<(u32, u64)> { + let mut found = Vec::new(); + for entry in std::fs::read_dir("/proc").expect("list /proc").flatten() { + let Some(pid) = entry + .file_name() + .to_str() + .and_then(|name| name.parse::().ok()) + else { + continue; + }; + let Some(stat) = crate::agent::process_stat_fields(pid) else { + continue; + }; + if i64::from(stat.pgrp) == i64::from(pgid) && stat.state != 'Z' { + found.push((pid, stat.start_time)); + } + } + found.sort_unstable(); + found + } + + #[cfg(target_os = "linux")] + fn execed_sleep(pid: u32) -> bool { + std::fs::read(format!("/proc/{pid}/cmdline")).is_ok_and(|raw| raw.starts_with(b"sleep\0")) + } + + #[cfg(target_os = "linux")] + /// A process group this test spawned. Bound before any assertion; its + /// drop SIGKILLs the group while the leader is still this test's + /// unreaped child (so the pgid cannot have been reused), then each + /// recorded member by pid only if its start time still matches, then + /// reaps the leader. It signals nothing the test did not create. + struct TestGroup { + leader: Child, + leader_reaped: bool, + pgid: u32, + leader_start: u64, + members: Vec<(u32, u64)>, + } + + #[cfg(target_os = "linux")] + impl TestGroup { + fn spawn(script: &str) -> Self { + use std::os::unix::process::CommandExt; + let leader = Command::new("sh") + .args(["-c", script]) + .process_group(0) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn the test group leader"); + let pgid = leader.id(); + let mut group = Self { + leader, + leader_reaped: false, + pgid, + leader_start: 0, + members: Vec::new(), + }; + group.leader_start = + crate::agent::process_start_time(pgid).expect("read the leader's start time"); + group + } + + /// Wait until the group has `sleepers` sleeping members (plus the + /// leader while it is alive), recording each as it appears. + fn settle(&mut self, sleepers: usize) { + let deadline = Instant::now() + SETTLE; + loop { + let live = live_in_group(self.pgid); + for member in &live { + if !self.members.contains(member) { + self.members.push(*member); + } + } + let sleeping = live.iter().filter(|(pid, _)| execed_sleep(*pid)).count(); + if sleeping == sleepers { + self.members = live; + return; + } + assert!( + Instant::now() < deadline, + "group {} never reached {sleepers} sleeping members: {live:?}", + self.pgid + ); + std::thread::sleep(POLL); + } + } + + fn all_alive(&self) -> bool { + self.members.iter().all(|&(pid, start)| alive(pid, start)) + } + + fn wait_all_gone(&self) { + let deadline = Instant::now() + SETTLE; + while self.members.iter().any(|&(pid, start)| alive(pid, start)) { + assert!( + Instant::now() < deadline, + "group {} members outlived the kill", + self.pgid + ); + std::thread::sleep(POLL); + } + } + } + + #[cfg(target_os = "linux")] + impl Drop for TestGroup { + fn drop(&mut self) { + if !self.leader_reaped + && let Ok(pgid) = libc::pid_t::try_from(self.pgid) + && pgid > 1 + { + // SAFETY: the leader is this test's unreaped child, so its + // pid, and the group it names, cannot have been reused. + unsafe { libc::killpg(pgid, libc::SIGKILL) }; + } + for &(pid, start) in &self.members { + if alive(pid, start) + && let Ok(signed) = libc::pid_t::try_from(pid) + { + // SAFETY: a positive pid this test spawned, start time + // re-checked just above. + unsafe { libc::kill(signed, libc::SIGKILL) }; + } + } + let _ = self.leader.kill(); + let _ = self.leader.wait(); + } + } + + #[cfg(target_os = "linux")] + fn member_ids(members: &[GroupMember]) -> Vec<(u32, u64)> { + let mut ids: Vec<(u32, u64)> = members + .iter() + .map(|member| (member.pid, member.start_time)) + .collect(); + ids.sort_unstable(); + ids + } + + /// V-18 (D-04, R-4): a STALE socket over a group whose leader is + /// present with the recorded start time and boot id is torn down + /// through the leader's pidfd. Control: a recorded start time one tick + /// off is a recycled leader, and nothing is signalled. + #[cfg(target_os = "linux")] + #[test] + fn backstop_kills_a_validated_group_whose_leader_is_present() { + let cache = short_cache(); + let boot = super::super::boot_id().expect("read the boot id"); + let mut group = TestGroup::spawn(LEADER_WAITS); + group.settle(2); + assert_eq!(group.members.len(), 3, "leader plus two sleeping members"); + + // Control: the wrong start time signals nothing. + let (socket, sidecar) = + stale_launch(cache.path(), group.pgid, group.leader_start + 1, &boot); + let outcome = teardown_stale(&socket, &sidecar, None).expect("teardown the control"); + assert_eq!( + outcome, + BackstopOutcome::RemovedNothingSignalled(BackstopDecision::LeaderRecycled), + "a start time one tick off must read as a recycled leader" + ); + std::thread::sleep(Duration::from_millis(200)); + assert!(group.all_alive(), "the control must signal nothing"); + assert_eq!( + files_exist(&socket), + (false, false), + "a recycled leader removes the files" + ); + + // The recorded start time: the group is killed and the files go. + let (socket, sidecar) = stale_launch(cache.path(), group.pgid, group.leader_start, &boot); + let outcome = teardown_stale(&socket, &sidecar, None).expect("teardown the group"); + let BackstopOutcome::Killed { via, members } = outcome else { + panic!("a validated leader-present group must be Killed, got {outcome:?}"); + }; + eprintln!("V-18 signal path: {via:?}"); + assert!(matches!( + via, + SignalPath::PidfdGroup | SignalPath::ValidatedKillpg + )); + assert_eq!( + member_ids(&members), + group.members, + "Killed reports the group as gathered" + ); + group.wait_all_gone(); + assert_eq!( + files_exist(&socket), + (false, false), + "a killed group's files are removed" + ); + } + + /// Answer one ping on `listener` from a thread, as a live supervisor + /// would. + fn answer_one_ping( + listener: std::os::unix::net::UnixListener, + launch_id: String, + ) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let (stream, _) = listener.accept().expect("accept the probe"); + let _ = protocol::read_request(&stream); + let reply = protocol::PingReply { + launch_id, + phase: "50".to_string(), + stage: "code".to_string(), + launch: "capture".to_string(), + owner: "agent_running".to_string(), + agent_pid: 4242, + agent_pgid: 4242, + agent_start_time: 1, + boot_id: super::super::boot_id(), + supervisor_pid: std::process::id(), + }; + protocol::write_reply(&stream, &protocol::Reply::Ping(reply)).expect("write the reply"); + }) + } + + /// `teardown_stale` acts only on a STALE socket: an Alive one, a Gone + /// one and a probe error are `NotStale`, with the files untouched. + /// Control: a STALE socket passes the probe gate and reaches a decision. + #[test] + fn teardown_stale_acts_only_on_a_stale_socket() { + let cache = short_cache(); + + let launch_id = new_launch_id(); + let socket = socket_for(cache.path(), &launch_id); + let listener = publish(&socket).expect("publish the socket"); + let sidecar = sidecar_for(&launch_id, 4242, 1, "boot-test"); + write_sidecar(&socket, &sidecar).expect("write the sidecar"); + let server = answer_one_ping(listener, launch_id); + let outcome = teardown_stale(&socket, &sidecar, None); + server.join().expect("ping server thread"); + assert!( + matches!(outcome, Err(BackstopError::NotStale(_))), + "an Alive socket is NotStale, got {outcome:?}" + ); + assert_eq!( + files_exist(&socket), + (true, true), + "an Alive socket keeps its files" + ); + + let launch_id = new_launch_id(); + let not_a_socket = socket_for(cache.path(), &launch_id); + std::fs::write(not_a_socket.as_path(), b"not a socket").expect("plant a regular file"); + let sidecar = sidecar_for(&launch_id, 4242, 1, "boot-test"); + let outcome = teardown_stale(¬_a_socket, &sidecar, None); + assert!( + matches!(outcome, Err(BackstopError::NotStale(_))), + "a ProbeError is NotStale, got {outcome:?}" + ); + assert!( + not_a_socket.as_path().exists(), + "a probe error removes nothing" + ); + + let gone = socket_for(cache.path(), &new_launch_id()); + assert!(matches!( + teardown_stale(&gone, &sidecar, None), + Err(BackstopError::NotStale(_)) + )); + + // Control: STALE passes the gate (pgid 1 then refuses as unsafe). + let (stale, stale_sidecar) = stale_launch(cache.path(), 1, 1, "boot-test"); + assert_eq!( + teardown_stale(&stale, &stale_sidecar, None).expect("a STALE socket is acted on"), + BackstopOutcome::Refused(BackstopDecision::RefuseUnsafeTarget) + ); + assert_eq!( + files_exist(&stale), + (true, true), + "a refusal keeps the files" + ); + } +} From fb44315b33b3d28a188c7ff9bbb7667fff9530e1 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 14:41:06 -0400 Subject: [PATCH 037/112] feat(50-09): tear down a validated group behind a STALE socket - classify: pure R-4/R-8 rule (unsafe target, identity unavailable, mismatch, previous boot, empty group, leader present, recycled, leader gone), with assess for read-only callers - teardown_stale re-probes, then acts: pidfd on the leader, start time re-read after opening, PIDFD_SIGNAL_PROCESS_GROUP, EINVAL falls back to validate-then-killpg; SIGTERM, wait, SIGKILL - MAC-2: the pidfd path and /proc gathering are Linux only; off Linux facts are unread and identity_provable is false - drop the ProcStat dead-code allows: the census uses it everywhere Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/agent.rs | 12 - .../devflow-core/src/supervisor/backstop.rs | 501 +++++++++++++++++- 2 files changed, 489 insertions(+), 24 deletions(-) diff --git a/crates/devflow-core/src/agent.rs b/crates/devflow-core/src/agent.rs index 49fe0906..e7eef06f 100644 --- a/crates/devflow-core/src/agent.rs +++ b/crates/devflow-core/src/agent.rs @@ -224,10 +224,6 @@ pub fn process_start_time(pid: u32) -> Option { /// ([`crate::supervisor::backstop`]): one read gives state, process group, /// session and start time together, so they describe the same instant. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[cfg_attr( - not(target_os = "linux"), - allow(dead_code, reason = "read only by the Linux backstop (MAC-2)") -)] pub(crate) struct ProcStat { /// Field 3, the one-letter state (`R`, `S`, `Z`, ...). pub(crate) state: char, @@ -243,10 +239,6 @@ pub(crate) struct ProcStat { /// Read [`ProcStat`] for `pid` from one read of `/proc//stat`. `None` /// when the file cannot be read or parsed (the process is gone, or this is /// not Linux): the fail-closed direction, as for [`process_start_time`]. -#[cfg_attr( - not(target_os = "linux"), - allow(dead_code, reason = "read only by the Linux backstop (MAC-2)") -)] pub(crate) fn process_stat_fields(pid: u32) -> Option { parse_stat_fields(&std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?) } @@ -254,10 +246,6 @@ pub(crate) fn process_stat_fields(pid: u32) -> Option { /// [`process_stat_fields`]'s parser. Splits after the FINAL `)`, because /// field 2 (comm) may contain spaces and parentheses; after that point field /// 3 is index 0, so field 5 is 2, field 6 is 3 and field 22 is 19. -#[cfg_attr( - not(target_os = "linux"), - allow(dead_code, reason = "read only by the Linux backstop (MAC-2)") -)] fn parse_stat_fields(stat: &str) -> Option { let rest = &stat[stat.rfind(')')? + 1..]; let fields: Vec<&str> = rest.split_whitespace().collect(); diff --git a/crates/devflow-core/src/supervisor/backstop.rs b/crates/devflow-core/src/supervisor/backstop.rs index bd642f04..26c6f286 100644 --- a/crates/devflow-core/src/supervisor/backstop.rs +++ b/crates/devflow-core/src/supervisor/backstop.rs @@ -36,12 +36,15 @@ //! back to validate-then-`killpg` ([`SignalPath::ValidatedKillpg`]). Off Linux //! no signalling path exists. -#[cfg(test)] -use super::Liveness; -use super::{Sidecar, SocketPath}; +use super::{Liveness, Sidecar, SocketPath}; +use crate::agent::ProcStat; use crate::state::SupervisorRecord; use std::io; +/// How long to wait for the group to empty after SIGKILL. SIGKILL cannot be +/// caught, but its delivery is not synchronous. +const KILL_VERIFY_WAIT: std::time::Duration = std::time::Duration::from_secs(1); + /// One live process of the recorded process group, as gathered from /// `/proc`. #[derive(Debug, Clone, PartialEq, Eq)] @@ -59,6 +62,56 @@ pub struct GroupMember { pub cmdline: String, } +/// The identity a launch recorded, read from its sidecar or its state +/// record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RecordedIdentity { + pub(crate) launch_id: String, + pub(crate) pgid: u32, + pub(crate) start_time: u64, + pub(crate) boot_id: String, +} + +impl RecordedIdentity { + fn of_sidecar(sidecar: &Sidecar) -> Self { + Self { + launch_id: sidecar.launch_id.clone(), + pgid: sidecar.agent_pgid, + start_time: sidecar.agent_start_time, + boot_id: sidecar.boot_id.clone(), + } + } + + fn of_record(record: &SupervisorRecord) -> Self { + Self { + launch_id: record.launch_id.clone(), + pgid: record.agent_pgid, + start_time: record.agent_start_time, + boot_id: record.boot_id.clone(), + } + } +} + +/// Everything [`classify`] decides on, gathered once. +#[derive(Debug, Clone)] +pub(crate) struct BackstopFacts { + /// The sidecar's identity. + pub(crate) recorded: RecordedIdentity, + /// The state record's identity, when state loaded. + pub(crate) state_record: Option, + /// The caller's own process group (`getpgrp`). + pub(crate) caller_pgrp: i32, + /// The current kernel boot id. + pub(crate) current_boot_id: Option, + /// `/proc//stat` when an own-uid process holds that pid. + pub(crate) leader: Option, + /// Own-uid, non-zombie processes whose pgrp is the recorded pgid. + pub(crate) members: Vec, + /// Whether this platform could gather the facts that prove identity + /// (MAC-2): `/proc` start times, the group census and the boot id. + pub(crate) identity_provable: bool, +} + /// What the backstop decides for a STALE socket. See the module doc for the /// order. #[derive(Debug, Clone, PartialEq, Eq)] @@ -141,22 +194,446 @@ pub enum BackstopError { Io(#[from] io::Error), } +/// `pgid` as a `pid_t` when it is a safe group to signal: it fits, is above +/// 1, and is not `caller_pgrp`. +fn safe_target(pgid: u32, caller_pgrp: i32) -> Option { + let signed = libc::pid_t::try_from(pgid).ok()?; + (signed > 1 && signed != caller_pgrp).then_some(signed) +} + +/// The first identity field on which the sidecar and the state record +/// disagree, as text. +fn first_disagreement(sidecar: &RecordedIdentity, record: &RecordedIdentity) -> Option { + if sidecar.launch_id != record.launch_id { + return Some(format!( + "launch id: sidecar {}, state {}", + sidecar.launch_id, record.launch_id + )); + } + if sidecar.pgid != record.pgid { + return Some(format!( + "pgid: sidecar {}, state {}", + sidecar.pgid, record.pgid + )); + } + if sidecar.start_time != record.start_time { + return Some(format!( + "agent start time: sidecar {}, state {}", + sidecar.start_time, record.start_time + )); + } + if sidecar.boot_id != record.boot_id { + return Some(format!( + "boot id: sidecar {}, state {}", + sidecar.boot_id, record.boot_id + )); + } + None +} + +/// The backstop's decision rule, pure. See the module doc for the order. +pub(crate) fn classify(facts: &BackstopFacts) -> BackstopDecision { + let recorded = &facts.recorded; + let Some(pgid) = safe_target(recorded.pgid, facts.caller_pgrp) else { + return BackstopDecision::RefuseUnsafeTarget; + }; + let current_boot = match (facts.identity_provable, facts.current_boot_id.as_deref()) { + (true, Some(boot)) => boot, + _ => return BackstopDecision::RefuseIdentityUnavailable, + }; + if let Some(why) = facts + .state_record + .as_ref() + .and_then(|record| first_disagreement(recorded, record)) + { + return BackstopDecision::RefuseMismatch(why); + } + if recorded.boot_id != current_boot { + return BackstopDecision::PreviousBoot; + } + if facts.members.is_empty() { + return BackstopDecision::EmptyGroup; + } + match facts.leader { + Some(leader) if leader.start_time != recorded.start_time => { + BackstopDecision::LeaderRecycled + } + Some(leader) if leader.pgrp == pgid => BackstopDecision::LeaderPresentSignal, + _ => BackstopDecision::RefuseLeaderGone(facts.members.clone()), + } +} + +/// The caller's own process group. +fn caller_pgrp() -> i32 { + // SAFETY: getpgrp has no preconditions and cannot fail. + unsafe { libc::getpgrp() } +} + +/// Facts with nothing read about any process: the shape every platform +/// starts from, and the whole answer off Linux. +fn unread_facts(sidecar: &Sidecar, state_record: Option<&SupervisorRecord>) -> BackstopFacts { + BackstopFacts { + recorded: RecordedIdentity::of_sidecar(sidecar), + state_record: state_record.map(RecordedIdentity::of_record), + caller_pgrp: caller_pgrp(), + current_boot_id: None, + leader: None, + members: Vec::new(), + identity_provable: false, + } +} + +/// Gather the facts [`classify`] needs. An unsafe target reads nothing. +#[cfg(target_os = "linux")] +fn gather(sidecar: &Sidecar, state_record: Option<&SupervisorRecord>) -> BackstopFacts { + let mut facts = unread_facts(sidecar, state_record); + let Some(pgid) = safe_target(facts.recorded.pgid, facts.caller_pgrp) else { + return facts; + }; + let Some(members) = group_members(pgid) else { + // The process table could not be listed: identity is unprovable. + return facts; + }; + facts.members = members; + facts.leader = own_process_stat(facts.recorded.pgid); + facts.current_boot_id = super::boot_id(); + facts.identity_provable = true; + facts +} + +/// Off Linux there is no `/proc` start time, group census or boot id +/// (backlog 999.158, 999.159), so nothing is read and identity is +/// unprovable: [`classify`] answers `RefuseIdentityUnavailable` (MAC-2). +#[cfg(not(target_os = "linux"))] +fn gather(sidecar: &Sidecar, state_record: Option<&SupervisorRecord>) -> BackstopFacts { + unread_facts(sidecar, state_record) +} + +/// `/proc//stat` for a process this uid owns; `None` for another uid's +/// process or no process. +#[cfg(target_os = "linux")] +fn own_process_stat(pid: u32) -> Option { + let owner = std::fs::metadata(format!("/proc/{pid}")).ok()?; + if std::os::unix::fs::MetadataExt::uid(&owner) != super::protocol::euid() { + return None; + } + crate::agent::process_stat_fields(pid) +} + +/// Own-uid, non-zombie processes whose process group is `pgid`, by pid. +/// The owner uid is checked before anything else is read (the +/// `discover_stray_devflow_processes` rule). `None` when `/proc` cannot be +/// listed, which is "cannot tell", never "empty". +fn group_members(pgid: libc::pid_t) -> Option> { + let entries = std::fs::read_dir("/proc").ok()?; + let my_uid = super::protocol::euid(); + let mut members = Vec::new(); + for entry in entries.flatten() { + let Some(pid) = entry + .file_name() + .to_str() + .and_then(|name| name.parse::().ok()) + else { + continue; + }; + let Ok(owner) = std::fs::metadata(entry.path()) else { + continue; + }; + if std::os::unix::fs::MetadataExt::uid(&owner) != my_uid { + continue; + } + let Some(stat) = crate::agent::process_stat_fields(pid) else { + continue; + }; + if stat.pgrp != pgid || stat.state == 'Z' { + continue; + } + let cmdline = std::fs::read(format!("/proc/{pid}/cmdline")) + .map(|raw| { + raw.split(|&byte| byte == 0) + .filter(|arg| !arg.is_empty()) + .map(String::from_utf8_lossy) + .collect::>() + .join(" ") + }) + .unwrap_or_default(); + members.push(GroupMember { + pid, + start_time: stat.start_time, + state: stat.state, + session: stat.session, + cmdline, + }); + } + members.sort_by_key(|member| member.pid); + Some(members) +} + /// Decide what the backstop would do for this launch, without acting: for /// read-only callers (doctor, the launch guard) that classify a STALE socket. -pub fn assess(_sidecar: &Sidecar, _state_record: Option<&SupervisorRecord>) -> BackstopDecision { - // RED stub (50-09 Task 1): the decision rule lands in GREEN. - BackstopDecision::RefuseIdentityUnavailable +pub fn assess(sidecar: &Sidecar, state_record: Option<&SupervisorRecord>) -> BackstopDecision { + classify(&gather(sidecar, state_record)) } /// Tear down the launch behind a STALE socket. +/// +/// Re-probes first and returns [`BackstopError::NotStale`] unless the probe +/// is `Ok(Liveness::Stale)`. Then gathers facts, [`classify`]s them and acts: +/// `PreviousBoot`, `EmptyGroup` and `LeaderRecycled` remove the socket and +/// sidecar and signal nothing; `LeaderPresentSignal` ends the group (SIGTERM, +/// [`crate::agent::TERMINATE_VERIFY_WAIT`], SIGKILL) and then removes them; +/// every `Refuse*` keeps them and signals nothing. pub fn teardown_stale( - _socket: &SocketPath, - _sidecar: &Sidecar, - _state_record: Option<&SupervisorRecord>, + socket: &SocketPath, + sidecar: &Sidecar, + state_record: Option<&SupervisorRecord>, ) -> Result { - // RED stub (50-09 Task 1): probing, the decision and acting land in GREEN. - Ok(BackstopOutcome::Refused( - BackstopDecision::RefuseIdentityUnavailable, + match super::probe(socket.as_path()) { + Ok(Liveness::Stale) => {} + Ok(Liveness::Gone) => return Err(BackstopError::NotStale("it is gone".to_string())), + Ok(Liveness::Alive(_)) => { + return Err(BackstopError::NotStale( + "its supervisor answered a ping".to_string(), + )); + } + Err(error) => { + return Err(BackstopError::NotStale(format!( + "it could not be probed: {error}" + ))); + } + } + let facts = gather(sidecar, state_record); + match classify(&facts) { + BackstopDecision::LeaderPresentSignal => kill_group(socket, sidecar, state_record, &facts), + decision => settle(socket, decision), + } +} + +/// Act on a decision that signals nothing: remove the files for the three +/// decisions that prove the group gone, keep them for anything else. +fn settle( + socket: &SocketPath, + decision: BackstopDecision, +) -> Result { + match decision { + BackstopDecision::PreviousBoot + | BackstopDecision::EmptyGroup + | BackstopDecision::LeaderRecycled => { + super::remove_launch_files(socket)?; + Ok(BackstopOutcome::RemovedNothingSignalled(decision)) + } + refusal => Ok(BackstopOutcome::Refused(refusal)), + } +} + +/// The result of one validated group signal. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + not(target_os = "linux"), + allow(dead_code, reason = "no signalling path exists off Linux (MAC-2)") +)] +enum GroupSignal { + /// Sent to the whole group. + Sent(SignalPath), + /// The leader exited, or its pid no longer names the recorded leader of + /// the group: nothing was sent. + LeaderGone, +} + +/// End a validated group: SIGTERM, wait for it to empty, then SIGKILL. +fn kill_group( + socket: &SocketPath, + sidecar: &Sidecar, + state_record: Option<&SupervisorRecord>, + facts: &BackstopFacts, +) -> Result { + let recorded = &facts.recorded; + let members = facts.members.clone(); + let pgid = safe_target(recorded.pgid, facts.caller_pgrp).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "unsafe process group target") + })?; + let via = match signal_validated_group(recorded.pgid, recorded.start_time, libc::SIGTERM)? { + GroupSignal::Sent(via) => via, + GroupSignal::LeaderGone => { + // The leader changed between gathering and opening its pidfd; + // nothing was sent. Decide again on fresh facts. + let fresh = gather(sidecar, state_record); + return match classify(&fresh) { + BackstopDecision::LeaderPresentSignal => Err(BackstopError::Io(io::Error::other( + "the group leader changed while it was being validated; nothing was signalled", + ))), + decision => settle(socket, decision), + }; + } + }; + if group_empties_within(pgid, crate::agent::TERMINATE_VERIFY_WAIT) { + super::remove_launch_files(socket)?; + return Ok(BackstopOutcome::Killed { via, members }); + } + if signal_validated_group(recorded.pgid, recorded.start_time, libc::SIGKILL)? + == GroupSignal::LeaderGone + { + // The leader died during the grace period and members outlived the + // SIGTERM. Without a leader nothing proves they are the agent's, so + // decide again: an empty group is done, anything else is kept. + return match classify(&gather(sidecar, state_record)) { + BackstopDecision::EmptyGroup => { + super::remove_launch_files(socket)?; + Ok(BackstopOutcome::Killed { via, members }) + } + decision => Ok(BackstopOutcome::Refused(decision)), + }; + } + if group_empties_within(pgid, KILL_VERIFY_WAIT) { + super::remove_launch_files(socket)?; + return Ok(BackstopOutcome::Killed { via, members }); + } + Err(BackstopError::Io(io::Error::other(format!( + "process group {pgid} still has members after SIGKILL; the socket and sidecar were kept" + )))) +} + +/// Poll the census until no live member of `pgid` remains, up to `wait`. +/// An unreadable process table never counts as empty. +fn group_empties_within(pgid: libc::pid_t, wait: std::time::Duration) -> bool { + let deadline = std::time::Instant::now() + wait; + loop { + if group_members(pgid).is_some_and(|members| members.is_empty()) { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + std::thread::sleep(crate::agent::TERMINATE_VERIFY_POLL); + } +} + +/// Whether `/proc/` is still the recorded leader (same start time) +/// and still leads group `pgid`. +#[cfg(target_os = "linux")] +fn is_original_leader(pgid: u32, start_time: u64) -> bool { + own_process_stat(pgid).is_some_and(|stat| { + stat.start_time == start_time && i64::from(stat.pgrp) == i64::from(pgid) + }) +} + +/// Signal the group whose leader is `pgid` with the recorded `start_time`. +/// +/// Opens a pidfd on the leader, which pins one process, then re-reads its +/// start time and group: if they still match, the fd names the original +/// leader (a recycled pid would show a later start time). Then +/// `pidfd_send_signal(fd, signal, NULL, PIDFD_SIGNAL_PROCESS_GROUP)`. On +/// `EINVAL` (a kernel before 6.9, RESEARCH A3) it re-validates and uses +/// `killpg` ([`SignalPath::ValidatedKillpg`]). `ESRCH` is +/// [`GroupSignal::LeaderGone`]. +#[cfg(target_os = "linux")] +fn signal_validated_group( + pgid: u32, + start_time: u64, + signal: libc::c_int, +) -> io::Result { + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + + let pid = safe_target(pgid, caller_pgrp()).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "unsafe process group target") + })?; + // SAFETY: pidfd_open(pid, 0) takes plain integers and returns a new fd + // or -1 with errno set. + let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0 as libc::c_uint) }; + if raw < 0 { + let error = io::Error::last_os_error(); + return match error.raw_os_error() { + Some(libc::ESRCH) => Ok(GroupSignal::LeaderGone), + _ => Err(error), + }; + } + let raw = libc::c_int::try_from(raw) + .map_err(|_| io::Error::other("pidfd_open returned an out-of-range fd"))?; + // SAFETY: `raw` is a pidfd this call just opened and nothing else owns. + let pidfd = unsafe { OwnedFd::from_raw_fd(raw) }; + if !is_original_leader(pgid, start_time) { + return Ok(GroupSignal::LeaderGone); + } + // SAFETY: a live pidfd, a valid signal, a null siginfo (the kernel fills + // in a SI_USER one) and a flag the kernel validates. + let rc = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + pidfd.as_raw_fd(), + signal, + std::ptr::null::(), + libc::PIDFD_SIGNAL_PROCESS_GROUP, + ) + }; + if rc == 0 { + return Ok(GroupSignal::Sent(SignalPath::PidfdGroup)); + } + let error = io::Error::last_os_error(); + match error.raw_os_error() { + Some(libc::ESRCH) => Ok(GroupSignal::LeaderGone), + Some(libc::EINVAL) => validated_killpg(&pidfd, pid, pgid, start_time, signal), + _ => Err(error), + } +} + +/// The `EINVAL` fallback: confirm through the pidfd that the pinned leader +/// is alive, re-read its start time and group, then `killpg`. The window +/// between the check and `killpg` is the documented TOCTOU this path keeps +/// (T-50-24); while the leader is alive its pid cannot name another group. +#[cfg(target_os = "linux")] +fn validated_killpg( + pidfd: &std::os::fd::OwnedFd, + pid: libc::pid_t, + pgid: u32, + start_time: u64, + signal: libc::c_int, +) -> io::Result { + use std::os::fd::AsRawFd; + + // SAFETY: signal 0 with no flags only checks that the pinned process is + // alive and signallable. + let rc = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + pidfd.as_raw_fd(), + 0 as libc::c_int, + std::ptr::null::(), + 0 as libc::c_uint, + ) + }; + if rc != 0 { + let error = io::Error::last_os_error(); + return match error.raw_os_error() { + Some(libc::ESRCH) => Ok(GroupSignal::LeaderGone), + _ => Err(error), + }; + } + if !is_original_leader(pgid, start_time) { + return Ok(GroupSignal::LeaderGone); + } + // SAFETY: `pid` passed `safe_target` (> 1, not our own group), so + // killpg reaches neither our group nor every process. + if unsafe { libc::killpg(pid, signal) } != 0 { + let error = io::Error::last_os_error(); + return match error.raw_os_error() { + Some(libc::ESRCH) => Ok(GroupSignal::LeaderGone), + _ => Err(error), + }; + } + Ok(GroupSignal::Sent(SignalPath::ValidatedKillpg)) +} + +/// Off Linux there is no pidfd and no way to prove the leader's identity, so +/// there is no signalling path: this sends nothing (MAC-2). +#[cfg(not(target_os = "linux"))] +fn signal_validated_group( + _pgid: u32, + _start_time: u64, + _signal: libc::c_int, +) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "group signalling needs Linux pidfds; nothing was signalled", )) } From cf25c8ba40b4c0b975888476f27ff965ea347fc0 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 14:43:51 -0400 Subject: [PATCH 038/112] test(50-09): add failing tests for the refusal text and unsound targets - V-19: a leader-gone group is refused and reported, never signalled; control: leader alive is killed - V-20: a reused pgid is LeaderRecycled, nothing signalled - V-21: a previous-boot socket is removed, nothing signalled - pgid 0, 1, own group and an unfitting pgid are RefuseUnsafeTarget - gathering skips another uid's processes (group 1 control) - MAC-2: the RefuseIdentityUnavailable text names the group, launch, files and manual repair; describe is a RED stub Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- .../devflow-core/src/supervisor/backstop.rs | 291 ++++++++++++++++++ 1 file changed, 291 insertions(+) diff --git a/crates/devflow-core/src/supervisor/backstop.rs b/crates/devflow-core/src/supervisor/backstop.rs index 26c6f286..92518d62 100644 --- a/crates/devflow-core/src/supervisor/backstop.rs +++ b/crates/devflow-core/src/supervisor/backstop.rs @@ -147,6 +147,15 @@ pub enum BackstopDecision { RefuseLeaderGone(Vec), } +impl BackstopDecision { + /// The text the CLI prints for this decision about the launch whose + /// socket is `socket` and whose sidecar is `sidecar`. + pub fn describe(&self, _socket: &SocketPath, _sidecar: &Sidecar) -> String { + // RED stub (50-09 Task 2). + String::new() + } +} + /// How a group was signalled. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] @@ -652,8 +661,11 @@ mod tests { /// A group whose leader stays: it waits on its two sleeping members. #[cfg(target_os = "linux")] const LEADER_WAITS: &str = "sleep 30 & sleep 30 & wait"; + /// A group whose leader forks two sleeping members and exits at once. #[cfg(target_os = "linux")] + const LEADER_EXITS: &str = "sleep 30 & sleep 30 & exit 0"; /// How long a fixture may take to reach the state a test needs. + #[cfg(target_os = "linux")] const SETTLE: Duration = Duration::from_secs(5); #[cfg(target_os = "linux")] const POLL: Duration = Duration::from_millis(20); @@ -819,6 +831,14 @@ mod tests { } } + /// Reap the leader (a script that exits on its own) and stop + /// counting it as a member. + fn reap_leader(&mut self) { + self.leader.wait().expect("reap the leader"); + self.leader_reaped = true; + self.members.retain(|(pid, _)| *pid != self.pgid); + } + fn all_alive(&self) -> bool { self.members.iter().all(|&(pid, start)| alive(pid, start)) } @@ -1007,4 +1027,275 @@ mod tests { "a refusal keeps the files" ); } + + /// A short pause after a call that must signal nothing, so a wrongly + /// sent signal has time to land before the survivors are checked. + #[cfg(target_os = "linux")] + const SIGNAL_LANDS: Duration = Duration::from_millis(200); + + /// Tear down a fixture group whose leader is present with the recorded + /// identity, and require it gone: the opposite-direction control of + /// V-19, V-20 and V-21. + #[cfg(target_os = "linux")] + fn assert_validated_group_is_killed(cache: &Path, boot: &str) { + let mut group = TestGroup::spawn(LEADER_WAITS); + group.settle(2); + let (socket, sidecar) = stale_launch(cache, group.pgid, group.leader_start, boot); + let outcome = teardown_stale(&socket, &sidecar, None).expect("teardown the control"); + assert!( + matches!(outcome, BackstopOutcome::Killed { .. }), + "the control group must be killed, got {outcome:?}" + ); + group.wait_all_gone(); + assert_eq!(files_exist(&socket), (false, false)); + } + + /// V-19 (R-4, RESEARCH Finding 1): the leader is gone and its members + /// live, so nothing proves they are the agent's. They are reported, + /// never signalled, and the files are kept. Control: the same shape with + /// the leader alive is killed. + #[cfg(target_os = "linux")] + #[test] + fn backstop_refuses_and_reports_a_group_whose_leader_is_gone() { + let cache = short_cache(); + let boot = super::super::boot_id().expect("read the boot id"); + let mut group = TestGroup::spawn(LEADER_EXITS); + group.reap_leader(); + group.settle(2); + assert_eq!(group.members.len(), 2, "two members outlive the leader"); + + let (socket, sidecar) = stale_launch(cache.path(), group.pgid, group.leader_start, &boot); + let outcome = teardown_stale(&socket, &sidecar, None).expect("teardown"); + let BackstopOutcome::Refused(decision) = outcome else { + panic!("a leader-gone group must be refused, got {outcome:?}"); + }; + let BackstopDecision::RefuseLeaderGone(members) = &decision else { + panic!("expected RefuseLeaderGone, got {decision:?}"); + }; + assert_eq!( + member_ids(members), + group.members, + "both members are reported" + ); + assert!( + members + .iter() + .all(|member| member.cmdline.starts_with("sleep 30")), + "each member is reported with its cmdline: {members:?}" + ); + std::thread::sleep(SIGNAL_LANDS); + assert!(group.all_alive(), "a leader-gone group is never signalled"); + assert_eq!( + files_exist(&socket), + (true, true), + "a refusal keeps the files" + ); + + let text = decision.describe(&socket, &sidecar); + assert!( + text.contains(&format!("ps -o pid,pgid,sid,lstart,args -g {}", group.pgid)), + "{text}" + ); + assert!( + text.contains( + "may be the original agent's children or an unrelated group that reused the id; \ + nothing was signalled" + ), + "{text}" + ); + for member in members { + assert!(text.contains(&format!("pid {},", member.pid)), "{text}"); + assert!( + text.contains(&format!("start time {},", member.start_time)), + "{text}" + ); + assert!( + text.contains(&format!("session {}", member.session)), + "{text}" + ); + } + + assert_validated_group_is_killed(cache.path(), &boot); + } + + /// V-20 (R-4, T-50-22): the recorded pgid names a live, unrelated group + /// whose leader started at another time. That is a recycled pid, so the + /// recorded group is gone: nothing is signalled and the files go. + /// Control: the matching start time is signalled. + #[cfg(target_os = "linux")] + #[test] + fn a_reused_pgid_is_never_signalled() { + let cache = short_cache(); + let boot = super::super::boot_id().expect("read the boot id"); + let mut unrelated = TestGroup::spawn(LEADER_WAITS); + unrelated.settle(2); + assert!( + unrelated.leader_start > 0, + "a start time an earlier agent could precede" + ); + + // The recorded agent started one tick before the group now at its pid. + let recorded_start = unrelated.leader_start - 1; + let (socket, sidecar) = stale_launch(cache.path(), unrelated.pgid, recorded_start, &boot); + assert_eq!( + teardown_stale(&socket, &sidecar, None).expect("teardown"), + BackstopOutcome::RemovedNothingSignalled(BackstopDecision::LeaderRecycled) + ); + std::thread::sleep(SIGNAL_LANDS); + assert!( + unrelated.all_alive(), + "the unrelated group must not be signalled" + ); + assert_eq!( + files_exist(&socket), + (false, false), + "a recycled pgid is GONE" + ); + + assert_validated_group_is_killed(cache.path(), &boot); + } + + /// V-21 (R-8): a sidecar from another boot over a live, otherwise valid + /// group is removed and nothing is signalled. Control: the current boot + /// id is signalled. + #[cfg(target_os = "linux")] + #[test] + fn a_previous_boot_socket_is_removed_and_nothing_is_signalled() { + let cache = short_cache(); + let boot = super::super::boot_id().expect("read the boot id"); + let previous = "00000000-0000-4000-8000-000000000000"; + assert_ne!( + boot, previous, + "the fabricated boot id must differ from this boot's" + ); + let mut group = TestGroup::spawn(LEADER_WAITS); + group.settle(2); + + let (socket, sidecar) = + stale_launch(cache.path(), group.pgid, group.leader_start, previous); + assert_eq!( + teardown_stale(&socket, &sidecar, None).expect("teardown"), + BackstopOutcome::RemovedNothingSignalled(BackstopDecision::PreviousBoot) + ); + std::thread::sleep(SIGNAL_LANDS); + assert!(group.all_alive(), "a previous-boot socket signals nothing"); + assert_eq!( + files_exist(&socket), + (false, false), + "a previous-boot socket is removed" + ); + + assert_validated_group_is_killed(cache.path(), &boot); + } + + /// T-50-25: pgid 0, 1, the caller's own group and a value that does not + /// fit `pid_t` are refused before anything is read or signalled. Were + /// the caller's group signalled, this test would die. Control: an + /// ordinary pgid is not refused as unsafe. + #[test] + fn the_backstop_never_targets_pgid_one_or_its_own_group() { + let cache = short_cache(); + // SAFETY: getpgrp has no preconditions and cannot fail. + let own = u32::try_from(unsafe { libc::getpgrp() }).expect("a positive pgrp"); + for pgid in [0, 1, own, u32::MAX] { + let (socket, sidecar) = stale_launch(cache.path(), pgid, 1, "boot-test"); + assert_eq!( + assess(&sidecar, None), + BackstopDecision::RefuseUnsafeTarget, + "pgid {pgid}" + ); + assert_eq!( + teardown_stale(&socket, &sidecar, None).expect("teardown"), + BackstopOutcome::Refused(BackstopDecision::RefuseUnsafeTarget), + "pgid {pgid}" + ); + assert_eq!( + files_exist(&socket), + (true, true), + "pgid {pgid} keeps its files" + ); + } + let ordinary = sidecar_for( + "ordinary", + u32::try_from(i32::MAX).expect("fits"), + 1, + "boot", + ); + assert_ne!( + assess(&ordinary, None), + BackstopDecision::RefuseUnsafeTarget + ); + } + + /// T-50-25: gathering reads only this uid's processes. Group 1 (init's) + /// is visible to an independent `/proc` scan but owned by root, so the + /// census and the leader read find nothing there. + #[cfg(target_os = "linux")] + #[test] + fn gathering_skips_processes_of_another_uid() { + if super::super::protocol::euid() == 0 { + eprintln!("skipped: running as root, so no process belongs to another uid"); + return; + } + assert!( + !live_in_group(1).is_empty(), + "control: an independent scan sees group 1" + ); + assert_eq!( + group_members(1), + Some(Vec::new()), + "another uid's group reads empty" + ); + assert_eq!( + own_process_stat(1), + None, + "another uid's leader reads absent" + ); + let me = std::process::id(); + assert!( + own_process_stat(me).is_some(), + "control: this uid's process is read" + ); + } + + /// MAC-2: the `RefuseIdentityUnavailable` text names the group, the + /// launch, both files and the manual repair, and no platform command. + /// Control: the leader-gone text does not claim identity is unprovable. + #[test] + fn the_identity_unavailable_refusal_names_the_group_and_the_manual_repair() { + let launch_id = "aaaa0000aaaa0000"; + let socket = socket_for(Path::new("/tmp/dfs-text-fixture"), launch_id); + let sidecar = sidecar_for(launch_id, 4242, 99, "boot-test"); + let text = BackstopDecision::RefuseIdentityUnavailable.describe(&socket, &sidecar); + for needle in [ + "nothing was signalled or removed".to_string(), + "cannot prove process group 4242".to_string(), + launch_id.to_string(), + socket.as_path().display().to_string(), + socket.sidecar_path().display().to_string(), + "end that group yourself".to_string(), + ] { + assert!(text.contains(&needle), "missing {needle:?} in {text}"); + } + for command in ["ps ", "kill", "pkill", "launchctl"] { + assert!( + !text.contains(command), + "names a command {command:?}: {text}" + ); + } + + let member = GroupMember { + pid: 4243, + start_time: 100, + state: 'S', + session: 4000, + cmdline: "sleep 30".to_string(), + }; + let gone = BackstopDecision::RefuseLeaderGone(vec![member]).describe(&socket, &sidecar); + assert!(!gone.contains("cannot prove process group"), "{gone}"); + assert!( + gone.contains("ps -o pid,pgid,sid,lstart,args -g 4242"), + "{gone}" + ); + } } From b4604086d98ce20fdc9eb1ea7b46dd55f06d4e08 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 14:45:41 -0400 Subject: [PATCH 039/112] feat(50-09): describe every backstop decision for the CLI - RefuseLeaderGone lists each member (pid, start time, state, session, cmdline), states the residual F1 limit and gives ps -o pid,pgid,sid,lstart,args -g - RefuseIdentityUnavailable (MAC-2) names the group, the launch, both files and the manual repair, and no platform command Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- .../devflow-core/src/supervisor/backstop.rs | 74 ++++++++++++++++++- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/crates/devflow-core/src/supervisor/backstop.rs b/crates/devflow-core/src/supervisor/backstop.rs index 92518d62..1c663f6f 100644 --- a/crates/devflow-core/src/supervisor/backstop.rs +++ b/crates/devflow-core/src/supervisor/backstop.rs @@ -150,9 +150,77 @@ pub enum BackstopDecision { impl BackstopDecision { /// The text the CLI prints for this decision about the launch whose /// socket is `socket` and whose sidecar is `sidecar`. - pub fn describe(&self, _socket: &SocketPath, _sidecar: &Sidecar) -> String { - // RED stub (50-09 Task 2). - String::new() + /// + /// A `RefuseLeaderGone` text lists each member's pid, start time, state, + /// session and command line, states the residual F1 limit and gives an + /// inspection command. A `RefuseIdentityUnavailable` text gives the + /// manual repair and names no platform command (MAC-2, backlog 999.158). + pub fn describe(&self, socket: &SocketPath, sidecar: &Sidecar) -> String { + use std::fmt::Write as _; + + let pgid = sidecar.agent_pgid; + let launch = &sidecar.launch_id; + let files = format!( + "{} and {}", + socket.as_path().display(), + socket.sidecar_path().display() + ); + match self { + Self::RefuseUnsafeTarget => format!( + "refused: process group {pgid} of launch {launch} is not a safe target (it is 0 \ + or 1, does not fit a pid, or is this process's own group); nothing was \ + signalled or removed" + ), + Self::RefuseIdentityUnavailable => format!( + "nothing was signalled or removed; this platform gives DevFlow no process start \ + time or boot id, so it cannot prove process group {pgid} is still the agent \ + launch {launch} started; if you have confirmed it is, end that group yourself, \ + then delete {files}" + ), + Self::RefuseMismatch(why) => format!( + "refused: the sidecar and the state record of launch {launch} disagree ({why}); \ + nothing was signalled or removed" + ), + Self::PreviousBoot => format!( + "launch {launch} was recorded in a previous boot (boot id {}), so process group \ + {pgid} cannot still be its agent; nothing is signalled, and {files} are removed", + sidecar.boot_id + ), + Self::EmptyGroup => format!( + "process group {pgid} of launch {launch} has no live process of this user; \ + nothing is signalled, and {files} are removed" + ), + Self::LeaderPresentSignal => format!( + "process group {pgid} is still led by launch {launch}'s agent (start time {}); \ + the group is ended with SIGTERM, then SIGKILL, and {files} are removed", + sidecar.agent_start_time + ), + Self::LeaderRecycled => format!( + "pid {pgid} now belongs to a different process than launch {launch}'s agent \ + (start time {}), so the agent's group is gone; nothing is signalled, and {files} \ + are removed", + sidecar.agent_start_time + ), + Self::RefuseLeaderGone(members) => { + let mut text = format!( + "refused: process group {pgid} of launch {launch} still has {} live \ + member(s), but its leader, the recorded agent (pid {pgid}), is gone or has \ + left the group. The leader is gone, so these may be the original agent's \ + children or an unrelated group that reused the id; nothing was signalled, \ + and {files} are kept. Inspect the group with: ps -o \ + pid,pgid,sid,lstart,args -g {pgid}", + members.len() + ); + for member in members { + let _ = write!( + text, + "\n pid {}, start time {}, state {}, session {}: {}", + member.pid, member.start_time, member.state, member.session, member.cmdline + ); + } + text + } + } } } From b437bb0a95f3deadc318397629d7db9a74addc4c Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 14:59:43 -0400 Subject: [PATCH 040/112] test(50-09): pin the classifier table and the kernel-capability check - classify_covers_every_rule_in_order: every step, its boundaries, the order checks and the MAC-2 identity_provable rows over synthetic facts - pidfd_group_signal_is_used_when_the_kernel_supports_it probes the kernel independently and prints a skip line when the flag is missing - the signal helper re-validates the leader after opening its pidfd (two cargo-mutants survivors in is_original_leader, now caught) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- .../devflow-core/src/supervisor/backstop.rs | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) diff --git a/crates/devflow-core/src/supervisor/backstop.rs b/crates/devflow-core/src/supervisor/backstop.rs index 1c663f6f..ca5cfdf1 100644 --- a/crates/devflow-core/src/supervisor/backstop.rs +++ b/crates/devflow-core/src/supervisor/backstop.rs @@ -1366,4 +1366,401 @@ mod tests { "{gone}" ); } + + const TABLE_PGID: u32 = 4242; + const TABLE_START: u64 = 100; + const TABLE_BOOT: &str = "boot-now"; + const TABLE_CALLER: i32 = 7; + + fn table_member(pid: u32) -> GroupMember { + GroupMember { + pid, + start_time: TABLE_START, + state: 'S', + session: 4000, + cmdline: "agent".to_string(), + } + } + + fn table_identity() -> RecordedIdentity { + RecordedIdentity { + launch_id: "launch-a".to_string(), + pgid: TABLE_PGID, + start_time: TABLE_START, + boot_id: TABLE_BOOT.to_string(), + } + } + + /// Facts for a validated, leader-present group: the one shape that + /// signals. Each table row changes one or two facts from here. + fn signalling_facts() -> BackstopFacts { + BackstopFacts { + recorded: table_identity(), + state_record: Some(table_identity()), + caller_pgrp: TABLE_CALLER, + current_boot_id: Some(TABLE_BOOT.to_string()), + leader: Some(ProcStat { + state: 'S', + pgrp: 4242, + session: 4000, + start_time: TABLE_START, + }), + members: vec![table_member(TABLE_PGID), table_member(TABLE_PGID + 1)], + identity_provable: true, + } + } + + fn with(change: impl FnOnce(&mut BackstopFacts)) -> BackstopFacts { + let mut facts = signalling_facts(); + change(&mut facts); + facts + } + + fn set_pgid(facts: &mut BackstopFacts, pgid: u32) { + facts.recorded.pgid = pgid; + if let Some(record) = facts.state_record.as_mut() { + record.pgid = pgid; + } + if let Some(leader) = facts.leader.as_mut() { + leader.pgrp = i32::try_from(pgid).unwrap_or(-1); + } + } + + fn mismatched(field: &str) -> BackstopFacts { + with(|facts| { + let record = facts.state_record.as_mut().expect("a state record"); + match field { + "launch id" => record.launch_id = "launch-b".to_string(), + "pgid" => record.pgid = TABLE_PGID + 9, + "agent start time" => record.start_time = TABLE_START + 9, + "boot id" => record.boot_id = "boot-other".to_string(), + other => panic!("no field {other}"), + } + }) + } + + fn previous_boot() -> BackstopFacts { + with(|facts| facts.current_boot_id = Some("boot-later".to_string())) + } + + fn empty_group() -> BackstopFacts { + with(|facts| facts.members.clear()) + } + + fn recycled_leader() -> BackstopFacts { + with(|facts| { + facts.leader = Some(ProcStat { + state: 'S', + pgrp: 4242, + session: 5000, + start_time: TABLE_START + 1, + }); + }) + } + + fn leader_gone() -> BackstopFacts { + with(|facts| facts.leader = None) + } + + fn gone_members() -> BackstopDecision { + BackstopDecision::RefuseLeaderGone(signalling_facts().members) + } + + /// The decision rule, case by case and order by order, over synthetic + /// facts. Runs on every target: `classify` has no `cfg`. + #[test] + fn classify_covers_every_rule_in_order() { + use BackstopDecision as D; + let own_group = u32::try_from(TABLE_CALLER).expect("positive"); + let largest_pid = u32::try_from(i32::MAX).expect("fits"); + let mut rows: Vec<(&str, BackstopFacts, BackstopDecision)> = vec![ + // Step 4 (the only signalling row), and its boundaries. + ( + "validated leader present", + signalling_facts(), + D::LeaderPresentSignal, + ), + ( + "no state record", + with(|f| f.state_record = None), + D::LeaderPresentSignal, + ), + ( + "pgid 2 is a safe target", + with(|f| set_pgid(f, 2)), + D::LeaderPresentSignal, + ), + ( + "i32::MAX fits pid_t", + with(|f| set_pgid(f, largest_pid)), + D::LeaderPresentSignal, + ), + ( + "a zombie leader still leads", + with(|f| { + f.leader.as_mut().expect("leader").state = 'Z'; + }), + D::LeaderPresentSignal, + ), + // The unsafe-target guard, first of all. + ("pgid 0", with(|f| set_pgid(f, 0)), D::RefuseUnsafeTarget), + ("pgid 1", with(|f| set_pgid(f, 1)), D::RefuseUnsafeTarget), + ( + "the caller's own group", + with(|f| set_pgid(f, own_group)), + D::RefuseUnsafeTarget, + ), + ( + "pgid beyond pid_t", + with(|f| set_pgid(f, largest_pid + 1)), + D::RefuseUnsafeTarget, + ), + ( + "u32::MAX", + with(|f| set_pgid(f, u32::MAX)), + D::RefuseUnsafeTarget, + ), + ( + "unsafe wins over unprovable identity", + with(|f| { + set_pgid(f, 1); + f.identity_provable = false; + }), + D::RefuseUnsafeTarget, + ), + ( + "unsafe wins over a mismatch", + { + let mut f = mismatched("launch id"); + set_pgid(&mut f, 0); + f + }, + D::RefuseUnsafeTarget, + ), + // An unreadable boot id is the identity refusal, never PreviousBoot. + ( + "no current boot id", + with(|f| f.current_boot_id = None), + D::RefuseIdentityUnavailable, + ), + // Step 3: previous boot, and it wins over an empty group. + ("previous boot", previous_boot(), D::PreviousBoot), + ( + "previous boot wins over an empty group", + with(|f| { + f.current_boot_id = Some("boot-later".to_string()); + f.members.clear(); + }), + D::PreviousBoot, + ), + ( + "previous boot wins over a gone leader", + with(|f| { + f.current_boot_id = Some("boot-later".to_string()); + f.leader = None; + }), + D::PreviousBoot, + ), + // Step 4: an empty group, and it wins over the leader checks. + ("empty group", empty_group(), D::EmptyGroup), + ( + "empty group wins over a recycled leader", + with(|f| { + f.members.clear(); + f.leader.as_mut().expect("leader").start_time = TABLE_START + 1; + }), + D::EmptyGroup, + ), + ( + "empty group with no leader", + with(|f| { + f.members.clear(); + f.leader = None; + }), + D::EmptyGroup, + ), + // Step 6: recycled leader, whatever its group. + ("recycled leader", recycled_leader(), D::LeaderRecycled), + ( + "recycled leader in another group", + with(|f| { + let leader = f.leader.as_mut().expect("leader"); + leader.start_time = TABLE_START - 1; + leader.pgrp = 9999; + }), + D::LeaderRecycled, + ), + // Step 7: leader gone, or the original leader left the group. + ("leader gone, members live", leader_gone(), gone_members()), + ( + "original leader left the group", + with(|f| { + f.leader.as_mut().expect("leader").pgrp = 9999; + }), + gone_members(), + ), + ]; + // Step 2: each mismatched field refuses, and a mismatch wins over + // everything after it. + for field in ["launch id", "pgid", "agent start time", "boot id"] { + rows.push(( + "mismatch", + mismatched(field), + D::RefuseMismatch(String::new()), + )); + } + for (name, later) in [ + ("mismatch wins over previous boot", previous_boot()), + ("mismatch wins over an empty group", empty_group()), + ("mismatch wins over a recycled leader", recycled_leader()), + ("mismatch wins over a gone leader", leader_gone()), + ] { + let mut facts = later; + facts.state_record.as_mut().expect("record").launch_id = "launch-b".to_string(); + rows.push((name, facts, D::RefuseMismatch(String::new()))); + } + // MAC-2: unprovable identity refuses before every step after the + // unsafe-target guard, whatever those steps would have answered. + for (name, base) in [ + ("unprovable over a mismatch", mismatched("pgid")), + ("unprovable over a previous boot", previous_boot()), + ("unprovable over an empty group", empty_group()), + ("unprovable over a present leader", signalling_facts()), + ("unprovable over a recycled leader", recycled_leader()), + ("unprovable over a gone leader", leader_gone()), + ] { + let mut facts = base; + facts.identity_provable = false; + rows.push((name, facts, D::RefuseIdentityUnavailable)); + } + + for (name, facts, expected) in &rows { + let actual = classify(facts); + match (expected, &actual) { + // A mismatch is matched by variant; its text is checked below. + (D::RefuseMismatch(_), D::RefuseMismatch(_)) => {} + _ => assert_eq!(&actual, expected, "row {name:?}"), + } + } + + // The mismatch text names the field that differs, and both values. + for (field, sidecar_value, state_value) in [ + ("launch id", "launch-a", "launch-b"), + ("pgid", "4242", "4251"), + ("agent start time", "100", "109"), + ("boot id", "boot-now", "boot-other"), + ] { + let D::RefuseMismatch(why) = classify(&mismatched(field)) else { + panic!("{field} must be a mismatch"); + }; + assert_eq!( + why, + format!("{field}: sidecar {sidecar_value}, state {state_value}") + ); + } + } + + /// The kernel-capability check (RESEARCH A3): on a kernel with + /// `PIDFD_SIGNAL_PROCESS_GROUP` the backstop must use it. The kernel is + /// probed independently of the code under test (signal 0 with the flag + /// on a pidfd of a group this test owns); a kernel without it is + /// reported on stderr, never passed silently. + #[cfg(target_os = "linux")] + #[test] + fn pidfd_group_signal_is_used_when_the_kernel_supports_it() { + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + + let cache = short_cache(); + let boot = super::super::boot_id().expect("read the boot id"); + let mut group = TestGroup::spawn(LEADER_WAITS); + group.settle(2); + + let leader = libc::pid_t::try_from(group.pgid).expect("fits pid_t"); + // SAFETY: pidfd_open on this test's own unreaped child. + let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, leader, 0 as libc::c_uint) }; + assert!(raw >= 0, "pidfd_open: {}", io::Error::last_os_error()); + // SAFETY: a pidfd this call just opened and nothing else owns. + let pidfd = unsafe { OwnedFd::from_raw_fd(libc::c_int::try_from(raw).expect("an fd")) }; + // SAFETY: signal 0 only checks permission; the flag is what is probed. + let rc = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + pidfd.as_raw_fd(), + 0 as libc::c_int, + std::ptr::null::(), + libc::PIDFD_SIGNAL_PROCESS_GROUP, + ) + }; + if rc != 0 { + let error = io::Error::last_os_error(); + assert_eq!( + error.raw_os_error(), + Some(libc::EINVAL), + "probe failed: {error}" + ); + eprintln!("skipped: kernel lacks PIDFD_SIGNAL_PROCESS_GROUP"); + return; + } + + let (socket, sidecar) = stale_launch(cache.path(), group.pgid, group.leader_start, &boot); + let outcome = teardown_stale(&socket, &sidecar, None).expect("teardown"); + let BackstopOutcome::Killed { via, .. } = outcome else { + panic!("a validated group must be Killed, got {outcome:?}"); + }; + assert_eq!( + via, + SignalPath::PidfdGroup, + "a supporting kernel uses the pidfd group signal" + ); + group.wait_all_gone(); + } + + /// T-50-24: after opening the leader's pidfd, the helper re-reads the + /// leader and sends nothing unless it is still the recorded process and + /// still leads the group its pid names. Signal 0 is used throughout, so + /// a wrongly "sent" signal shows in the return value and harms nothing. + #[cfg(target_os = "linux")] + #[test] + fn the_group_signal_revalidates_the_leader_after_opening_its_pidfd() { + struct Reaped(Child); + impl Drop for Reaped { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + let mut group = TestGroup::spawn(LEADER_WAITS); + group.settle(2); + assert_eq!( + signal_validated_group(group.pgid, group.leader_start + 1, 0).expect("signal"), + GroupSignal::LeaderGone, + "a different start time sends nothing" + ); + + // A process that does not lead a group of its own pid: it stays in + // this test's group. + let outsider = Reaped( + Command::new("sleep") + .arg("30") + .stdin(Stdio::null()) + .spawn() + .expect("spawn a process in this test's group"), + ); + let outsider_pid = outsider.0.id(); + let outsider_start = + crate::agent::process_start_time(outsider_pid).expect("read its start time"); + assert_eq!( + signal_validated_group(outsider_pid, outsider_start, 0).expect("signal"), + GroupSignal::LeaderGone, + "a process that does not lead the group its pid names sends nothing" + ); + + // Control: the recorded leader of its own group is signalled. + assert!(matches!( + signal_validated_group(group.pgid, group.leader_start, 0).expect("signal"), + GroupSignal::Sent(_) + )); + assert!(group.all_alive(), "signal 0 ends nothing"); + } } From d481b0163fedc1e6b4e6d0e7115b18321d720807 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 15:15:45 -0400 Subject: [PATCH 041/112] test(50-10): add failing tracer for the in-crate monitor harness entry - monitor: LaunchContext and MonitorExec (HarnessEntry under test-support); spawn_monitor takes &LaunchContext (R-9 break, semver gate 50-10 match); the harness-entry re-exec is still a stub that re-execs __monitor - pipeline_launch: launch_context() with a cfg(test) branch that needs a hermetic DEVFLOW_CACHE_DIR; spawn_agent_and_record passes the context - tracer: an in-crate stream launch must run its fake agent, with a Subcommand control that must not; source scans for register_in only and for the feature-gated HarnessEntry variant - every existing spawn_monitor caller passes a context with the same exe Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_launch.rs | 299 +++++++++++++++++- crates/devflow-core/src/monitor.rs | 281 ++++++++++++++-- .../tests/devflow_dir_gitignore.rs | 13 + crates/devflow-core/tests/monitor_e2e.rs | 26 +- 4 files changed, 593 insertions(+), 26 deletions(-) diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index ec7e147f..077d8661 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -992,6 +992,76 @@ pub(crate) fn run_monitor( advance_for_stage(project_root, Some(phase), stage) } +/// Resolve, once per launch, the cache directory and the monitor executable +/// a stage launch uses (50-10). +/// +/// The released binary always re-execs itself as `__monitor` and registers +/// under [`devflow_core::registry::cache_dir`]. There is no environment +/// override of either: the test-build branch below is selected at compile +/// time. +#[cfg(not(test))] +pub(crate) fn launch_context() -> Result { + let cache_dir = devflow_core::registry::cache_dir().ok_or_else(|| { + CliError::Message( + "cannot resolve DevFlow's cache directory for this launch: set HOME, \ + XDG_CACHE_HOME or DEVFLOW_CACHE_DIR" + .to_string(), + ) + })?; + let exe = std::env::current_exe().map_err(|err| { + CliError::Message(format!( + "could not resolve the devflow executable for the stage monitor: {err}" + )) + })?; + Ok(monitor::LaunchContext::new( + cache_dir, + monitor::MonitorExec::Subcommand { exe }, + )) +} + +/// Selects the advance-running monitor harness entry for a `cfg(test)` stage +/// launch when set to `1` (50-10). Read only by the test build of +/// [`launch_context`]. +#[cfg(test)] +pub(crate) const TEST_MONITOR_FULL_ENV: &str = "DEVFLOW_TEST_MONITOR_FULL"; + +/// The test build of [`launch_context`] (50-10, D-07). +/// +/// Inside this crate's unit-test binary, `current_exe()` is libtest, which +/// rejects `__monitor`'s arguments, so the monitor is re-entered at an +/// `#[ignore]`d harness entry in `test_support` that runs the same +/// `run_monitor` body. The cache dir must come from the test process's own +/// `DEVFLOW_CACHE_DIR`, which only a child test process given one through +/// `run_test_in_child`'s `extra_env` has: an in-crate test that launches a +/// stage any other way fails here, loudly, instead of registering under the +/// developer's real HOME. +#[cfg(test)] +pub(crate) fn launch_context() -> Result { + let cache_dir = std::env::var_os("DEVFLOW_CACHE_DIR") + .map(PathBuf::from) + .ok_or_else(|| { + CliError::Message( + "in-crate test launched a stage without a hermetic DEVFLOW_CACHE_DIR; \ + run it through run_test_in_child with extra_env" + .to_string(), + ) + })?; + let exe = std::env::current_exe() + .map_err(|err| CliError::Message(format!("could not resolve the test binary: {err}")))?; + let test_name = if std::env::var_os(TEST_MONITOR_FULL_ENV).is_some_and(|value| value == "1") { + "test_support::monitor_entry_full" + } else { + "test_support::monitor_entry_supervise_only" + }; + Ok(monitor::LaunchContext::new( + cache_dir, + monitor::MonitorExec::HarnessEntry { + exe, + test_name: test_name.to_string(), + }, + )) +} + /// The tail of [`launch_stage_inner`]: clear the stale monitor pid, validate /// the agent binary, archive the prior capture, spawn the monitor, and /// record the launch. Extracted (28-03, Task 2) so @@ -1059,7 +1129,8 @@ fn spawn_agent_and_record( }), ); } - let pid = monitor::spawn_monitor(state, program, args, extra_env, launch) + let ctx = launch_context()?; + let pid = monitor::spawn_monitor(state, program, args, extra_env, launch, &ctx) .map_err(|err| CliError::Message(format!("could not spawn monitor: {err}")))?; // `transition()` calls `workflow::save_state` BEFORE `launch_stage`, so a // pid recorded only in memory here is lost unless it is written again @@ -2192,6 +2263,232 @@ mod tests { since transition() saves state before launch_stage runs" ); } + + /// Poll `done` every 50 ms until it holds or `limit` passes; returns its + /// last answer. + fn poll_until(limit: Duration, mut done: impl FnMut() -> bool) -> bool { + let deadline = std::time::Instant::now() + limit; + loop { + if done() { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(50)); + } + } + + /// A `claude` stand-in that leaves `agent-ran` in its working directory + /// (the project root) and exits 0 without reading its stdin. + fn marker_writing_claude_dir() -> tempfile::TempDir { + let path_dir = agent_free_dir_with_agent_stub("claude"); + std::fs::write( + path_dir.path().join("claude"), + "#!/bin/sh\necho ran > agent-ran\nexit 0\n", + ) + .unwrap(); + path_dir + } + + /// 50-10 tracer (D-07, T-50-27): an in-crate stream launch really runs its + /// agent, because `launch_context()`'s test branch re-enters this test + /// binary at the `test_support::monitor_entry_supervise_only` harness + /// entry, which runs the real `run_monitor` body; and the launch registers + /// only under the cache dir its parent handed it. + /// + /// The control launches the same shape through + /// `MonitorExec::Subcommand` at this test binary, which is what every + /// in-crate stream launch did before 50-10: libtest rejects `__monitor`'s + /// arguments, so the agent never runs. The same "agent ran" check that + /// passes above must fail there, or it proves nothing. + #[test] + fn an_in_crate_stream_launch_runs_its_monitor_through_the_harness_entry() { + const NAME: &str = "pipeline_launch::tests::an_in_crate_stream_launch_runs_its_monitor_through_the_harness_entry"; + if !devflow_core::test_support::in_child_test(NAME) { + let cache = tempfile::Builder::new() + .prefix("dfu") + .tempdir_in("/tmp") + .unwrap(); + let path_dir = marker_writing_claude_dir(); + let output = devflow_core::test_support::run_test_in_child( + NAME, + path_dir.path(), + &[("DEVFLOW_CACHE_DIR", cache.path().as_os_str())], + ); + devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, NAME); + return; + } + let cache = PathBuf::from( + std::env::var_os("DEVFLOW_CACHE_DIR").expect("the parent passes DEVFLOW_CACHE_DIR"), + ); + let prompt = || monitor::MonitorLaunch::PipeOwning { + prompt: "50-10 tracer".to_string(), + }; + + // The harness-entry launch. + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + init_repo(root); + let phase = PhaseId::new(1010); + let mut state = State::new( + phase, + AgentKind::Claude, + Mode::Supervise, + root.to_path_buf(), + ); + state.stage = Stage::Code; + workflow::save_state(&state).unwrap(); + let launched = spawn_agent_and_record(&mut state, "claude", &[], &[], None, prompt()); + let _reap = ReapMonitorOnDrop::after_launch(&state); + launched.unwrap(); + let exit_path = devflow_core::agent_result::exit_code_path(root, phase); + let exited = poll_until(Duration::from_secs(30), || { + std::fs::read_to_string(&exit_path).is_ok_and(|code| !code.trim().is_empty()) + }); + assert!( + root.join("agent-ran").is_file(), + "the harness entry must run the agent: no marker at {} (exit file written: {exited})", + root.join("agent-ran").display() + ); + assert_eq!( + std::fs::read_to_string(&exit_path).unwrap().trim(), + "0", + "the supervised agent's exit code must be recorded" + ); + let entry = devflow_core::registry::entry_path_in(&cache, &state.project_root, phase); + assert!( + entry.is_file(), + "the launch must register under its own cache dir: no {}", + entry.display() + ); + + // The control: the pre-50-10 re-exec of this test binary. + let control_dir = tempfile::tempdir().unwrap(); + let control_root = control_dir.path(); + let control_phase = PhaseId::new(1011); + let mut control = State::new( + control_phase, + AgentKind::Claude, + Mode::Supervise, + control_root.to_path_buf(), + ); + control.stage = Stage::Code; + let subcommand = monitor::LaunchContext::new( + cache.clone(), + monitor::MonitorExec::Subcommand { + exe: std::env::current_exe().unwrap(), + }, + ); + let control_pid = + monitor::spawn_monitor(&control, "claude", &[], &[], prompt(), &subcommand).unwrap(); + control.monitor_pid = Some(control_pid); + let _control_reap = ReapMonitorOnDrop::after_launch(&control); + assert!( + poll_until(Duration::from_secs(30), || { + !devflow_core::agent::agent_running(control_pid) + }), + "the control monitor (libtest given __monitor's arguments) must exit by itself" + ); + assert!( + !control_root.join("agent-ran").exists(), + "control: re-execing the test binary as __monitor must NOT run the agent" + ); + assert!( + !devflow_core::agent_result::exit_code_path(control_root, control_phase).exists(), + "control: no monitor body ran, so no exit file may exist" + ); + } + + /// The CLI source files under `src/`, found from `CARGO_MANIFEST_DIR`. + fn cli_source_files() -> Vec { + fn walk(dir: &Path, found: &mut Vec) { + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + walk(&path, found); + } else if path.extension().is_some_and(|ext| ext == "rs") { + found.push(path); + } + } + } + let mut found = Vec::new(); + walk( + &Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), + &mut found, + ); + found.sort(); + assert!( + found + .iter() + .any(|path| path.ends_with("src/pipeline_launch.rs")), + "the scan must see this file, or it scanned the wrong directory: {found:?}" + ); + found + } + + /// `source` with every line whose first non-blank characters are `//` + /// dropped. + fn code_lines(source: &str) -> impl Iterator { + source + .lines() + .filter(|line| !line.trim_start().starts_with("//")) + } + + /// Lines of `source` (comments dropped) that register a root through the + /// env-reading `registry::register(`. `register_in(` does not match: `_` + /// follows `register` there. + fn unscoped_register_calls(source: &str) -> Vec { + let needle = concat!("registry::", "register("); + code_lines(source) + .filter(|line| line.contains(needle)) + .map(str::to_string) + .collect() + } + + /// T-50-27: every root registration in the CLI goes through + /// `register_in(ctx.cache_dir, …)`, so a test's cache dir is the only + /// place an in-crate launch can register. + #[test] + fn cli_source_registers_roots_only_through_register_in() { + let planted = concat!( + " let _ = devflow_core::registry::", + "register(&root, phase);\n" + ); + assert_eq!( + unscoped_register_calls(planted).len(), + 1, + "planted control: a live unscoped register call must be flagged" + ); + let commented = concat!( + " // let _ = devflow_core::registry::", + "register(&root, phase);\n" + ); + assert!( + unscoped_register_calls(commented).is_empty(), + "planted control: the same call behind // must not be flagged" + ); + let scoped = concat!( + " let _ = devflow_core::registry::", + "register_in(&ctx.cache_dir, &root, phase);\n" + ); + assert!( + unscoped_register_calls(scoped).is_empty(), + "register_in( must not be flagged" + ); + + let mut offenders = Vec::new(); + for path in cli_source_files() { + let source = std::fs::read_to_string(&path).unwrap(); + for line in unscoped_register_calls(&source) { + offenders.push(format!("{}: {}", path.display(), line.trim())); + } + } + assert!( + offenders.is_empty(), + "CLI source must register roots only through registry::register_in: {offenders:#?}" + ); + } /// 20c (review: Codex MEDIUM — resume semantics): a phase halted by /// `--until ` has `stopped`/`stop_reason`/`stop_until` persisted. /// `resume` must clear all three BEFORE relaunching — otherwise diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index d29ffa8f..fcb4804e 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -304,6 +304,70 @@ pub enum MonitorLaunch { Legacy, } +/// Name of the environment variable that carries the `__monitor` argv (the +/// arguments after `__monitor`, as a JSON array of strings) to a test +/// binary's monitor harness entry. Set only by +/// [`MonitorExec::HarnessEntry`]; no release build reads it. +#[cfg(feature = "test-support")] +pub const MONITOR_ARGV_ENV: &str = "DEVFLOW_MONITOR_ARGV"; + +/// Which executable a stream launch's detached monitor re-execs (50-10). +/// +/// In the `devflow` binary this is always [`MonitorExec::Subcommand`]: the +/// binary itself, run as its hidden `__monitor` subcommand. Inside a test +/// binary, `current_exe()` is libtest, which rejects `__monitor`'s arguments, +/// so a test build can instead name an `#[ignore]`d test that runs the same +/// monitor body. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MonitorExec { + /// ` __monitor `. + Subcommand { + /// The `devflow` executable to re-exec. + exe: PathBuf, + }, + /// ` --exact --include-ignored --test-threads=1`, with + /// the `__monitor` argv in [`MONITOR_ARGV_ENV`]. Test builds only: the + /// variant does not exist without the `test-support` feature. + #[cfg(feature = "test-support")] + HarnessEntry { + /// The test binary to re-exec. + exe: PathBuf, + /// The harness entry's full test name. + test_name: String, + }, +} + +impl MonitorExec { + /// The executable this monitor runs, whichever way it is entered. + fn exe(&self) -> &Path { + match self { + Self::Subcommand { exe } => exe, + #[cfg(feature = "test-support")] + Self::HarnessEntry { exe, .. } => exe, + } + } +} + +/// Everything a stage launch resolves once, in one place, before it spawns a +/// monitor (50-10): the machine-global cache directory its registration (and, +/// from 50-11, its socket) lives under, and the executable its monitor runs. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LaunchContext { + /// The DevFlow cache directory for this launch. + pub cache_dir: PathBuf, + /// The monitor executable and how it is entered. + pub exec: MonitorExec, +} + +impl LaunchContext { + /// A launch context from its two parts. + pub fn new(cache_dir: PathBuf, exec: MonitorExec) -> Self { + Self { cache_dir, exec } + } +} + /// Strip `DEVFLOW_GATE_NOTIFY_CMD` from a detached monitor chain /// (49-REVIEW.md WR-07). Every gate in the chain then resolves the command /// from the run's state file at the moment it fires, so editing @@ -325,7 +389,10 @@ pub(crate) fn strip_notify_env(child_cmd: &mut std::process::Command) { /// 3. Runs `devflow advance --phase N` to advance the workflow through its /// remaining stages /// -/// `launch` selects the supervision shape — see [`MonitorLaunch`]. +/// `launch` selects the supervision shape — see [`MonitorLaunch`]. `ctx` +/// names the monitor executable: the `PipeOwning` arm re-execs it as +/// [`LaunchContext::exec`] says, and the `Legacy` arm's `advance` tail runs +/// the same executable. /// /// Returns the PID of the spawned monitor. pub fn spawn_monitor( @@ -334,8 +401,17 @@ pub fn spawn_monitor( args: &[String], envs: &[(String, String)], launch: MonitorLaunch, + ctx: &LaunchContext, ) -> Result { - spawn_monitor_inner(state, program, args, envs, launch, MonitorTail::Advance) + spawn_monitor_inner( + state, + program, + args, + envs, + launch, + ctx, + MonitorTail::Advance, + ) } /// What the Legacy monitor runs after it records the agent's exit code. @@ -355,6 +431,7 @@ fn spawn_monitor_inner( args: &[String], envs: &[(String, String)], launch: MonitorLaunch, + ctx: &LaunchContext, tail: MonitorTail, ) -> Result { let project_root = state @@ -362,8 +439,9 @@ fn spawn_monitor_inner( .to_str() .ok_or(MonitorError::NonUtf8Path)?; - let binary = std::env::current_exe() - .map_err(|_| MonitorError::NoBinaryPath)? + let binary = ctx + .exec + .exe() .to_str() .ok_or(MonitorError::NonUtf8Path)? .to_string(); @@ -466,18 +544,17 @@ fn spawn_monitor_inner( // comment gives: `hermetic_command` does its `env_remove`s at // construction and `.envs(...)` runs after, so deliberate // configuration survives while inherited pollution does not. - let mut child_cmd = hermetic_command(&binary, workdir_path); - child_cmd - .args(pipe_owning_monitor_args( - project_root, - state, - workdir, - prompt_file, - idle.timeout.as_secs(), - )) - .arg(program) - .args(args) - .envs(envs.iter().map(|(k, v)| (k.as_str(), v.as_str()))); + let mut monitor_argv = pipe_owning_monitor_args( + project_root, + state, + workdir, + prompt_file, + idle.timeout.as_secs(), + ); + monitor_argv.push(program.to_string()); + monitor_argv.extend(args.iter().cloned()); + let mut child_cmd = monitor_command(&ctx.exec, &binary, workdir_path, &monitor_argv)?; + child_cmd.envs(envs.iter().map(|(k, v)| (k.as_str(), v.as_str()))); strip_notify_env(&mut child_cmd); @@ -642,6 +719,30 @@ fn pipe_owning_monitor_args( ] } +/// The `PipeOwning` re-exec command for `exec`, before the adapter env is +/// applied. `monitor_argv` starts with `__monitor` and ends with the child's +/// program and argv; `binary` is `exec`'s executable as UTF-8. +fn monitor_command( + exec: &MonitorExec, + binary: &str, + workdir: &Path, + monitor_argv: &[String], +) -> Result { + let mut command = hermetic_command(binary, workdir); + match exec { + MonitorExec::Subcommand { .. } => { + command.args(monitor_argv); + } + // RED (50-10 Task 1): the harness entry is not wired yet, so a test + // build still re-execs the test binary as `__monitor`. + #[cfg(feature = "test-support")] + MonitorExec::HarnessEntry { .. } => { + command.args(monitor_argv); + } + } + Ok(command) +} + fn advance_tail(binary: &str, project_root: &str, state: &State) -> String { format!( "; {binary} advance {project_root} --phase {phase} --stage {stage}", @@ -2000,6 +2101,91 @@ mod tests { state } + /// The launch context every pre-50-10 `spawn_monitor` test passes: this + /// test binary as the monitor executable, exactly what `current_exe()` + /// resolved to inside `spawn_monitor` before 50-10, and a cache dir under + /// the test's own project tempdir. + fn test_launch_context(state: &State) -> LaunchContext { + LaunchContext::new( + state.project_root.join("cache"), + MonitorExec::Subcommand { + exe: std::env::current_exe().unwrap(), + }, + ) + } + + /// T-50-26's source scan: whether the first `HarnessEntry {` after + /// `enum MonitorExec` in `source` carries `cfg(feature = "test-support")` + /// on its own line or on one of the two lines above it. The source is cut + /// at its first `mod tests {` line and `//` lines are dropped first, so + /// neither this test nor a comment can satisfy or fail the scan. `None` + /// when there is no such enum or variant. + fn harness_entry_is_feature_gated(source: &str) -> Option { + let code: Vec<&str> = source + .lines() + .take_while(|line| line.trim() != concat!("mod ", "tests {")) + .filter(|line| !line.trim_start().starts_with("//")) + .collect(); + let enum_at = code + .iter() + .position(|line| line.contains(concat!("enum ", "MonitorExec")))?; + let variant_at = enum_at + + code[enum_at..] + .iter() + .position(|line| line.contains(concat!("HarnessEntry", " {")))?; + let gate = concat!("cfg(feature = ", "\"test-support\")"); + let window_start = variant_at.saturating_sub(2).max(enum_at); + Some( + code[window_start..=variant_at] + .iter() + .any(|line| line.contains(gate)), + ) + } + + /// T-50-26: the test-only monitor entry cannot exist in a release build, + /// because its variant is compiled only under `test-support`. + #[test] + fn the_harness_entry_variant_is_declared_only_under_the_test_support_feature() { + let ungated = concat!( + "pub enum ", + "MonitorExec {\n", + " Subcommand { exe: PathBuf },\n", + " /// docs\n", + " HarnessEntry", + " { exe: PathBuf, test_name: String },\n", + "}\n" + ); + assert_eq!( + harness_entry_is_feature_gated(ungated), + Some(false), + "planted control: an ungated HarnessEntry must be reported ungated" + ); + let gated = concat!( + "pub enum ", + "MonitorExec {\n", + " Subcommand { exe: PathBuf },\n", + " #[cfg(feature = ", + "\"test-support\")]\n", + " HarnessEntry", + " { exe: PathBuf, test_name: String },\n", + "}\n" + ); + assert_eq!( + harness_entry_is_feature_gated(gated), + Some(true), + "planted control: a gated HarnessEntry must be reported gated" + ); + + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/monitor.rs"); + let source = std::fs::read_to_string(&path).unwrap(); + assert_eq!( + harness_entry_is_feature_gated(&source), + Some(true), + "{}: MonitorExec::HarnessEntry must be declared under cfg(feature = \"test-support\")", + path.display() + ); + } + // ---- 45-01 Task 3: idle-timeout commit evidence resolves its trunk // from the project, not from the phase worktree it runs `git log` in. @@ -3078,7 +3264,15 @@ kill -9 $$ // Stub agent: write a known marker to stdout, then exit cleanly. let args = vec!["-c".to_string(), "echo MONITOR_READY".to_string()]; - let monitor_pid = spawn_monitor(&state, "sh", &args, &[], MonitorLaunch::Legacy).unwrap(); + let monitor_pid = spawn_monitor( + &state, + "sh", + &args, + &[], + MonitorLaunch::Legacy, + &test_launch_context(&state), + ) + .unwrap(); assert!(monitor_pid > 0); // Observable side effect #1: the monitor records the agent PID to its @@ -3156,7 +3350,15 @@ kill -9 $$ // window this test needs to send SIGTERM and check liveness. let args = vec!["-c".to_string(), "sleep 30".to_string()]; - let monitor_pid = spawn_monitor(&state, "sh", &args, &[], MonitorLaunch::Legacy).unwrap(); + let monitor_pid = spawn_monitor( + &state, + "sh", + &args, + &[], + MonitorLaunch::Legacy, + &test_launch_context(&state), + ) + .unwrap(); let agent_pid = wait_for_agent_pid(dir.path(), state.phase) .expect("monitor should record the agent pid"); assert!( @@ -3373,8 +3575,15 @@ pid_t fork(void) { .map(|(key, value)| (key.to_string(), value.to_string())); let state = state_in(root); let args = vec!["-c".to_string(), "sleep 30".to_string()]; - let monitor_pid = - spawn_monitor(&state, "sh", &args, &envs, MonitorLaunch::Legacy).unwrap(); + let monitor_pid = spawn_monitor( + &state, + "sh", + &args, + &envs, + MonitorLaunch::Legacy, + &test_launch_context(&state), + ) + .unwrap(); Self { state, monitor_pid, @@ -3704,6 +3913,7 @@ pid_t fork(void) { &args, &[], MonitorLaunch::Legacy, + &test_launch_context(&state), MonitorTail::Script(String::new()), ) .unwrap(); @@ -3772,6 +3982,7 @@ pid_t fork(void) { &args, &[], MonitorLaunch::Legacy, + &test_launch_context(&state), MonitorTail::Script(tail), ) .unwrap(); @@ -3850,7 +4061,15 @@ pid_t fork(void) { // directories before launching the agent. let args = vec!["-c".to_string(), "pwd; echo WORKTREE_READY".to_string()]; - let monitor_pid = spawn_monitor(&state, "sh", &args, &[], MonitorLaunch::Legacy).unwrap(); + let monitor_pid = spawn_monitor( + &state, + "sh", + &args, + &[], + MonitorLaunch::Legacy, + &test_launch_context(&state), + ) + .unwrap(); assert!(monitor_pid > 0); let agent_pid = wait_for_agent_pid(dir.path(), state.phase) @@ -3936,7 +4155,15 @@ pid_t fork(void) { "git rev-parse --absolute-git-dir".to_string(), ]; - spawn_monitor(&state, "sh", &args, &[], MonitorLaunch::Legacy).unwrap(); + spawn_monitor( + &state, + "sh", + &args, + &[], + MonitorLaunch::Legacy, + &test_launch_context(&state), + ) + .unwrap(); wait_for_agent_pid(&root, state.phase).expect("monitor should record the agent pid"); let stdout_path = crate::agent_result::stdout_path(&root, state.phase); @@ -4017,7 +4244,15 @@ pid_t fork(void) { payload.to_string(), ]; - spawn_monitor(&state, "sh", &args, &[], MonitorLaunch::Legacy).unwrap(); + spawn_monitor( + &state, + "sh", + &args, + &[], + MonitorLaunch::Legacy, + &test_launch_context(&state), + ) + .unwrap(); wait_for_agent_pid(dir.path(), state.phase).expect("monitor should record the agent pid"); let stdout_path = crate::agent_result::stdout_path(dir.path(), state.phase); diff --git a/crates/devflow-core/tests/devflow_dir_gitignore.rs b/crates/devflow-core/tests/devflow_dir_gitignore.rs index b8740615..4dba93be 100644 --- a/crates/devflow-core/tests/devflow_dir_gitignore.rs +++ b/crates/devflow-core/tests/devflow_dir_gitignore.rs @@ -82,6 +82,18 @@ fn wait_for_pid_to_die(pid: u32, timeout: Duration) { /// against its own fresh temp project root. Failures accumulate into a /// vector so one regressed site doesn't hide another (mirrors the /// accumulate-then-assert shape `gitignore_coverage.rs` already uses). +/// The launch context case 3 passes `spawn_monitor` (50-10): this test binary +/// as the monitor executable, which is what `spawn_monitor` resolved itself +/// before it took a context, and a cache dir under the case's own tempdir. +fn launch_context(root: &Path) -> monitor::LaunchContext { + monitor::LaunchContext::new( + root.join("cache"), + monitor::MonitorExec::Subcommand { + exe: std::env::current_exe().unwrap(), + }, + ) +} + #[test] fn all_seven_devflow_constructors_produce_the_gitignore() { let mut failures: Vec<&str> = Vec::new(); @@ -145,6 +157,7 @@ fn all_seven_devflow_constructors_produce_the_gitignore() { &["-c".to_string(), "exit 0".to_string()], &[], monitor::MonitorLaunch::Legacy, + &launch_context(root), ) .expect("spawn_monitor"); diff --git a/crates/devflow-core/tests/monitor_e2e.rs b/crates/devflow-core/tests/monitor_e2e.rs index 16c0e1da..f4a17f38 100644 --- a/crates/devflow-core/tests/monitor_e2e.rs +++ b/crates/devflow-core/tests/monitor_e2e.rs @@ -16,7 +16,9 @@ use devflow_core::agent_result::{ }; use devflow_core::config::GitFlowConfig; use devflow_core::mode::Mode; -use devflow_core::monitor::{MonitorLaunch, spawn_monitor, wait_for_agent_pid}; +use devflow_core::monitor::{ + LaunchContext, MonitorExec, MonitorLaunch, spawn_monitor, wait_for_agent_pid, +}; use devflow_core::phase_id::PhaseId; use devflow_core::stage::Stage; use devflow_core::state::{AgentKind, State}; @@ -58,6 +60,18 @@ fn init_repo(root: &Path, phase: PhaseId) { git(root, &["commit", "-q", "-m", "agent work"]); } +/// The launch context this suite passes `spawn_monitor` (50-10): this test +/// binary as the monitor executable, which is what `spawn_monitor` resolved +/// itself before it took a context, and a cache dir under the test's tempdir. +fn launch_context(root: &Path) -> LaunchContext { + LaunchContext::new( + root.join("cache"), + MonitorExec::Subcommand { + exe: std::env::current_exe().unwrap(), + }, + ) +} + #[test] fn monitor_owns_fake_agent_and_records_devflow_result() { let dir = tempfile::tempdir().unwrap(); @@ -75,7 +89,15 @@ fn monitor_owns_fake_agent_and_records_devflow_result() { .to_string(), ]; - spawn_monitor(&state, "sh", &args, &[], MonitorLaunch::Legacy).expect("spawn monitor"); + spawn_monitor( + &state, + "sh", + &args, + &[], + MonitorLaunch::Legacy, + &launch_context(root), + ) + .expect("spawn monitor"); // The monitor records the agent PID. let agent_pid = wait_for_agent_pid(root, phase).expect("agent pid recorded"); From 26fdc5b77d367e0e95bfba27d8c961fb1b5207b3 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 15:18:48 -0400 Subject: [PATCH 042/112] feat(50-10): re-enter in-crate stream monitors at a test harness entry - monitor: MonitorExec::HarnessEntry re-execs the test binary at the named entry with --exact --include-ignored --test-threads=1 and passes the __monitor argv as JSON in DEVFLOW_MONITOR_ARGV - test_support: #[ignore]d monitor_entry_full and monitor_entry_supervise_only parse that argv with the real Cli and run run_monitor's body, with and without the advance tail - pipeline_launch: supervise_monitor carries the advance switch behind run_monitor; spawn_agent_and_record registers with register_in under the launch context's cache dir Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_launch.rs | 37 +++++++++++- crates/devflow-cli/src/test_support.rs | 68 +++++++++++++++++++++++ crates/devflow-core/src/monitor.rs | 16 ++++-- 3 files changed, 116 insertions(+), 5 deletions(-) diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index 077d8661..e11dfa1f 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -924,6 +924,38 @@ pub(crate) fn run_monitor( idle_timeout_secs: u64, agent: AgentKind, argv: &[String], +) -> Result<(), CliError> { + supervise_monitor( + project_root, + phase, + stage, + workdir, + prompt_file, + idle_timeout_secs, + agent, + argv, + true, + ) +} + +/// [`run_monitor`]'s body. `advance` is `true` on every production path; the +/// test build's supervise-only monitor harness entry (50-10) passes `false` +/// so an in-crate launch supervises its agent without moving the stage +/// machine on. +#[allow( + clippy::too_many_arguments, + reason = "run_monitor's independently parsed monitor arguments plus the advance switch" +)] +pub(crate) fn supervise_monitor( + project_root: &Path, + phase: PhaseId, + stage: Option, + workdir: &Path, + prompt_file: &Path, + idle_timeout_secs: u64, + agent: AgentKind, + argv: &[String], + advance: bool, ) -> Result<(), CliError> { let prompt = std::fs::read_to_string(prompt_file).map_err(|err| { CliError::Message(format!( @@ -989,6 +1021,9 @@ pub(crate) fn run_monitor( ) .map_err(|err| CliError::Message(format!("pipe-owning monitor failed: {err}")))?; + if !advance { + return Ok(()); + } advance_for_stage(project_root, Some(phase), stage) } @@ -1161,7 +1196,7 @@ fn spawn_agent_and_record( // registry on the same code path that just recorded monitor_pid, so a // running phase cannot be missing from `devflow gate list --all-roots`. // Best-effort observability — never a reason to fail a launch. - let _ = devflow_core::registry::register(&state.project_root, state.phase); + let _ = devflow_core::registry::register_in(&ctx.cache_dir, &state.project_root, state.phase); events::emit( &state.project_root, state.phase, diff --git a/crates/devflow-cli/src/test_support.rs b/crates/devflow-cli/src/test_support.rs index 1f0e7dbf..cf228802 100644 --- a/crates/devflow-cli/src/test_support.rs +++ b/crates/devflow-cli/src/test_support.rs @@ -635,6 +635,74 @@ impl Drop for ReapMonitorOnDrop { } } +/// Parse the `__monitor` argv a [`MonitorExec::HarnessEntry`] re-exec put in +/// [`MONITOR_ARGV_ENV`] and run it as `main` dispatches `__monitor`, with or +/// without the advance tail. Returns without doing anything when the variable +/// is absent, so a run of ignored tests that reaches an entry is a no-op. +/// +/// [`MonitorExec::HarnessEntry`]: devflow_core::monitor::MonitorExec::HarnessEntry +/// [`MONITOR_ARGV_ENV`]: devflow_core::monitor::MONITOR_ARGV_ENV +fn run_monitor_harness_entry(advance: bool) { + use clap::Parser as _; + let Some(encoded) = std::env::var_os(devflow_core::monitor::MONITOR_ARGV_ENV) else { + return; + }; + let encoded = encoded + .into_string() + .expect("the monitor argv must be UTF-8 JSON"); + let after_subcommand: Vec = + serde_json::from_str(&encoded).expect("the monitor argv must be a JSON array of strings"); + let argv = ["devflow", "__monitor"] + .into_iter() + .map(str::to_string) + .chain(after_subcommand); + let cli = crate::Cli::try_parse_from(argv).expect("the monitor argv must parse as __monitor"); + let crate::Command::Monitor { + project, + phase, + stage, + workdir, + prompt_file, + idle_timeout_secs, + agent, + argv, + } = cli.command + else { + panic!("the monitor argv parsed as a command other than __monitor"); + }; + let project_root = crate::project_root(project).expect("monitor project root"); + crate::pipeline_launch::supervise_monitor( + &project_root, + phase, + stage, + &workdir, + &prompt_file, + idle_timeout_secs, + agent, + &argv, + advance, + ) + .expect("the harness-entry monitor must supervise its agent"); +} + +/// The test build's full monitor entry (50-10): `run_monitor` exactly as the +/// `__monitor` subcommand runs it, advance tail included. Selected by +/// `pipeline_launch::launch_context` when `DEVFLOW_TEST_MONITOR_FULL=1`. +#[test] +#[ignore = "monitor re-exec entry; run only by spawn_monitor under cfg(test)"] +fn monitor_entry_full() { + run_monitor_harness_entry(true); +} + +/// The test build's default monitor entry (50-10): `run_monitor`'s +/// supervision without its advance tail, so an in-crate launch supervises its +/// fake agent and leaves the stage machine where the test put it. +#[test] +#[ignore = "monitor re-exec entry; run only by spawn_monitor under cfg(test)"] +fn monitor_entry_supervise_only() { + run_monitor_harness_entry(false); +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index fcb4804e..08a41ca4 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -733,11 +733,19 @@ fn monitor_command( MonitorExec::Subcommand { .. } => { command.args(monitor_argv); } - // RED (50-10 Task 1): the harness entry is not wired yet, so a test - // build still re-execs the test binary as `__monitor`. + // libtest cannot parse `__monitor`'s arguments, so they travel in the + // environment and the test binary runs exactly the named entry. #[cfg(feature = "test-support")] - MonitorExec::HarnessEntry { .. } => { - command.args(monitor_argv); + MonitorExec::HarnessEntry { test_name, .. } => { + let after_subcommand = match monitor_argv.split_first() { + Some((first, rest)) if first == "__monitor" => rest, + _ => monitor_argv, + }; + let encoded = serde_json::to_string(after_subcommand).map_err(std::io::Error::other)?; + command + .arg(test_name) + .args(["--exact", "--include-ignored", "--test-threads=1"]) + .env(MONITOR_ARGV_ENV, encoded); } } Ok(command) From 6633b34306d8fe3eac124702be0dcdb415a872ea Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 15:26:14 -0400 Subject: [PATCH 043/112] test(50-10): run every in-crate child test with a hermetic cache dir - test_support: run_hermetic_test_in_child passes a parent-owned /tmp/dfu... DEVFLOW_CACHE_DIR to each child test (and run_test_without_git does the same); every CLI child-test call site uses it, so no launch registers under the developer's HOME - pipeline_launch: permanent source scan that no set_var/remove_var names a hermetic variable and the crate keeps at most the 36 guarded 48-09 env mutations, with planted controls - doc-check-allowlist: reasoned source_to_docs exceptions for the test-only DEVFLOW_MONITOR_ARGV and DEVFLOW_TEST_MONITOR_FULL Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_gate.rs | 2 +- crates/devflow-cli/src/pipeline_launch.rs | 189 ++++++++++++++++---- crates/devflow-cli/src/pipeline_outcomes.rs | 82 ++++----- crates/devflow-cli/src/preflight.rs | 2 +- crates/devflow-cli/src/staleness.rs | 2 +- crates/devflow-cli/src/test_support.rs | 37 +++- doc-check-allowlist.toml | 10 ++ 7 files changed, 245 insertions(+), 79 deletions(-) diff --git a/crates/devflow-cli/src/pipeline_gate.rs b/crates/devflow-cli/src/pipeline_gate.rs index 7c3ad9c3..3c38c6d5 100644 --- a/crates/devflow-cli/src/pipeline_gate.rs +++ b/crates/devflow-cli/src/pipeline_gate.rs @@ -727,7 +727,7 @@ mod tests { if !devflow_core::test_support::in_child_test($name) { let path_dir = $path_dir; let output = - devflow_core::test_support::run_test_in_child($name, path_dir.path(), &[]); + crate::test_support::run_hermetic_test_in_child($name, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test( &output, $name, ); diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index e11dfa1f..ecb9e796 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -2081,7 +2081,7 @@ mod tests { if !devflow_core::test_support::in_child_test($name) { let path_dir = agent_free_dir_with_agent_stub($program); let output = - devflow_core::test_support::run_test_in_child($name, path_dir.path(), &[]); + crate::test_support::run_hermetic_test_in_child($name, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test( &output, $name, ); @@ -2241,7 +2241,8 @@ mod tests { const NAME: &str = "pipeline_launch::tests::launch_stage_persists_monitor_pid_for_reload"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let output = + crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, NAME); return; } @@ -2341,16 +2342,10 @@ mod tests { fn an_in_crate_stream_launch_runs_its_monitor_through_the_harness_entry() { const NAME: &str = "pipeline_launch::tests::an_in_crate_stream_launch_runs_its_monitor_through_the_harness_entry"; if !devflow_core::test_support::in_child_test(NAME) { - let cache = tempfile::Builder::new() - .prefix("dfu") - .tempdir_in("/tmp") - .unwrap(); + // The child gets a parent-owned `/tmp/dfu…` DEVFLOW_CACHE_DIR. let path_dir = marker_writing_claude_dir(); - let output = devflow_core::test_support::run_test_in_child( - NAME, - path_dir.path(), - &[("DEVFLOW_CACHE_DIR", cache.path().as_os_str())], - ); + let output = + crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, NAME); return; } @@ -2524,6 +2519,123 @@ mod tests { "CLI source must register roots only through registry::register_in: {offenders:#?}" ); } + + /// The 48-09 D-09 ENV_MUTEX-guarded `set_var`/`remove_var` calls already + /// in this crate's tests (preflight 19, pipeline_launch 7, pipeline_gate + /// 6, pipeline_outcomes 4; counted 2026-10-14 and again 2026-10-16). + const GUARDED_ENV_MUTATION_CEILING: usize = 36; + + /// The process-env mutators, spelled through `concat!` so this file adds + /// nothing to its own scan. + const ENV_MUTATORS: [&str; 2] = [concat!("env::", "set_var"), concat!("env::", "remove_var")]; + + /// The variables 50-10's cache isolation and harness-entry selection must + /// reach a child only through `run_test_in_child`'s `extra_env`. + const HERMETIC_ENV_LITERALS: [&str; 6] = [ + concat!("\"DEVFLOW_", "CACHE_DIR\""), + concat!("\"HO", "ME\""), + concat!("\"XDG_", "CACHE_HOME\""), + concat!("\"XDG_", "CONFIG_HOME\""), + concat!("\"DEVFLOW_", "MONITOR_ARGV\""), + concat!("\"DEVFLOW_", "TEST_MONITOR_FULL\""), + ]; + + /// `source` with `//` lines dropped and the rest joined, so a call split + /// across lines still reads as one. + fn joined_code(source: &str) -> String { + code_lines(source).collect::>().join("\n") + } + + /// Calls in comment-stripped `source` that mutate one of + /// [`HERMETIC_ENV_LITERALS`]: a mutator, `(`, optional whitespace, then + /// the variable's string literal. + fn hermetic_env_mutations(source: &str) -> Vec { + let code = joined_code(source); + let mut found = Vec::new(); + for mutator in ENV_MUTATORS { + let call = format!("{mutator}("); + for (at, _) in code.match_indices(&call) { + let argument = code[at + call.len()..].trim_start(); + if let Some(literal) = HERMETIC_ENV_LITERALS + .iter() + .find(|literal| argument.starts_with(**literal)) + { + found.push(format!("{mutator}({literal}")); + } + } + } + found + } + + /// Every `set_var`/`remove_var` in comment-stripped `source`. + fn env_mutation_count(source: &str) -> usize { + let code = joined_code(source); + ENV_MUTATORS + .iter() + .map(|mutator| code.matches(mutator).count()) + .sum() + } + + /// 50-10 Task 2: the hermetic variables reach a child test process only + /// through `extra_env`, never through a process-global mutation, and this + /// plan adds no other process-env mutation to the crate (the 36 guarded + /// 48-09 calls stay as they are). + #[test] + fn cli_source_sets_no_hermetic_env_var_and_adds_no_process_env_mutation() { + let split_call = concat!( + " std::env::set", + "_var(\n", + " \"DEVFLOW_", + "CACHE_DIR\", dir);\n" + ); + assert_eq!( + hermetic_env_mutations(split_call).len(), + 1, + "planted control: a two-line mutation of the cache variable must be flagged" + ); + let other_var = concat!( + " std::env::set", + "_var(\n", + " \"DEVFLOW_", + "OTHER\", dir);\n" + ); + assert!( + hermetic_env_mutations(other_var).is_empty(), + "planted control: a mutation of an unrelated variable must not be flagged" + ); + let three_calls = concat!( + " std::env::set", + "_var(\"A\", \"1\");\n", + " std::env::remove", + "_var(\"A\");\n", + " // std::env::set", + "_var(\"A\", \"2\");\n" + ); + assert_eq!( + env_mutation_count(three_calls), + 2, + "planted control: one set, one remove and one commented-out set count 2" + ); + + let mut hermetic = Vec::new(); + let mut total = 0; + for path in cli_source_files() { + let source = std::fs::read_to_string(&path).unwrap(); + for call in hermetic_env_mutations(&source) { + hermetic.push(format!("{}: {call}", path.display())); + } + total += env_mutation_count(&source); + } + assert!( + hermetic.is_empty(), + "the hermetic variables must reach a child only through extra_env: {hermetic:#?}" + ); + assert!( + total <= GUARDED_ENV_MUTATION_CEILING, + "CLI source holds {total} set_var/remove_var calls, more than the \ + {GUARDED_ENV_MUTATION_CEILING} ENV_MUTEX-guarded 48-09 calls" + ); + } /// 20c (review: Codex MEDIUM — resume semantics): a phase halted by /// `--until ` has `stopped`/`stop_reason`/`stop_until` persisted. /// `resume` must clear all three BEFORE relaunching — otherwise @@ -2545,7 +2657,8 @@ mod tests { "pipeline_launch::tests::resume_clears_stop_marker_and_advances_past_stop_point"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let output = + crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, NAME); return; } @@ -2779,7 +2892,8 @@ mod tests { const NAME: &str = "pipeline_launch::tests::resume_preserves_unfired_until_cap"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let output = + crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, NAME); return; } @@ -2856,7 +2970,8 @@ mod tests { const NAME: &str = "pipeline_launch::tests::resume_without_a_cap_is_unchanged"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let output = + crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, NAME); return; } @@ -2911,7 +3026,8 @@ mod tests { const NAME: &str = "pipeline_launch::tests::resume_with_agent_hands_off_and_relaunches_under_the_new_driver"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_dir_with_agent_stub("codex"); - let output = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let output = + crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, NAME); return; } @@ -2964,7 +3080,8 @@ mod tests { "pipeline_launch::tests::resume_without_agent_leaves_the_saved_agent_untouched"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let output = + crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, NAME); return; } @@ -3062,7 +3179,8 @@ mod tests { "pipeline_launch::tests::resume_with_same_agent_is_an_ordinary_idempotent_resume"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let output = + crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, NAME); return; } @@ -3102,7 +3220,8 @@ mod tests { const NAME: &str = "pipeline_launch::tests::resume_with_agent_allows_plan_stage"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_dir_with_agent_stub("codex"); - let output = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let output = + crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, NAME); return; } @@ -3132,7 +3251,8 @@ mod tests { const NAME: &str = "pipeline_launch::tests::resume_with_agent_preserves_every_state_field_except_agent_and_monitor_pid"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_dir_with_agent_stub("codex"); - let output = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let output = + crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, NAME); return; } @@ -3205,7 +3325,8 @@ mod tests { "pipeline_launch::tests::resume_with_agent_from_a_rate_limited_state_relaunches"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_dir_with_agent_stub("codex"); - let output = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let output = + crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, NAME); return; } @@ -4489,7 +4610,7 @@ mod tests { workflow::save_state(&state).unwrap(); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -4554,7 +4675,7 @@ mod tests { workflow::save_state(&state).unwrap(); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -4799,7 +4920,7 @@ mod tests { Mode::Auto, ); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -4900,7 +5021,7 @@ mod tests { Mode::Supervise, ); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -4976,7 +5097,7 @@ mod tests { Mode::Auto, ); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -5131,7 +5252,7 @@ mod tests { Mode::Supervise, ); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -5241,7 +5362,7 @@ mod tests { Mode::Auto, ); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -5305,7 +5426,7 @@ mod tests { Mode::Auto, ); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -5342,7 +5463,7 @@ mod tests { Mode::Auto, ); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -5380,7 +5501,7 @@ mod tests { Mode::Auto, ); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -5425,7 +5546,7 @@ mod tests { Mode::Auto, ); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -5475,7 +5596,7 @@ mod tests { Mode::Supervise, ); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -5670,7 +5791,7 @@ mod tests { workflow::save_state(&state).unwrap(); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -5745,7 +5866,7 @@ mod tests { workflow::save_state(&state).unwrap(); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], diff --git a/crates/devflow-cli/src/pipeline_outcomes.rs b/crates/devflow-cli/src/pipeline_outcomes.rs index 2143e46d..c0fe1abd 100644 --- a/crates/devflow-cli/src/pipeline_outcomes.rs +++ b/crates/devflow-cli/src/pipeline_outcomes.rs @@ -1509,7 +1509,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::validate_gaps_does_not_advance_to_ship"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -1540,7 +1540,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::validate_missing_verdict_does_not_advance"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -1573,7 +1573,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::validate_pass_advances"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -1607,7 +1607,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::external_verify_agreement_advances_to_ship"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -1672,7 +1672,7 @@ mod tests { "pipeline_outcomes::tests::external_verify_disagreement_gates_immediately"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -1725,7 +1725,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::external_verify_no_verdict_gates_immediately"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -2024,7 +2024,7 @@ mod tests { "pipeline_outcomes::tests::grafted_failure_shape_gates_instead_of_shipping"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -2100,7 +2100,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::resource_killed_on_code_bumps_infra_failures_not_consecutive_failures"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -2147,7 +2147,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::resource_killed_on_validate_bumps_infra_not_consecutive_failures"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -2239,7 +2239,7 @@ mod tests { "pipeline_outcomes::tests::consecutive_failures_reaches_ceiling_across_cycles"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -2306,7 +2306,7 @@ mod tests { "pipeline_outcomes::tests::healthy_multi_wave_progress_does_not_reach_the_ceiling"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -2375,7 +2375,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::repeated_failure_without_new_commits_still_reaches_the_ceiling"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -2566,7 +2566,7 @@ mod tests { // CYCLE 2 — `git` runs again and reports the same real count as // before. Nothing was committed in between, so this is not progress. let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child( + let out = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, root.as_os_str())], @@ -2638,7 +2638,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::phase_validate_failure_ceiling_gates_despite_trivial_commit_progress"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -2722,7 +2722,7 @@ mod tests { "pipeline_outcomes::tests::validate_gate_message_leads_with_the_per_phase_total"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -2812,7 +2812,7 @@ mod tests { .expect("sh must be resolvable on the parent PATH"); std::os::unix::fs::symlink(real_sh, path_dir.path().join("sh")) .expect("add sh to the isolated child PATH"); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -2879,7 +2879,7 @@ mod tests { "pipeline_outcomes::tests::loop_back_reason_is_distinct_when_no_commit_baseline_exists"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -2946,7 +2946,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::phase_validate_failures_reset_on_operator_approval_at_the_ceiling_gate"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -3035,7 +3035,7 @@ mod tests { "pipeline_outcomes::tests::a_passing_validate_at_the_ceiling_explains_why_it_gated"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -3122,7 +3122,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::the_ceiling_reset_records_the_total_it_spent"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -3209,7 +3209,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::phase_validate_failures_increment_saturates"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -3575,7 +3575,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::evaluate_agent_result_with_real_git_and_empty_branch_still_reports_failed"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -3631,7 +3631,7 @@ mod tests { "pipeline_outcomes::tests::mid_arc_loop_back_issues_plain_execute_command"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -3672,7 +3672,7 @@ mod tests { "pipeline_outcomes::tests::genuine_gaps_loop_back_still_issues_gaps_only"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -3737,7 +3737,7 @@ mod tests { "pipeline_outcomes::tests::worktree_mode_genuine_gaps_loop_back_issues_gaps_only"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -3814,7 +3814,7 @@ mod tests { "pipeline_outcomes::tests::worktree_mode_mid_arc_loop_back_issues_plain_execute"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -3870,7 +3870,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::worktree_mode_main_checkout_only_artifact_is_the_or_both_roots_discriminator"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -3934,7 +3934,7 @@ mod tests { "pipeline_outcomes::tests::stale_verification_artifact_dispatches_full_execute"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -4012,7 +4012,7 @@ mod tests { "pipeline_outcomes::tests::verification_written_this_run_dispatches_gaps_only"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -4273,7 +4273,7 @@ mod tests { "pipeline_outcomes::tests::an_uncaptured_baseline_does_not_claim_an_inherited_artifact"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -4376,7 +4376,7 @@ mod tests { "pipeline_outcomes::tests::an_idempotent_rewrite_is_authored_not_inherited"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -4473,7 +4473,7 @@ mod tests { "pipeline_outcomes::tests::ambiguous_gate_loop_back_respects_the_mid_arc_check"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -4530,7 +4530,7 @@ mod tests { "pipeline_outcomes::tests::failure_gate_loop_back_respects_the_mid_arc_check"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -4587,7 +4587,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::ship_loop_back_still_issues_gaps_only_when_verification_absent"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -4640,7 +4640,7 @@ mod tests { "pipeline_outcomes::tests::external_verify_cycles_reach_ceiling_without_unbounded_loop"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -4755,7 +4755,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::consecutive_failures_increment_saturates"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -4872,7 +4872,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::rate_limited_with_unparseable_retry_hint_gates_instead_of_stalling_silently"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -5007,7 +5007,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::ship_agent_failed_fires_gate"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -5229,7 +5229,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::handle_ship_outcome_without_yes_ship_writes_gate_but_no_response"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } @@ -5332,7 +5332,7 @@ mod tests { }; std::os::unix::fs::symlink(resolve("sh"), path_dir.path().join("sh")).unwrap(); let touch = resolve("touch"); - let out = devflow_core::test_support::run_test_in_child( + let out = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(TOUCH_ENV, touch.as_os_str())], @@ -5419,7 +5419,7 @@ mod tests { const NAME: &str = "pipeline_outcomes::tests::stage_failure_retry_cleans_stale_response"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_git_only_path_dir(); - let out = devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &[]); + let out = crate::test_support::run_hermetic_test_in_child(NAME, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&out, NAME); return; } diff --git a/crates/devflow-cli/src/preflight.rs b/crates/devflow-cli/src/preflight.rs index 9078a516..b78cd29b 100644 --- a/crates/devflow-cli/src/preflight.rs +++ b/crates/devflow-cli/src/preflight.rs @@ -1487,7 +1487,7 @@ mod tests { if !devflow_core::test_support::in_child_test($name) { let path_dir = $path_dir; let output = - devflow_core::test_support::run_test_in_child($name, path_dir.path(), &[]); + crate::test_support::run_hermetic_test_in_child($name, path_dir.path(), &[]); devflow_core::test_support::assert_child_ran_exactly_one_passing_test( &output, $name, ); diff --git a/crates/devflow-cli/src/staleness.rs b/crates/devflow-cli/src/staleness.rs index b20a9440..bd6a3c82 100644 --- a/crates/devflow-cli/src/staleness.rs +++ b/crates/devflow-cli/src/staleness.rs @@ -1196,7 +1196,7 @@ mod tests { // identical "self-dogfood stale build blocked" error produced above. workflow::save_state(&state).unwrap(); let path_dir = agent_free_dir_with_agent_stub("claude"); - let output = devflow_core::test_support::run_test_in_child( + let output = crate::test_support::run_hermetic_test_in_child( NAME, path_dir.path(), &[(ROOT_ENV, project_root.as_os_str())], diff --git a/crates/devflow-cli/src/test_support.rs b/crates/devflow-cli/src/test_support.rs index cf228802..b217d298 100644 --- a/crates/devflow-cli/src/test_support.rs +++ b/crates/devflow-cli/src/test_support.rs @@ -372,6 +372,9 @@ pub(crate) fn child_no_git_root() -> Option { pub(crate) fn run_test_without_git(test_name: &str, root: &Path) -> std::process::Output { // Deliberately empty. Nothing is written into it, by design. let empty = tempfile::tempdir().unwrap(); + // 50-10 (D-07): a launch inside this child registers here, never under + // the developer's real HOME. + let cache = hermetic_cache_dir(); let exe = std::env::current_exe().expect("current_exe for child test re-invocation"); let output = std::process::Command::new(&exe) .arg(test_name) @@ -379,10 +382,42 @@ pub(crate) fn run_test_without_git(test_name: &str, root: &Path) -> std::process .arg("--test-threads=1") .env("PATH", empty.path()) .env(CHILD_NO_GIT_ROOT, root) + .env("DEVFLOW_CACHE_DIR", cache.path()) .output() .expect("spawn child test process with an empty PATH"); - // Explicit, not incidental: the empty directory must outlive the child. + // Explicit, not incidental: both directories must outlive the child. drop(empty); + drop(cache); + output +} + +/// A parent-owned DevFlow cache dir for one child test process (50-10, D-07): +/// short and under `/tmp`, so 50-11's socket paths inside it stay under the +/// 107-byte `sun_path` limit. +pub(crate) fn hermetic_cache_dir() -> tempfile::TempDir { + tempfile::Builder::new() + .prefix("dfu") + .tempdir_in("/tmp") + .expect("create a hermetic DevFlow cache dir under /tmp") +} + +/// [`devflow_core::test_support::run_test_in_child`] with a fresh +/// [`hermetic_cache_dir`] passed as the child's `DEVFLOW_CACHE_DIR` (50-10, +/// D-07). Every in-crate test that may launch a stage runs its child through +/// this, so the launch registers (and, from 50-11, binds its socket) inside +/// that dir and never under the developer's real HOME. The cache dir lives +/// until the child has exited. A `DEVFLOW_CACHE_DIR` in `extra_env` wins. +pub(crate) fn run_hermetic_test_in_child( + test_name: &str, + path_dir: &Path, + extra_env: &[(&str, &std::ffi::OsStr)], +) -> std::process::Output { + let cache = hermetic_cache_dir(); + let mut env: Vec<(&str, &std::ffi::OsStr)> = + vec![("DEVFLOW_CACHE_DIR", cache.path().as_os_str())]; + env.extend_from_slice(extra_env); + let output = devflow_core::test_support::run_test_in_child(test_name, path_dir, &env); + drop(cache); output } diff --git a/doc-check-allowlist.toml b/doc-check-allowlist.toml index 525f8623..ff4a08b2 100644 --- a/doc-check-allowlist.toml +++ b/doc-check-allowlist.toml @@ -49,3 +49,13 @@ reason = "Named in the unattended-mode guide as a flag that does NOT exist (D-09 kind = "docs_to_source" token = "DEVFLOW_ALLOW_OFF_WORKTREE" reason = "Read by scripts/lint-phase-worktree.sh (the pre-commit phase-worktree guard), not by Rust source." + +[[exceptions]] +kind = "source_to_docs" +token = "DEVFLOW_MONITOR_ARGV" +reason = "Test-only: carries the __monitor argv to the CLI #[cfg(test)] monitor harness entries (set by MonitorExec::HarnessEntry, test-support feature only); no release build reads it, so it is not an operator setting (Phase 50, plan 50-10)." + +[[exceptions]] +kind = "source_to_docs" +token = "DEVFLOW_TEST_MONITOR_FULL" +reason = "Test-only: selects the advance-running monitor harness entry for #[cfg(test)] stage launches in the CLI own unit tests; no release build reads it, so it is not an operator setting (Phase 50, plan 50-10)." From 1306fc7bae1cd389a765a758e835e24319ddc9e7 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 15:39:57 -0400 Subject: [PATCH 044/112] test(50-24): add failing tracer for the publish wait's teardown - monitor: private PUBLISH_WAIT, ChildStat and stubs for wait_for_publish (polls, then NotReady with no teardown) and parse_proc_stat_line (None) - tracer: a supervisor that never publishes, run with process_group(0), must be ended with the sleeper group it started; control: a bare kill of the child's own group leaves the sleeper alive - parse test: ppid, pgrp and start time after the last ')' on a fixed line with a hostile comm, and on /proc/self/stat Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/monitor.rs | 280 +++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index 08a41ca4..3643d6e7 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -1694,6 +1694,67 @@ fn stage_sidecar(spec: &StageSpec, agent_pid: u32) -> crate::supervisor::Sidecar } } +/// How long a launcher waits for a new supervisor to answer a ping with its +/// launch id before it ends what it started (50-24). +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +const PUBLISH_WAIT: Duration = Duration::from_secs(10); + +/// The `/proc//stat` fields the not-ready teardown picks its targets +/// by, from one read. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ChildStat { + /// Field 4, the parent pid. + ppid: u32, + /// Field 5, the process group id. + pgid: i32, + /// Field 22, the start time in clock ticks since boot. + start_time: u64, +} + +/// Parse one `/proc//stat` line into [`ChildStat`]. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn parse_proc_stat_line(_line: &str) -> Option { + None +} + +/// Wait up to `wait` for the supervisor `child` to publish `socket` with +/// `launch_id`. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn wait_for_publish( + child: &mut std::process::Child, + socket: &SocketPath, + launch_id: &str, + wait: Duration, +) -> Result { + let deadline = std::time::Instant::now() + wait; + while std::time::Instant::now() < deadline { + if let Ok(crate::supervisor::Liveness::Alive(reply)) = + crate::supervisor::probe(socket.as_path()) + && reply.launch_id == launch_id + { + return Ok(reply); + } + std::thread::sleep(Duration::from_millis(20)); + } + Err(MonitorError::NotReady(format!( + "supervisor {} did not publish", + child.id() + ))) +} + /// The idle-timeout firing sequence, in the ONE order it may run (D-05). /// /// 1. Enumerate the commits the agent made. @@ -6079,4 +6140,223 @@ exit 0 "responsibilities not met: {failures:#?}" ); } + + // ---- 50-24: the publish wait and its not-ready teardown ------------- + + /// Kills (by the pid of the `Child` this test spawned) and reaps the + /// child on every exit path. A child the helper already reaped is not + /// signalled again: `Child::kill` on a reaped child sends nothing. + struct OwnedChild(std::process::Child); + + impl Drop for OwnedChild { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + /// A process group led by `pgid`, whose leader started at `start_time`. + /// On drop it SIGKILLs the group, and only while the leader still has + /// that start time, so a recycled pgid is never signalled. + struct LeaderGroup { + pgid: u32, + start_time: u64, + } + + impl LeaderGroup { + /// Record the group `pid` leads, by its start time now. + fn of(pid: u32) -> Self { + let start_time = crate::agent::process_start_time(pid) + .unwrap_or_else(|| panic!("read the start time of {pid}")); + Self { + pgid: pid, + start_time, + } + } + + fn alive(&self) -> bool { + crate::agent::is_same_process(self.pgid, self.start_time) + } + } + + impl Drop for LeaderGroup { + fn drop(&mut self) { + let Ok(signed) = libc::pid_t::try_from(self.pgid) else { + return; + }; + // SAFETY: getpgrp has no preconditions. + if signed > 1 && signed != unsafe { libc::getpgrp() } && self.alive() { + // SAFETY: `signed > 1`, not our own group, and its leader is + // still the recorded process. + unsafe { libc::kill(-signed, libc::SIGKILL) }; + } + } + } + + /// Whether no process is left in group `pgid` (`kill(-pgid, 0)` fails + /// with ESRCH), polled every 20 ms up to `within`. + fn group_gone_within(pgid: u32, within: Duration) -> bool { + let signed = libc::pid_t::try_from(pgid).expect("pgid fits pid_t"); + assert!(signed > 1, "refusing to probe group {signed}"); + stage_wait_until(within, || { + // SAFETY: signal 0 only checks for existence; `signed > 1`. + let rc = unsafe { libc::kill(-signed, 0) }; + rc != 0 && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + }) + } + + /// The env var naming the file the never-publishing supervisor writes + /// its sleeper's pid to (read by `sh`, not by Rust). + const SLEEPER_PID_ENV: &str = "SLEEPER_PID_FILE"; + + /// A fake supervisor that never binds: it ignores SIGTERM, starts a + /// sleeper in its own group with util-linux `setsid` as a background job + /// (a non-interactive `sh` job is not a group leader, so `setsid` does + /// not fork and `$!` is the sleeper), records the sleeper's pid, then + /// becomes `sleep 300`. + fn never_publishing_supervisor(fixture: &StageFixture) -> PathBuf { + fixture.agent( + "never-publishes", + &format!( + "trap '' TERM\n\ + setsid sleep 300 &\n\ + echo $! > \"${SLEEPER_PID_ENV}.tmp\"\n\ + mv \"${SLEEPER_PID_ENV}.tmp\" \"${SLEEPER_PID_ENV}\"\n\ + exec sleep 300" + ), + ) + } + + /// Spawn `script` as `spawn_monitor` will spawn a supervisor + /// (`process_group(0)`), wait up to 2 s for its sleeper's pid, and return + /// the child and the sleeper's group, both guarded. + fn spawn_never_publishing(script: &Path, pid_file: &Path) -> (OwnedChild, LeaderGroup) { + let child = OwnedChild( + std::process::Command::new(script) + .env(SLEEPER_PID_ENV, pid_file) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .process_group(0) + .spawn() + .expect("spawn the never-publishing supervisor"), + ); + assert!( + stage_wait_until(Duration::from_secs(2), || pid_file.exists()), + "the supervisor recorded no sleeper pid within 2 s" + ); + let sleeper: u32 = std::fs::read_to_string(pid_file) + .expect("read the sleeper pid") + .trim() + .parse() + .expect("the sleeper pid parses"); + let group = LeaderGroup::of(sleeper); + assert_eq!( + crate::agent::process_stat_fields(sleeper).map(|stat| stat.pgrp), + libc::pid_t::try_from(sleeper).ok(), + "the sleeper leads its own group" + ); + (child, group) + } + + #[test] + fn a_supervisor_that_never_publishes_is_ended_with_its_agent_group() { + let fixture = StageFixture::new(); + let script = never_publishing_supervisor(&fixture); + + // Control: a SIGKILL of the child's own group alone orphans the + // sleeper, which leads a group of its own. + { + let (mut child, sleeper) = + spawn_never_publishing(&script, &fixture.path("control-pid")); + let pid = libc::pid_t::try_from(child.0.id()).expect("pid fits pid_t"); + assert!(pid > 1, "refusing to signal group {pid}"); + // SAFETY: `pid > 1` leads the group this test just spawned and + // still owns unreaped. + assert_eq!(unsafe { libc::kill(-pid, libc::SIGKILL) }, 0); + child.0.wait().expect("reap the control child"); + std::thread::sleep(Duration::from_millis(500)); + assert!( + sleeper.alive(), + "control: a bare kill of the child's group must leave its sleeper alive" + ); + let pgid = sleeper.pgid; + drop(sleeper); + assert!(group_gone_within(pgid, Duration::from_secs(2))); + } + + let (mut child, sleeper) = spawn_never_publishing(&script, &fixture.path("sleeper-pid")); + let child_pid = child.0.id(); + let socket = SocketPath::for_launch( + fixture.cache.path(), + &fixture.root, + STAGE_PHASE, + "never-publishes", + ) + .expect("socket path fits sun_path"); + assert!( + Duration::from_secs(1) < PUBLISH_WAIT, + "this test runs on a wait shorter than the production one" + ); + let started = std::time::Instant::now(); + let result = wait_for_publish( + &mut child.0, + &socket, + "never-publishes", + Duration::from_secs(1), + ); + let elapsed = started.elapsed(); + match &result { + Err(MonitorError::NotReady(text)) => assert!( + text.contains(&child_pid.to_string()), + "the NotReady text names the child {child_pid}: {text}" + ), + other => panic!("expected NotReady, got {other:?}"), + } + let bound = Duration::from_secs(1) + + 2 * crate::agent::TERMINATE_VERIFY_WAIT + + Duration::from_secs(3); + assert!(elapsed <= bound, "took {elapsed:?}, bound {bound:?}"); + assert!( + group_gone_within(sleeper.pgid, Duration::from_secs(2)), + "the teardown must end the sleeper's group {}", + sleeper.pgid + ); + assert!( + child.0.try_wait().expect("poll the child").is_some(), + "the helper reaped the child" + ); + } + + #[test] + fn proc_stat_parse_reads_ppid_pgid_and_start_time_after_the_last_paren() { + // comm `(a) b (c)` holds spaces and parentheses; every field the + // parse could confuse carries a distinct value (session 555, tty 0, + // itrealvalue 0, vsize 1000). + let head = "1234 ((a) b (c)) S 4242 777 555 0 -1 4194560 100 0 0 0 1 2 0 0 20 0 1 0"; + let line = format!("{head} 98765 1000 200 18446744073709551615"); + assert_eq!( + parse_proc_stat_line(&line), + Some(ChildStat { + ppid: 4242, + pgid: 777, + start_time: 98765, + }) + ); + assert_eq!( + parse_proc_stat_line(head), + None, + "a line cut off before field 22" + ); + + let own = std::fs::read_to_string("/proc/self/stat").expect("read /proc/self/stat"); + let parsed = parse_proc_stat_line(&own).expect("parse /proc/self/stat"); + assert_eq!(parsed.ppid, std::os::unix::process::parent_id()); + assert_eq!( + Some(parsed.start_time), + crate::agent::process_start_time(std::process::id()) + ); + // SAFETY: getpgrp has no preconditions. + assert_eq!(parsed.pgid, unsafe { libc::getpgrp() }); + } } From 5712478c93c3586676200ac9a3eec682c56bbf96 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 15:42:12 -0400 Subject: [PATCH 045/112] feat(50-24): end an unpublished supervisor and the agent group it started - wait_for_publish polls every 20 ms: a ping with this launch id returns the reply and leaves the child running; an early exit is reaped; a timeout SIGTERMs the child, waits 2 x TERMINATE_VERIFY_WAIT, then SIGSTOPs it, lists its direct children from /proc, SIGKILLs its group, reaps it and SIGKILLs each listed child's own group whose leader keeps its listed start time - once the child is reaped the launch files are removed and the result is NotReady naming the child, how it ended and the groups ended - private parse_proc_stat_line / read_child_stat (ppid, pgrp, start time after the last ')'), one guarded group-signal fn (pid_t, > 1, not our own group); each item carries the non-test dead-code expectation until 50-11's spawn_monitor calls wait_for_publish Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/monitor.rs | 230 +++++++++++++++++++++++++++-- 1 file changed, 216 insertions(+), 14 deletions(-) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index 3643d6e7..10a3e75d 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -58,7 +58,8 @@ pub enum MonitorError { /// The process group the agent was found in (`-1` when unreadable). pgid: i32, }, - /// A supervision shape that this build cannot run yet. + /// A supervision shape that this build cannot run yet, or a supervisor + /// that never published its socket (it and what it started were ended). #[error("supervision shape not ready: {0}")] NotReady(String), } @@ -1702,8 +1703,24 @@ fn stage_sidecar(spec: &StageSpec, agent_pid: u32) -> crate::supervisor::Sidecar )] const PUBLISH_WAIT: Duration = Duration::from_secs(10); +/// How often [`wait_for_publish`] polls the child and probes the socket. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +const PUBLISH_POLL: Duration = Duration::from_millis(20); + +/// How long the not-ready teardown waits for its SIGSTOP to show in +/// `/proc` before it lists the child's children. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +const STOP_SETTLE_WAIT: Duration = Duration::from_millis(100); + /// The `/proc//stat` fields the not-ready teardown picks its targets -/// by, from one read. +/// by, from one read. 50-09's `agent::ProcStat` carries no parent pid, so +/// this is a separate, private parse. #[cfg_attr( not(test), expect(dead_code, reason = "spawn_monitor calls it from 50-11") @@ -1714,21 +1731,188 @@ struct ChildStat { ppid: u32, /// Field 5, the process group id. pgid: i32, - /// Field 22, the start time in clock ticks since boot. + /// Field 22, the start time in clock ticks since boot, as + /// [`crate::agent::is_same_process`] compares it. start_time: u64, } -/// Parse one `/proc//stat` line into [`ChildStat`]. +/// Parse one `/proc//stat` line into [`ChildStat`]. `comm` (field 2) +/// may hold spaces and parentheses, so the line is split after its last +/// `)`: index 0 is then field 3, so the parent pid (field 4) is index 1, +/// the process group (field 5) index 2 and the start time (field 22) +/// index 19. A missing or unparsable field is `None`, never a guess. #[cfg_attr( not(test), expect(dead_code, reason = "spawn_monitor calls it from 50-11") )] -fn parse_proc_stat_line(_line: &str) -> Option { - None +fn parse_proc_stat_line(line: &str) -> Option { + let rest = &line[line.rfind(')')? + 1..]; + let fields: Vec<&str> = rest.split_whitespace().collect(); + Some(ChildStat { + ppid: fields.get(1)?.parse().ok()?, + pgid: fields.get(2)?.parse().ok()?, + start_time: fields.get(19)?.parse().ok()?, + }) +} + +/// [`ChildStat`] for `pid` from one read of `/proc//stat`, or `None` +/// when it cannot be read or parsed. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn read_child_stat(pid: u32) -> Option { + parse_proc_stat_line(&std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?) +} + +/// Every process whose parent pid is `parent`, each with the stat it was +/// chosen by. An entry that cannot be read is skipped. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn direct_children(parent: u32) -> Vec<(u32, ChildStat)> { + let Ok(entries) = std::fs::read_dir("/proc") else { + warn!("could not list /proc; no child of supervisor {parent} can be found"); + return Vec::new(); + }; + entries + .filter_map(Result::ok) + .filter_map(|entry| entry.file_name().to_str()?.parse::().ok()) + .filter_map(|pid| Some((pid, read_child_stat(pid)?))) + .filter(|(_, stat)| stat.ppid == parent) + .collect() +} + +/// Signal process group `pgid`: the one place this module's launch paths +/// send a group signal. It refuses a pgid that does not fit `pid_t`, is `0` +/// or `1`, or is this process's own group, because `kill(0, …)` signals our +/// own group and `kill(-1, …)` every process we may signal (the runtime's +/// rule, T-50-18). Callers pass only a group they started or one whose +/// leader they have just re-validated. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn signal_guarded_group(pgid: u32, signal: libc::c_int) -> bool { + let Ok(signed) = libc::pid_t::try_from(pgid) else { + warn!("process group {pgid} does not fit pid_t; not signalling"); + return false; + }; + if signed <= 1 { + warn!("refusing to signal process group {signed}"); + return false; + } + // SAFETY: getpgrp has no preconditions and cannot fail. + if signed == unsafe { libc::getpgrp() } { + warn!("refusing to signal this process's own group {signed}"); + return false; + } + // SAFETY: `signed > 1`, so `-signed < -1`: neither our own group (`0`) + // nor every process (`-1`). + unsafe { libc::kill(-signed, signal) == 0 } +} + +/// Send `signal` to `child` alone, by the pid of the `Child` this process +/// spawned and has not reaped (so the pid cannot name another process). +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn signal_own_child(child: &std::process::Child, signal: libc::c_int) -> bool { + match libc::pid_t::try_from(child.id()) { + // SAFETY: `pid > 1` names one unreaped child of this process. + Ok(pid) if pid > 1 => unsafe { libc::kill(pid, signal) == 0 }, + _ => false, + } } -/// Wait up to `wait` for the supervisor `child` to publish `socket` with -/// `launch_id`. +/// Poll `child` until it exits (reaping it) or `wait` elapses. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn exit_within( + child: &mut std::process::Child, + wait: Duration, +) -> Option { + let deadline = std::time::Instant::now() + wait; + loop { + if let Ok(Some(status)) = child.try_wait() { + return Some(status); + } + if std::time::Instant::now() >= deadline { + return None; + } + std::thread::sleep(PUBLISH_POLL); + } +} + +/// End a supervisor that did not publish in time, and what it started, +/// using only its own `Child` and values read from `/proc` while it is +/// stopped, never a pid from disk. Returns how it ended. +/// +/// 1. SIGTERM: a supervisor that reached `serve` treats it as a stop (D-05) +/// and ends its agent group itself. Wait up to +/// `2 × agent::TERMINATE_VERIFY_WAIT`. +/// 2. Still running: SIGSTOP it, so it cannot spawn or reap meanwhile, and +/// list its direct children with their group and start time. +/// 3. SIGKILL its own group (it leads it, `process_group(0)`) and reap it. +/// 4. SIGKILL the group of each listed child that leads its own group +/// (R-3's agent shape) and still has the listed start time. A bare kill +/// of the child's group would orphan such an agent. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn end_unpublished_child(child: &mut std::process::Child, wait: Duration) -> String { + signal_own_child(child, libc::SIGTERM); + if let Some(status) = exit_within(child, 2 * crate::agent::TERMINATE_VERIFY_WAIT) { + return format!( + "did not publish within {wait:?} and was ended by SIGTERM ({status}); \ + 0 child process groups ended" + ); + } + let pid = child.id(); + signal_own_child(child, libc::SIGSTOP); + let deadline = std::time::Instant::now() + STOP_SETTLE_WAIT; + while !crate::agent::process_stat_fields(pid).is_some_and(|stat| stat.state == 'T') + && std::time::Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(5)); + } + let children = direct_children(pid); + signal_guarded_group(pid, libc::SIGKILL); + // By pid as well, in case it leads no group: a pid still in use as a + // live non-leader names no group, so the group signal reached nothing. + let _ = child.kill(); + let _ = child.wait(); + let ended = children + .iter() + .filter(|(child_pid, stat)| { + i64::from(stat.pgid) == i64::from(*child_pid) + && crate::agent::is_same_process(*child_pid, stat.start_time) + && signal_guarded_group(*child_pid, libc::SIGKILL) + }) + .count(); + format!( + "did not publish within {wait:?} and was ended (SIGTERM ignored for {grace:?}, then \ + SIGSTOP and SIGKILL of its group); {ended} child process group(s) ended", + grace = 2 * crate::agent::TERMINATE_VERIFY_WAIT + ) +} + +/// Wait up to `wait` for the supervisor `child` (spawned by this process +/// with `process_group(0)`) to publish `socket` with `launch_id`. +/// +/// Polls every [`PUBLISH_POLL`]: a ping reply with this launch id returns +/// it and leaves the child running; Gone, Stale, a probe error or another +/// launch id count as not yet published. If the child exits first it is +/// reaped; if the wait expires it is ended with what it started +/// ([`end_unpublished_child`]). Either way nothing can answer on this +/// launch's address any more (its only server was the reaped child), so its +/// launch files are removed, and the result is [`MonitorError::NotReady`] +/// naming the child, how it ended and how many child groups were ended. #[cfg_attr( not(test), expect(dead_code, reason = "spawn_monitor calls it from 50-11") @@ -1739,20 +1923,32 @@ fn wait_for_publish( launch_id: &str, wait: Duration, ) -> Result { + let pid = child.id(); let deadline = std::time::Instant::now() + wait; - while std::time::Instant::now() < deadline { + let how = loop { + if let Ok(Some(status)) = child.try_wait() { + break format!( + "exited by itself before publishing ({status}); 0 child process groups ended" + ); + } if let Ok(crate::supervisor::Liveness::Alive(reply)) = crate::supervisor::probe(socket.as_path()) && reply.launch_id == launch_id { return Ok(reply); } - std::thread::sleep(Duration::from_millis(20)); + if std::time::Instant::now() >= deadline { + break end_unpublished_child(child, wait); + } + std::thread::sleep(PUBLISH_POLL); + }; + if let Err(error) = crate::supervisor::remove_launch_files(socket) { + warn!( + "could not remove the launch files of {}: {error}", + socket.as_path().display() + ); } - Err(MonitorError::NotReady(format!( - "supervisor {} did not publish", - child.id() - ))) + Err(MonitorError::NotReady(format!("supervisor {pid} {how}"))) } /// The idle-timeout firing sequence, in the ONE order it may run (D-05). @@ -6313,6 +6509,12 @@ exit 0 ), other => panic!("expected NotReady, got {other:?}"), } + if let Err(MonitorError::NotReady(text)) = &result { + assert!( + text.contains("1 child process group(s) ended"), + "the teardown ended exactly the sleeper's group: {text}" + ); + } let bound = Duration::from_secs(1) + 2 * crate::agent::TERMINATE_VERIFY_WAIT + Duration::from_secs(3); From 3bafbd48cffc25688c78b5622a433178dba2adad Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 15:44:53 -0400 Subject: [PATCH 046/112] test(50-24): add failing tests for the pre-publish agent teardown - an #[ignore]d child entry runs run_supervised_stage (capture shape, sh -c 'exec sleep 300') on a spec from its env; the parent spawns its own test binary for it under process_group(0) - fails-to-publish: a symlinked sock/ makes the child exit before publishing and leaves no agent; control: a plain cache dir publishes, is returned and stops on request_shutdown - RV-03: a child parked after its sidecar write and SIGKILLed has its agent group ended through the sidecar; controls: no sidecar leaves the agent alive, a start time one tick off is LeaderRecycled - test-hold scan with planted controls; the verdict is a stub Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/monitor.rs | 331 +++++++++++++++++++++++++++++ 1 file changed, 331 insertions(+) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index 10a3e75d..fa03260e 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -6561,4 +6561,335 @@ exit 0 // SAFETY: getpgrp has no preconditions. assert_eq!(parsed.pgid, unsafe { libc::getpgrp() }); } + + // ---- 50-24: run_supervised_stage before it publishes ---------------- + + /// The cache dir the publish-wait child entry builds its socket under. + const PUBLISH_CACHE_ENV: &str = "DEVFLOW_TEST_PUBLISH_CACHE"; + /// The project root the publish-wait child entry supervises. + const PUBLISH_ROOT_ENV: &str = "DEVFLOW_TEST_PUBLISH_ROOT"; + /// The launch id the publish-wait child entry publishes. + const PUBLISH_LAUNCH_ENV: &str = "DEVFLOW_TEST_PUBLISH_LAUNCH"; + /// `monitor_test_hold`'s variable, set (never read) by these tests. + const TEST_HOLD_ENV: &str = "DEVFLOW_TEST_HOLD"; + /// The child entry the publish-wait tests re-run. + const PUBLISH_ENTRY: &str = "monitor::tests::supervise_a_stage_for_the_publish_wait"; + + /// The child half of the publish-wait tests: supervise `sh -c 'exec + /// sleep 300'` in the capture shape on the spec the parent's env names. + /// Returns at once without that env, so an `--include-ignored` run is + /// harmless; panics with the error when `run_supervised_stage` fails. + #[test] + #[ignore = "child entry of the 50-24 publish-wait tests; returns at once without its env"] + fn supervise_a_stage_for_the_publish_wait() { + let (Some(cache), Some(root), Some(launch_id)) = ( + std::env::var_os(PUBLISH_CACHE_ENV), + std::env::var_os(PUBLISH_ROOT_ENV), + std::env::var_os(PUBLISH_LAUNCH_ENV), + ) else { + return; + }; + let launch_id = launch_id.into_string().expect("utf-8 launch id"); + let root = PathBuf::from(root); + let socket = SocketPath::for_launch(Path::new(&cache), &root, STAGE_PHASE, &launch_id) + .expect("socket path fits sun_path"); + let mut spec = StageSpec::new( + root, + STAGE_PHASE, + AgentKind::Codex, + "/bin/sh".to_string(), + StageIo::Capture, + socket, + launch_id, + ); + spec.args = vec!["-c".to_string(), "exec sleep 300".to_string()]; + spec.stage = Some(Stage::Code); + let run = run_supervised_stage(spec) + .unwrap_or_else(|err| panic!("run_supervised_stage failed: {err}")); + run.supervisor.finish(); + } + + /// Spawn this test binary for [`PUBLISH_ENTRY`] under `process_group(0)` + /// (the argv shape `run_test_in_child` uses), with its stdout and stderr + /// in `log`. `hold` sets `DEVFLOW_TEST_HOLD=after_sidecar:`. + fn spawn_publish_entry( + cache: &Path, + fixture: &StageFixture, + launch_id: &str, + hold: Option<&Path>, + log: &Path, + ) -> OwnedChild { + let out = std::fs::File::create(log).expect("create the entry log"); + let err = out.try_clone().expect("share the entry log"); + let mut cmd = std::process::Command::new( + std::env::current_exe().expect("current_exe for the child entry"), + ); + cmd.arg(PUBLISH_ENTRY) + .arg("--exact") + .arg("--include-ignored") + .arg("--test-threads=1") + .env(PUBLISH_CACHE_ENV, cache) + .env(PUBLISH_ROOT_ENV, &fixture.root) + .env(PUBLISH_LAUNCH_ENV, launch_id) + .stdin(Stdio::null()) + .stdout(Stdio::from(out)) + .stderr(Stdio::from(err)) + .process_group(0); + match hold { + Some(path) => cmd.env(TEST_HOLD_ENV, format!("after_sidecar:{}", path.display())), + None => cmd.env_remove(TEST_HOLD_ENV), + }; + OwnedChild(cmd.spawn().expect("spawn the publish-wait child entry")) + } + + /// The agent pid `run_supervised_stage` wrote before it published. + fn publish_agent_pid(fixture: &StageFixture) -> u32 { + fixture + .pid_file() + .parse() + .expect("the agent pid file holds a pid") + } + + #[test] + fn a_supervisor_that_fails_to_publish_ends_its_agent_group() { + // Failure direction: `sock/` is a symlink, which publishing refuses. + let fixture = StageFixture::new(); + let elsewhere = fixture.path("elsewhere"); + std::fs::create_dir(&elsewhere).expect("create the symlink target"); + let bad_cache = fixture.path("bad-cache"); + std::fs::create_dir(&bad_cache).expect("create the bad cache dir"); + std::os::unix::fs::symlink(&elsewhere, bad_cache.join("sock")).expect("plant sock/"); + let launch_id = crate::supervisor::new_launch_id(); + let socket = SocketPath::for_launch(&bad_cache, &fixture.root, STAGE_PHASE, &launch_id) + .expect("socket path fits sun_path"); + let log = fixture.path("fail.log"); + let mut child = spawn_publish_entry(&bad_cache, &fixture, &launch_id, None, &log); + let result = wait_for_publish(&mut child.0, &socket, &launch_id, PUBLISH_WAIT); + let entry_log = std::fs::read_to_string(&log).unwrap_or_default(); + match &result { + Err(MonitorError::NotReady(text)) => assert!( + text.contains("exited by itself before publishing"), + "the child exited before publishing: {text}\n{entry_log}" + ), + other => panic!("expected NotReady, got {other:?}\n{entry_log}"), + } + let agent = publish_agent_pid(&fixture); + let survivor = crate::agent::process_start_time(agent).map(|start_time| LeaderGroup { + pgid: agent, + start_time, + }); + assert!( + stage_wait_until(Duration::from_secs(2), || { + let signed = libc::pid_t::try_from(agent).expect("pid fits pid_t"); + // SAFETY: signal 0 only checks for existence. + let rc = unsafe { libc::kill(signed, 0) }; + rc != 0 && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + }), + "no agent survives a publish error: {agent} is alive\n{entry_log}" + ); + drop(survivor); + + // Control: a plain cache dir publishes, and the helper returns the + // reply and leaves the supervisor and its agent running. + let fixture = StageFixture::new(); + let launch_id = crate::supervisor::new_launch_id(); + let socket = + SocketPath::for_launch(fixture.cache.path(), &fixture.root, STAGE_PHASE, &launch_id) + .expect("socket path fits sun_path"); + let log = fixture.path("ok.log"); + let mut child = spawn_publish_entry(fixture.cache.path(), &fixture, &launch_id, None, &log); + let reply = wait_for_publish(&mut child.0, &socket, &launch_id, PUBLISH_WAIT) + .unwrap_or_else(|err| { + panic!( + "a published supervisor is returned: {err}\n{}", + std::fs::read_to_string(&log).unwrap_or_default() + ) + }); + let agent = LeaderGroup::of(reply.agent_pid); + assert_eq!(reply.launch_id, launch_id); + assert_eq!(publish_agent_pid(&fixture), reply.agent_pid); + assert!( + crate::agent::is_same_process(reply.agent_pid, reply.agent_start_time), + "the published agent is alive and is the recorded process" + ); + assert!( + child.0.try_wait().expect("poll the child").is_none(), + "the published supervisor is left running" + ); + let stopped = + crate::supervisor::request_shutdown(socket.as_path(), Duration::from_secs(10)) + .expect("request the stop"); + assert!( + matches!(stopped, crate::supervisor::ShutdownOutcome::Stopped { .. }), + "the supervisor stops: {stopped:?}" + ); + assert!( + exit_within(&mut child.0, Duration::from_secs(5)).is_some(), + "the child exits within 5 s of the stop" + ); + let entry_log = std::fs::read_to_string(&log).unwrap_or_default(); + assert!( + entry_log.contains(&format!("test {PUBLISH_ENTRY} ... ok")) + && entry_log.contains("1 passed"), + "the child ran its one entry: {entry_log}" + ); + assert!(group_gone_within(agent.pgid, Duration::from_secs(2))); + } + + /// What one killed-before-publishing run left: the wait's text, the + /// agent's group (guarded), and the launch's socket. + struct KilledRun { + text: String, + agent: LeaderGroup, + socket: SocketPath, + } + + /// Launch the child entry parked after its sidecar write, read the + /// sidecar, let `tamper` change it, SIGKILL the child by its own `Child` + /// and run the publish wait. + fn kill_before_publishing( + fixture: &StageFixture, + tamper: impl FnOnce(&SocketPath, &crate::supervisor::Sidecar), + ) -> KilledRun { + let launch_id = crate::supervisor::new_launch_id(); + let socket = + SocketPath::for_launch(fixture.cache.path(), &fixture.root, STAGE_PHASE, &launch_id) + .expect("socket path fits sun_path"); + let hold = fixture.path("never-created"); + let log = fixture.path("killed.log"); + let mut child = + spawn_publish_entry(fixture.cache.path(), fixture, &launch_id, Some(&hold), &log); + let mut sidecar = None; + let parked = stage_wait_until(Duration::from_secs(5), || { + sidecar = crate::supervisor::read_sidecar(&socket.sidecar_path()).ok(); + sidecar.is_some() || child.0.try_wait().ok().flatten().is_some() + }); + let entry_log = || std::fs::read_to_string(&log).unwrap_or_default(); + let sidecar = sidecar + .filter(|_| parked) + .unwrap_or_else(|| panic!("no sidecar within 5 s\n{}", entry_log())); + let agent = LeaderGroup { + pgid: sidecar.agent_pgid, + start_time: sidecar.agent_start_time, + }; + assert_eq!(sidecar.launch_id, launch_id); + assert_eq!( + sidecar.agent_pgid, sidecar.agent_pid, + "the agent leads its group" + ); + assert!(agent.alive(), "the agent runs while the child is parked"); + tamper(&socket, &sidecar); + child.0.kill().expect("SIGKILL the child by its own Child"); + let result = wait_for_publish(&mut child.0, &socket, &launch_id, PUBLISH_WAIT); + let text = match result { + Err(MonitorError::NotReady(text)) => text, + other => panic!("expected NotReady, got {other:?}\n{}", entry_log()), + }; + KilledRun { + text, + agent, + socket, + } + } + + #[test] + fn a_supervisor_killed_before_publishing_has_its_agent_group_ended_through_its_sidecar() { + // Run 1: the sidecar is untouched, so the launcher ends its group. + let fixture = StageFixture::new(); + let run = kill_before_publishing(&fixture, |_, _| {}); + let pgid = run.agent.pgid; + assert!( + run.text + .contains(&format!("ended the agent group {pgid} its sidecar records")), + "{}", + run.text + ); + assert!( + group_gone_within(pgid, Duration::from_secs(2)), + "the recorded agent group {pgid} is ended" + ); + assert!(!run.socket.as_path().exists(), "no socket is left"); + assert!(!run.socket.sidecar_path().exists(), "no sidecar is left"); + + // Run 2, control: without the sidecar nothing names the agent, so it + // survives the child's SIGKILL. + let fixture = StageFixture::new(); + let run = kill_before_publishing(&fixture, |socket, _| { + std::fs::remove_file(socket.sidecar_path()).expect("remove the sidecar"); + }); + assert!(run.text.contains("no sidecar"), "{}", run.text); + std::thread::sleep(Duration::from_millis(500)); + assert!( + run.agent.alive(), + "control: the agent survives without a sidecar" + ); + let pgid = run.agent.pgid; + drop(run.agent); + assert!(group_gone_within(pgid, Duration::from_secs(2))); + + // Run 3, identity control: a recorded start time one tick off is + // LeaderRecycled, and nothing is signalled. + let fixture = StageFixture::new(); + let run = kill_before_publishing(&fixture, |socket, sidecar| { + let wrong = crate::supervisor::Sidecar { + agent_start_time: sidecar.agent_start_time + 1, + ..sidecar.clone() + }; + crate::supervisor::write_sidecar(socket, &wrong).expect("rewrite the sidecar"); + }); + assert!( + run.text.contains("LeaderRecycled") && run.text.contains("nothing was signalled"), + "{}", + run.text + ); + std::thread::sleep(Duration::from_millis(500)); + assert!( + run.agent.alive(), + "identity control: the agent is not signalled" + ); + let pgid = run.agent.pgid; + drop(run.agent); + assert!(group_gone_within(pgid, Duration::from_secs(2))); + } + + /// Whether non-test `monitor.rs` defines `fn monitor_test_hold(` exactly + /// once, directly under attribute lines that include `#[cfg(test)]`. + fn test_hold_guard_verdict(_non_test: &str) -> Result<(), String> { + Ok(()) + } + + #[test] + fn the_test_hold_is_compiled_out_of_non_test_builds() { + let guarded = "#[cfg(test)]\nfn monitor_test_hold(point: &str) {}\n"; + assert_eq!(test_hold_guard_verdict(guarded), Ok(())); + let guarded_and_more = "/// Doc.\n#[cfg(test)]\n#[allow(clippy::needless_pass_by_value)]\nfn monitor_test_hold(point: &str) {}\n"; + assert_eq!(test_hold_guard_verdict(guarded_and_more), Ok(())); + let unguarded = "fn monitor_test_hold(point: &str) {}\n"; + assert!(test_hold_guard_verdict(unguarded).is_err(), "no attribute"); + let wrong_attribute = "#[allow(dead_code)]\nfn monitor_test_hold(point: &str) {}\n"; + assert!( + test_hold_guard_verdict(wrong_attribute).is_err(), + "only #[allow(dead_code)]" + ); + let commented = "// #[cfg(test)]\n// fn monitor_test_hold(point: &str) {}\n"; + assert!( + test_hold_guard_verdict(commented).is_err(), + "a commented-out definition is zero definitions" + ); + let twice = format!("{guarded}{guarded}"); + assert!(test_hold_guard_verdict(&twice).is_err(), "two definitions"); + + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/src/monitor.rs"); + let source = + std::fs::read_to_string(path).unwrap_or_else(|err| panic!("read {path}: {err}")); + let lines: Vec<&str> = source.lines().collect(); + let tests_at = lines + .iter() + .position(|line| *line == "mod tests {") + .unwrap_or_else(|| panic!("{path} has no `mod tests {{` line")); + let non_test = lines[..tests_at].join("\n"); + if let Err(why) = test_hold_guard_verdict(&non_test) { + panic!("{path}: {why}"); + } + } } From 4e689097ba47c30d8f78b73dce9790b362cfa73a Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 15:51:06 -0400 Subject: [PATCH 047/112] feat(50-24): end a pre-publish agent group in the stage and launcher - run_supervised_stage hands the agent's Child to AgentGroupGuard once the pgid check passes; its Drop SIGKILLs the group then reaps the leader, so an error or panic before serve leaves no agent - the sidecar is written right after the pid file, before serve, with a cfg(test) monitor_test_hold("after_sidecar") between them - wait_for_publish's ending step reads the launch's sidecar and SIGKILLs its group only on backstop::assess = LeaderPresentSignal plus a fresh is_same_process; no sidecar, another launch id or any other decision signals nothing and says which; Refuse* keeps the sidecar (RV-03) - the test-hold scan's verdict; DEVFLOW_TEST_HOLD source_to_docs entry Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/monitor.rs | 368 +++++++++++++++++++++++++++-- doc-check-allowlist.toml | 5 + 2 files changed, 347 insertions(+), 26 deletions(-) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index fa03260e..b78d9fbf 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -1496,8 +1496,12 @@ const PGID_SETTLE_WAIT: Duration = Duration::from_millis(100); /// `process_group(0)`, then require that it leads its own process group /// before anything is recorded (R-3). A mismatch kills and reaps it and /// returns [`MonitorError::PgidMismatch`]. -/// 3. Write the agent pid file, then [`Supervisor::serve`]: the sidecar is -/// written before the socket name appears (R-7). +/// 3. Hand the agent's `Child` to an `AgentGroupGuard`, which ends the +/// agent's group on any error or panic until `serve` succeeds. Write the +/// agent pid file and the sidecar, then [`Supervisor::serve`]: the +/// sidecar exists before the socket name appears (R-7), and it is the +/// launcher's only record of the agent if this process is SIGKILLed +/// before it publishes (RV-03; `serve` rewrites it with the same content). /// 4. The stream shape only: drive the agent's pipes through the same /// helper `run_pipe_owning_monitor` uses (first turn, capture tee, close /// rule, idle timeout). An idle-timeout group kill is a natural outcome, @@ -1524,17 +1528,16 @@ pub fn run_supervised_stage(spec: StageSpec) -> Result { abandon_agent(&mut child); return Err(MonitorError::PgidMismatch { pid, pgid }); } - if let Err(err) = std::fs::write(&pid_file, format!("{pid}\n")) { - abandon_agent(&mut child); - return Err(err.into()); - } - let supervisor = match Supervisor::serve(spec.socket.clone(), stage_sidecar(&spec, pid)) { - Ok(supervisor) => supervisor, - Err(err) => { - abandon_agent(&mut child); - return Err(err.into()); - } - }; + // Armed only now: before the pgid check the agent's pid may not number + // its group. Until `serve` succeeds, any error or panic ends the group. + let guard = AgentGroupGuard::arm(child); + std::fs::write(&pid_file, format!("{pid}\n"))?; + let sidecar = stage_sidecar(&spec, pid); + crate::supervisor::write_sidecar(&spec.socket, &sidecar)?; + #[cfg(test)] + monitor_test_hold("after_sidecar"); + let supervisor = Supervisor::serve(spec.socket.clone(), sidecar)?; + let mut child = guard.disarm(); let threads = match (&spec.io, stream_capture) { ( @@ -1584,6 +1587,80 @@ pub fn run_supervised_stage(spec: StageSpec) -> Result { }) } +/// Owns a just-spawned agent's unreaped `Child` from the moment its pgid +/// check passes until [`Supervisor::serve`] succeeds, so a pid-file write +/// error, a sidecar or publish error, or a panic in that span never leaves +/// the agent running when `__monitor` exits. Dropped while armed, it +/// SIGKILLs the agent's group and only then reaps the leader: because it +/// holds the unreaped leader, the pgid cannot be recycled before its own +/// signal. [`disarm`](Self::disarm) hands the `Child` back once the +/// [`Supervisor`] owns teardown. +struct AgentGroupGuard { + /// The agent, unreaped; `None` only once disarmed. + child: Option, + /// The agent's pgid: its pid, checked by `confirm_own_group`. + pgid: u32, +} + +impl AgentGroupGuard { + /// Take ownership of `child`, which leads its own group. + fn arm(child: std::process::Child) -> Self { + let pgid = child.id(); + Self { + child: Some(child), + pgid, + } + } + + /// Hand the agent back: the [`Supervisor`] now owns its teardown. + fn disarm(mut self) -> std::process::Child { + self.child + .take() + .expect("an armed AgentGroupGuard holds its child until disarm") + } +} + +impl Drop for AgentGroupGuard { + fn drop(&mut self) { + if let Some(mut child) = self.child.take() { + warn!( + "ending agent group {}: its supervisor failed before publishing", + self.pgid + ); + signal_guarded_group(self.pgid, libc::SIGKILL); + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +/// How long [`monitor_test_hold`] waits for its release file. +#[cfg(test)] +const MONITOR_TEST_HOLD_WAIT: Duration = Duration::from_secs(30); + +/// A test-only hold point: when `DEVFLOW_TEST_HOLD` is `:` for +/// this `point`, wait up to [`MONITOR_TEST_HOLD_WAIT`] for `` to +/// exist, so a test can land a fault in a chosen window. Compiled out of +/// non-test builds (pinned by +/// `monitor::tests::the_test_hold_is_compiled_out_of_non_test_builds`). +#[cfg(test)] +fn monitor_test_hold(point: &str) { + let Ok(hold) = std::env::var("DEVFLOW_TEST_HOLD") else { + return; + }; + let Some((at, path)) = hold.split_once(':') else { + return; + }; + if at != point { + return; + } + let release = Path::new(path); + let deadline = std::time::Instant::now() + MONITOR_TEST_HOLD_WAIT; + while !release.exists() && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } +} + /// Spawn `spec`'s agent in its own process group (`process_group(0)`, R-3), /// through `hermetic_command` with the adapter env re-applied after its /// scrub and the gate notify command stripped. Returns the child and, for @@ -1772,15 +1849,27 @@ fn read_child_stat(pid: u32) -> Option { expect(dead_code, reason = "spawn_monitor calls it from 50-11") )] fn direct_children(parent: u32) -> Vec<(u32, ChildStat)> { + proc_table() + .into_iter() + .filter(|(_, stat)| stat.ppid == parent) + .collect() +} + +/// Every readable `/proc//stat`, each from one read. An unlistable +/// `/proc` is an empty table (logged); an unreadable entry is skipped. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn proc_table() -> Vec<(u32, ChildStat)> { let Ok(entries) = std::fs::read_dir("/proc") else { - warn!("could not list /proc; no child of supervisor {parent} can be found"); + warn!("could not list /proc"); return Vec::new(); }; entries .filter_map(Result::ok) .filter_map(|entry| entry.file_name().to_str()?.parse::().ok()) .filter_map(|pid| Some((pid, read_child_stat(pid)?))) - .filter(|(_, stat)| stat.ppid == parent) .collect() } @@ -1790,10 +1879,6 @@ fn direct_children(parent: u32) -> Vec<(u32, ChildStat)> { /// own group and `kill(-1, …)` every process we may signal (the runtime's /// rule, T-50-18). Callers pass only a group they started or one whose /// leader they have just re-validated. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn signal_guarded_group(pgid: u32, signal: libc::c_int) -> bool { let Ok(signed) = libc::pid_t::try_from(pgid) else { warn!("process group {pgid} does not fit pid_t; not signalling"); @@ -1942,13 +2027,199 @@ fn wait_for_publish( } std::thread::sleep(PUBLISH_POLL); }; - if let Err(error) = crate::supervisor::remove_launch_files(socket) { + let recorded = end_recorded_agent_group(socket, launch_id); + let removed = if recorded.keep_sidecar { + match std::fs::remove_file(socket.as_path()) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } + } else { + crate::supervisor::remove_launch_files(socket) + }; + if let Err(error) = removed { warn!( "could not remove the launch files of {}: {error}", socket.as_path().display() ); } - Err(MonitorError::NotReady(format!("supervisor {pid} {how}"))) + Err(MonitorError::NotReady(format!( + "supervisor {pid} {how}; {}", + recorded.text + ))) +} + +/// How long the ending step waits for a SIGKILLed recorded agent group to +/// empty. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +const RECORDED_GROUP_GONE_WAIT: Duration = Duration::from_secs(2); + +/// What the ending step did about the agent group a launch's sidecar +/// records. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +struct RecordedGroupEnd { + /// The sentence the `NotReady` message carries. + text: String, + /// Keep the sidecar, so `doctor` still lists the launch: the backstop + /// refused the recorded group, or it did not empty after its SIGKILL. + keep_sidecar: bool, +} + +/// End the agent group the sidecar beside `socket` records, once the +/// supervisor that wrote it is reaped (RV-03). A `__monitor` SIGKILLed +/// after its sidecar write and before it published ran neither its +/// `AgentGroupGuard` nor its signal handler, and its agent (which leads its +/// own group, R-3) was reparented away, so the sidecar is the only record of +/// it. +/// +/// This reads a pgid from disk, so it is held to D-04's rule for the +/// backstop: the group is signalled only when the sidecar is this launch's, +/// [`backstop::assess`](crate::supervisor::backstop::assess) returns +/// `LeaderPresentSignal` (boot id current, leader present with the recorded +/// start time and leading the recorded group) and +/// [`agent::is_same_process`](crate::agent::is_same_process) re-confirms +/// the leader immediately before the SIGKILL. No sidecar, another launch +/// id or any other decision signals nothing, and the text says which. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn end_recorded_agent_group(socket: &SocketPath, launch_id: &str) -> RecordedGroupEnd { + use crate::supervisor::backstop::BackstopDecision; + let done = |text: String| RecordedGroupEnd { + text, + keep_sidecar: false, + }; + let sidecar = match crate::supervisor::read_sidecar(&socket.sidecar_path()) { + Ok(sidecar) => sidecar, + Err(crate::supervisor::SidecarError::Io(error)) + if error.kind() == std::io::ErrorKind::NotFound => + { + return done(format!( + "no sidecar for launch {launch_id}; no agent group could be identified" + )); + } + Err(error) => { + return done(format!( + "the sidecar for launch {launch_id} is unusable ({error}); nothing was signalled" + )); + } + }; + if sidecar.launch_id != launch_id { + return done(format!( + "the sidecar names launch {}, not {launch_id}; nothing was signalled", + sidecar.launch_id + )); + } + let pgid = sidecar.agent_pgid; + let decision = crate::supervisor::backstop::assess(&sidecar, None); + let name = decision_name(&decision); + match decision { + BackstopDecision::LeaderPresentSignal => { + if !crate::agent::is_same_process(pgid, sidecar.agent_start_time) { + return done(format!( + "the leader of the agent group {pgid} its sidecar records changed before \ + the signal; nothing was signalled" + )); + } + if !signal_guarded_group(pgid, libc::SIGKILL) { + return done(format!( + "the agent group {pgid} its sidecar records could not be signalled" + )); + } + if group_empties_within(pgid, RECORDED_GROUP_GONE_WAIT) { + done(format!("ended the agent group {pgid} its sidecar records")) + } else { + RecordedGroupEnd { + text: format!( + "ended the agent group {pgid} its sidecar records, but it still had \ + members after {RECORDED_GROUP_GONE_WAIT:?}; the sidecar is kept" + ), + keep_sidecar: true, + } + } + } + BackstopDecision::EmptyGroup + | BackstopDecision::LeaderRecycled + | BackstopDecision::PreviousBoot => done(format!( + "the agent group {pgid} its sidecar records reads {name}; nothing was signalled" + )), + BackstopDecision::RefuseLeaderGone(_) + | BackstopDecision::RefuseMismatch(_) + | BackstopDecision::RefuseUnsafeTarget + | BackstopDecision::RefuseIdentityUnavailable => RecordedGroupEnd { + text: format!( + "the agent group {pgid} its sidecar records reads {name} with {members} \ + member(s); nothing was signalled and the sidecar is kept", + members = group_member_count(pgid) + ), + keep_sidecar: true, + }, + } +} + +/// The variant name of a backstop decision, for the `NotReady` text. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn decision_name(decision: &crate::supervisor::backstop::BackstopDecision) -> &'static str { + use crate::supervisor::backstop::BackstopDecision; + match decision { + BackstopDecision::RefuseUnsafeTarget => "RefuseUnsafeTarget", + BackstopDecision::RefuseIdentityUnavailable => "RefuseIdentityUnavailable", + BackstopDecision::RefuseMismatch(_) => "RefuseMismatch", + BackstopDecision::PreviousBoot => "PreviousBoot", + BackstopDecision::EmptyGroup => "EmptyGroup", + BackstopDecision::LeaderPresentSignal => "LeaderPresentSignal", + BackstopDecision::LeaderRecycled => "LeaderRecycled", + BackstopDecision::RefuseLeaderGone(_) => "RefuseLeaderGone", + } +} + +/// How many processes `/proc` lists in group `pgid` right now. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn group_member_count(pgid: u32) -> usize { + proc_table() + .iter() + .filter(|(_, stat)| i64::from(stat.pgid) == i64::from(pgid)) + .count() +} + +/// Poll until no process is left in group `pgid` (`kill(-pgid, 0)` fails +/// with ESRCH), up to `wait`. A pgid that is not a safe group target never +/// counts as empty. +#[cfg_attr( + not(test), + expect(dead_code, reason = "spawn_monitor calls it from 50-11") +)] +fn group_empties_within(pgid: u32, wait: Duration) -> bool { + let Ok(signed) = libc::pid_t::try_from(pgid) else { + return false; + }; + if signed <= 1 { + return false; + } + let deadline = std::time::Instant::now() + wait; + loop { + // SAFETY: signal 0 only checks for existence; `signed > 1`. + let rc = unsafe { libc::kill(-signed, 0) }; + if rc != 0 && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + std::thread::sleep(PUBLISH_POLL); + } } /// The idle-timeout firing sequence, in the ONE order it may run (D-05). @@ -6765,9 +7036,23 @@ exit 0 sidecar.is_some() || child.0.try_wait().ok().flatten().is_some() }); let entry_log = || std::fs::read_to_string(&log).unwrap_or_default(); - let sidecar = sidecar - .filter(|_| parked) - .unwrap_or_else(|| panic!("no sidecar within 5 s\n{}", entry_log())); + let Some(sidecar) = sidecar.filter(|_| parked) else { + // End an agent the child may already have started, by the pid + // file it wrote and the start time read now, before failing. + let _orphan = std::fs::read_to_string(crate::agent_result::agent_pid_path( + &fixture.root, + STAGE_PHASE, + )) + .ok() + .and_then(|pid| pid.trim().parse::().ok()) + .and_then(|pid| { + crate::agent::process_start_time(pid).map(|start_time| LeaderGroup { + pgid: pid, + start_time, + }) + }); + panic!("no sidecar within 5 s\n{}", entry_log()); + }; let agent = LeaderGroup { pgid: sidecar.agent_pgid, start_time: sidecar.agent_start_time, @@ -6854,8 +7139,39 @@ exit 0 /// Whether non-test `monitor.rs` defines `fn monitor_test_hold(` exactly /// once, directly under attribute lines that include `#[cfg(test)]`. - fn test_hold_guard_verdict(_non_test: &str) -> Result<(), String> { - Ok(()) + fn test_hold_guard_verdict(non_test: &str) -> Result<(), String> { + let lines: Vec<&str> = non_test + .lines() + .filter(|line| !line.trim_start().starts_with("//")) + .collect(); + let definitions: Vec = lines + .iter() + .enumerate() + .filter(|(_, line)| line.trim_start().starts_with("fn monitor_test_hold(")) + .map(|(at, _)| at) + .collect(); + let [at] = definitions[..] else { + return Err(format!( + "expected exactly one `fn monitor_test_hold(` in non-test monitor.rs, found {}; \ + if the hold is removed on purpose, delete this test with it", + definitions.len() + )); + }; + let guarded = lines[..at] + .iter() + .rev() + .take_while(|line| line.trim_start().starts_with("#[")) + .any(|line| line.trim() == "#[cfg(test)]"); + if guarded { + Ok(()) + } else { + Err( + "`fn monitor_test_hold(` is not directly under `#[cfg(test)]`, so the hold \ + would ship in release builds; if the hold is removed on purpose, delete this \ + test with it" + .to_string(), + ) + } } #[test] diff --git a/doc-check-allowlist.toml b/doc-check-allowlist.toml index ff4a08b2..b8e0a720 100644 --- a/doc-check-allowlist.toml +++ b/doc-check-allowlist.toml @@ -59,3 +59,8 @@ reason = "Test-only: carries the __monitor argv to the CLI #[cfg(test)] monitor kind = "source_to_docs" token = "DEVFLOW_TEST_MONITOR_FULL" reason = "Test-only: selects the advance-running monitor harness entry for #[cfg(test)] stage launches in the CLI own unit tests; no release build reads it, so it is not an operator setting (Phase 50, plan 50-10)." + +[[exceptions]] +kind = "source_to_docs" +token = "DEVFLOW_TEST_HOLD" +reason = "Test-only: #[cfg(test)] hold points in the devflow-core monitor (50-24) and the CLI stage handoff (50-13, 50-23) wait while it names : and that path does not exist, so tests can land a fault in a chosen window; compiled out of release builds, so it is not an operator setting." From 1b810f9a131cced57998cd8c639c4b9af94e6f48 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 16:07:15 -0400 Subject: [PATCH 048/112] test(50-11): add failing tests for the published supervisor launch - supervisor_e2e: devflow start records a socket under the cache dir that answers ALIVE with the launch shape's label, for stream and capture - monitor scan: non-test monitor.rs starts two process groups and both defines and calls wait_for_publish Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/tests/supervisor_e2e.rs | 218 +++++++++++++++++++++ crates/devflow-core/src/monitor.rs | 48 +++++ 2 files changed, 266 insertions(+) create mode 100644 crates/devflow-cli/tests/supervisor_e2e.rs diff --git a/crates/devflow-cli/tests/supervisor_e2e.rs b/crates/devflow-cli/tests/supervisor_e2e.rs new file mode 100644 index 00000000..03069e5e --- /dev/null +++ b/crates/devflow-cli/tests/supervisor_e2e.rs @@ -0,0 +1,218 @@ +//! End-to-end proof that a stage launch runs under an addressed Rust +//! supervisor (50-11, criteria 2 and 3). +//! +//! The launch path is `commands::start` → `launch_stage` → +//! `monitor::spawn_monitor` → the hidden `__monitor` subcommand, all inside +//! the `devflow` BINARY crate (no `lib.rs`), so integration tests cannot call +//! any of it in-process. Like `stop_e2e.rs`, this drives the real compiled +//! binary through `common::hermetic_devflow` and observes what an operator +//! can: the persisted state, the socket it names, and the files the +//! supervisor writes. +//! +//! Every `devflow` child gets a tempdir `HOME` and `DEVFLOW_CACHE_DIR`, so the +//! sockets these launches bind live under that tempdir, never under the +//! operator's real `~/.cache/devflow`. + +mod common; + +use devflow_core::phase_id::PhaseId; +use devflow_core::supervisor::{self, Liveness, ShutdownOutcome}; +use std::fs; +use std::path::Path; +use std::process::Output; +use std::time::Duration; + +/// How long a shutdown waits for the supervisor's final line. +const SHUTDOWN_WAIT: Duration = Duration::from_secs(30); + +/// Hermetic `git` for fixture setup (999.37), never a bare `Command::new("git")`. +fn git(root: &Path, args: &[&str]) { + let output = devflow_core::test_support::git_command(root) + .args(args) + .output() + .expect("spawn git"); + assert!( + output.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// A repository on `develop` with a roadmap entry for `phase`. +fn init_repo(root: &Path, phase: PhaseId) { + git(root, &["init", "-q"]); + git(root, &["config", "user.email", "devflow@example.com"]); + git(root, &["config", "user.name", "DevFlow Tests"]); + git(root, &["config", "commit.gpgsign", "false"]); + git(root, &["config", "core.hooksPath", "/dev/null"]); + git(root, &["checkout", "-q", "-b", "develop"]); + fs::create_dir_all(root.join(".planning")).unwrap(); + fs::write(root.join("README.md"), "base\n").unwrap(); + fs::write( + root.join(".planning/ROADMAP.md"), + format!("# Roadmap\n\n### Phase {phase}: supervisor fixture\n"), + ) + .unwrap(); + git(root, &["add", "README.md", ".planning/ROADMAP.md"]); + git(root, &["commit", "-q", "-m", "base"]); +} + +/// A fake `claude` that answers `--version` (the launch's health check), the +/// delivery canary's planted token (so a stream launch passes +/// `canary_gate`), and otherwise stays running until it is ended. Under the +/// capture shape its stdin is `/dev/null`, so the `read` yields nothing and +/// it goes straight to waiting. +const WAITING_CLAUDE: &str = r#"if [ "$1" = "--version" ]; then echo "2.1.220 (Claude Code)"; exit 0; fi +read -r turn +case "$turn" in + *DEVFLOW_DELIVERY_CANARY_*) + token=$(printf '%s' "$turn" | grep -o 'DEVFLOW_DELIVERY_CANARY_[0-9a-f]*' | head -1) + printf '{"type":"result","subtype":"success","is_error":false,"session_id":"s-fake","result":"%s"}\n' "$token" + exit 0 + ;; +esac +sleep 120"#; + +/// One launch shape as `devflow start` selects it. +#[derive(Debug, Clone, Copy)] +enum Shape { + /// The default Claude launch: the stream shape. + Stream, + /// `--legacy-claude-launch`: the capture shape. + Capture, +} + +impl Shape { + fn label(self) -> &'static str { + match self { + Shape::Stream => "stream", + Shape::Capture => "capture", + } + } + + fn start_flags(self) -> &'static [&'static str] { + match self { + Shape::Stream => &[], + Shape::Capture => &["--legacy-claude-launch"], + } + } +} + +/// `devflow start` for `phase` in supervise mode, with `fake_bin` first on PATH. +fn run_start( + home: &common::HermeticHome, + root: &Path, + phase: PhaseId, + fake_bin: &common::FakeBin, + shape: Shape, +) -> Output { + let existing = std::env::var_os("PATH").unwrap_or_default(); + let path = format!("{}:{}", fake_bin.path.display(), existing.to_string_lossy()); + common::hermetic_devflow(home) + .args([ + "start", + "--phase", + &phase.to_string(), + "--agent", + "claude", + "--mode", + "supervise", + "--no-worktree", + ]) + .args(shape.start_flags()) + .arg(root) + .current_dir(root) + .env("PATH", path) + .output() + .expect("run devflow start") +} + +/// Criterion 3 for one shape: after `devflow start` returns, the state names +/// a socket under the hermetic cache dir that answers ALIVE with the +/// shape's label, its sidecar names this root and phase, and a socket +/// shutdown stops the run and leaves the socket Gone. +fn start_publishes_for(shape: Shape, phase: PhaseId) { + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path().canonicalize().unwrap(); + init_repo(&root, phase); + let fake_bin = common::fake_bin_dir(WAITING_CLAUDE); + let _reaper = common::LaunchedRunReaper::new(&root, phase, vec![]); + + // Control: nothing names a supervisor before the launch, so an Alive + // answer below can only come from this launch. + assert!( + devflow_core::workflow::load_state(&root, phase).is_err(), + "{shape:?}: no state may exist before start" + ); + + let output = run_start(&home, &root, phase, &fake_bin, shape); + assert!( + output.status.success(), + "{shape:?}: start failed\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let state = devflow_core::workflow::load_state(&root, phase).expect("start saved state"); + let record = state + .supervisor + .clone() + .unwrap_or_else(|| panic!("{shape:?}: start must record state.supervisor: {state:?}")); + let socket = record.socket_path.clone(); + let sock_dir = home.cache_dir().join("sock"); + assert!( + socket.starts_with(&sock_dir), + "{shape:?}: socket {} must be under the child's cache dir {}", + socket.display(), + sock_dir.display() + ); + assert!( + !socket.starts_with(&root), + "{shape:?}: the socket must not live in the project tree" + ); + assert!( + socket.as_os_str().len() <= 107, + "{shape:?}: socket path is {} bytes", + socket.as_os_str().len() + ); + assert!( + state.monitor_pid.is_some(), + "{shape:?}: monitor_pid stays populated until 50-22" + ); + + let reply = match supervisor::probe(&socket).expect("probe the recorded socket") { + Liveness::Alive(reply) => reply, + other => panic!("{shape:?}: the recorded socket must answer ALIVE, got {other:?}"), + }; + assert_eq!(reply.launch, shape.label(), "{shape:?}: launch label"); + assert_eq!(reply.launch_id, record.launch_id, "{shape:?}: launch id"); + assert_eq!(reply.agent_pgid, record.agent_pgid, "{shape:?}: agent pgid"); + assert_eq!( + Some(reply.supervisor_pid), + state.monitor_pid, + "{shape:?}: monitor_pid names the supervisor" + ); + + let sidecar_path = socket.with_extension("json"); + let sidecar = supervisor::read_sidecar(&sidecar_path).expect("read the sidecar"); + assert_eq!(sidecar.project_root, root, "{shape:?}: sidecar root"); + assert_eq!(sidecar.phase, phase, "{shape:?}: sidecar phase"); + assert_eq!(sidecar.launch, shape.label(), "{shape:?}: sidecar launch"); + + match supervisor::request_shutdown(&socket, SHUTDOWN_WAIT).expect("shutdown the run") { + ShutdownOutcome::Stopped { .. } => {} + other => panic!("{shape:?}: shutdown must stop the run, got {other:?}"), + } + assert_eq!( + supervisor::probe(&socket).expect("probe after shutdown"), + Liveness::Gone, + "{shape:?}: the socket is gone once the supervisor stopped" + ); +} + +#[test] +fn start_publishes_a_live_supervisor_socket_for_each_launch_shape() { + start_publishes_for(Shape::Stream, PhaseId::new(61)); + start_publishes_for(Shape::Capture, PhaseId::new(62)); +} diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index b78d9fbf..77690cfe 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -7208,4 +7208,52 @@ exit 0 panic!("{path}: {why}"); } } + + /// `(process_group(0) calls, wait_for_publish( occurrences)` in `source` + /// cut at its first `mod tests {` line, with every line whose first + /// non-blank characters are `//` dropped, so neither comments nor this + /// module's own literals count. + fn launch_scan_counts(source: &str) -> (usize, usize) { + let code: Vec<&str> = source + .lines() + .take_while(|line| !line.starts_with("mod tests {")) + .filter(|line| !line.trim_start().starts_with("//")) + .collect(); + let code = code.join("\n"); + ( + code.matches("process_group(0)").count(), + code.matches("wait_for_publish(").count(), + ) + } + + /// R6-G4, permanent: the agent and the `__monitor` supervisor each lead + /// their own process group, and `spawn_monitor` waits for the publish + /// through 50-24's `wait_for_publish` (its definition plus the call). + #[test] + fn spawn_monitor_groups_both_processes_and_waits_for_publish() { + let planted = "fn f() {\n command.process_group(0);\n // .process_group(0)\n \ + wait_for_publish(&mut child);\n}\nmod tests {\n process_group(0) \ + process_group(0) wait_for_publish( wait_for_publish(\n}\n"; + assert_eq!( + launch_scan_counts(planted), + (1, 1), + "planted control: one call and one publish wait count once each; the comment \ + line and the test module do not count" + ); + + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/src/monitor.rs"); + let source = + std::fs::read_to_string(path).unwrap_or_else(|err| panic!("read {path}: {err}")); + let (groups, waits) = launch_scan_counts(&source); + assert!( + groups >= 2, + "{path}: non-test code must start both the agent and the `__monitor` supervisor \ + with `process_group(0)`; found {groups}" + ); + assert!( + waits >= 2, + "{path}: non-test code must define `wait_for_publish(` and call it from \ + `spawn_monitor`; found {waits}" + ); + } } From 1a6e59f4493296a398daf51bbcde06f6c8161803 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 16:15:41 -0400 Subject: [PATCH 049/112] feat(50-11): launch both shapes under the published Rust supervisor - spawn_monitor re-execs __monitor for capture and stream alike, in its own process group, and returns Published only after wait_for_publish sees this launch id answer a ping; 50-24's dead-code expectations go - delete the sh monitor script, its stop marker, MonitorTail, advance_tail, shell_escape, pipe_owning_monitor_args and run_pipe_owning_monitor (R-9 break, semver gate verdict=match) - __monitor gains --launch, --socket, --launch-id and --keep-env; run_monitor runs run_supervised_stage, advances only on Completed and records state.stopped on a supervisor stop - spawn_agent_and_record saves state.supervisor with monitor_pid in one save; stage_launched gains socket_path, launch_id and launch - the stream-shape pipe tests drive spawn_stage_agent + drive_stream_io; three capture tests move onto run_supervised_stage in a child process; devflow_dir_gitignore case 3 proves the .gitignore through NotReady Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/main.rs | 60 +- crates/devflow-cli/src/pipeline_launch.rs | 378 ++- crates/devflow-cli/src/test_support.rs | 29 +- crates/devflow-core/src/monitor.rs | 2033 +++++------------ crates/devflow-core/src/supervisor/runtime.rs | 2 +- .../tests/devflow_dir_gitignore.rs | 109 +- 6 files changed, 902 insertions(+), 1709 deletions(-) diff --git a/crates/devflow-cli/src/main.rs b/crates/devflow-cli/src/main.rs index 4ecdddf1..bdbe7f41 100644 --- a/crates/devflow-cli/src/main.rs +++ b/crates/devflow-cli/src/main.rs @@ -128,12 +128,13 @@ enum Command { #[arg(long)] stage: Option, }, - /// Internal: the pipe-owning monitor's own process entry point (Phase 31). + /// Internal: the stage supervisor's own process entry point (50-11). /// - /// `monitor::spawn_monitor`'s `PipeOwning` arm re-execs this binary as this - /// subcommand, detached. Everything after `--` is the supervised child's - /// program and argv. Hidden for the same reason `advance` is: it is an - /// implementation detail of the monitor chain, never an operator verb. + /// `monitor::spawn_monitor` re-execs this binary as this subcommand, + /// detached and in its own process group, for both launch shapes. + /// Everything after `--` is the supervised agent's program and argv. + /// Hidden for the same reason `advance` is: it is an implementation + /// detail of the monitor chain, never an operator verb. #[command(name = "__monitor", hide = true)] Monitor { /// Project root whose `.devflow/` capture files this monitor writes. @@ -149,17 +150,28 @@ enum Command { /// worktree mode is active, else the project root). #[arg(long)] workdir: PathBuf, - /// File holding the stage prompt. A file, not argv: argv has a hard - /// length ceiling and DevFlow stage prompts routinely exceed it. - #[arg(long)] - prompt_file: PathBuf, - /// Seconds of stream silence before the idle timeout fires. - #[arg(long)] - idle_timeout_secs: u64, /// Which agent's transport the monitor supervises (selects the /// first-turn schema and the close-rule marker predicate). #[arg(long)] agent: AgentKind, + /// Supervision shape: `capture` (stdin null, stdout to the capture file) or `stream`. + #[arg(long, value_enum)] + launch: pipeline_launch::LaunchShape, + /// Socket path this supervisor publishes, under the DevFlow cache dir. + #[arg(long)] + socket: PathBuf, + /// Per-launch id the launcher waits to see in this supervisor's ping reply. + #[arg(long)] + launch_id: String, + /// Adapter env var NAME to re-apply to the agent (repeatable; the value stays off argv). + #[arg(long = "keep-env")] + keep_env: Vec, + /// File holding the stage prompt (stream shape; a file, not argv, for length). + #[arg(long)] + prompt_file: Option, + /// Seconds of stream silence before the idle timeout fires (stream shape). + #[arg(long)] + idle_timeout_secs: Option, /// The supervised child's program and arguments, after `--`. #[arg(trailing_var_arg = true, allow_hyphen_values = true, required = true)] argv: Vec, @@ -590,20 +602,28 @@ fn run() -> Result<(), CliError> { phase, stage, workdir, + agent, + launch, + socket, + launch_id, + keep_env, prompt_file, idle_timeout_secs, - agent, argv, - } => run_monitor( - &project_root(project)?, + } => run_monitor(&pipeline_launch::MonitorInvocation { + project_root: project_root(project)?, phase, stage, - &workdir, - &prompt_file, - idle_timeout_secs, + workdir, agent, - &argv, - ), + launch, + socket, + launch_id, + keep_env, + prompt_file, + idle_timeout_secs, + argv, + }), Command::Resume { phase, agent, diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index ecb9e796..02627941 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -41,6 +41,7 @@ use devflow_core::phase_id::PhaseId; use devflow_core::prompt; use devflow_core::stage::Stage; use devflow_core::state::{AgentKind, ApprovedCheckpoint, CheckpointApproval, State}; +use devflow_core::supervisor::runtime::AgentOutcome; use devflow_core::verify::CheckpointDeclaration; use devflow_core::{ agent_result, agents, canary, events, gsd_config, lock, mode, monitor, verify, workflow, @@ -900,131 +901,253 @@ impl Drop for AutoChainGuard { } } -/// The detached pipe-owning monitor's own process body (Phase 31): supervise -/// the child, then advance the stage machine exactly as the shell monitor's -/// `devflow advance` tail did. -/// -/// Runs in the `__monitor` process, never in the operator's CLI. +/// The `__monitor` subcommand's `--launch` value: which supervision shape the +/// supervisor wires the agent's stdio for. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +pub(crate) enum LaunchShape { + /// Stdin `/dev/null`, stdout to the phase capture file + /// (`MonitorLaunch::Legacy`). + Capture, + /// The supervisor owns both pipes and delivers the prompt as the first + /// turn (`MonitorLaunch::PipeOwning`). + Stream, +} + +/// One `__monitor` invocation, as `main` (or the test build's harness entry) +/// parsed it from the argv `monitor::spawn_monitor` built. +#[derive(Debug, Clone)] +pub(crate) struct MonitorInvocation { + /// Project root whose `.devflow/` files the supervisor writes. + pub(crate) project_root: PathBuf, + /// Phase whose stage machine to advance once the agent completes. + pub(crate) phase: PhaseId, + /// Stage the launch ran; passed to advance. + pub(crate) stage: Option, + /// The agent's working directory. + pub(crate) workdir: PathBuf, + /// Which coding agent this is. + pub(crate) agent: AgentKind, + /// The supervision shape. + pub(crate) launch: LaunchShape, + /// Where the supervisor publishes its socket. + pub(crate) socket: PathBuf, + /// The per-launch id the launcher waits to see in a ping reply. + pub(crate) launch_id: String, + /// Adapter environment variable NAMES to re-apply to the agent from this + /// process's environment (values never travel on argv, T-50-41). + pub(crate) keep_env: Vec, + /// The stream shape's prompt file. + pub(crate) prompt_file: Option, + /// The stream shape's idle timeout in seconds. + pub(crate) idle_timeout_secs: Option, + /// The agent's program and argv. + pub(crate) argv: Vec, +} + +/// The stop reason a run ended through its supervisor records. +pub(crate) const SUPERVISOR_STOP_REASON: &str = + "stopped through the supervisor (socket shutdown or SIGTERM/SIGINT)"; + +/// How long a stopped supervisor waits for the per-phase lock before it +/// records the stop. +const STOP_RECORD_LOCK_WAIT: Duration = Duration::from_secs(30); + +/// The `__monitor` process body (50-11): supervise the stage's agent under +/// an addressed supervisor (`monitor::run_supervised_stage`), then, only +/// when the agent completed on its own, advance the stage machine +/// in-process (D-02). A stopped run is recorded as stopped and never +/// advances. /// -/// `envs` is deliberately empty here — the adapter's extra env was applied to -/// THIS process by `spawn_monitor` and rides down by inheritance. That is -/// sufficient only because the sole adapter routed through the pipe-owning arm -/// (Claude) declares no extra env; see the note at `spawn_monitor`'s -/// `PipeOwning` arm before widening it. -#[allow( - clippy::too_many_arguments, - reason = "the hidden CLI dispatch supplies these independently parsed monitor arguments" -)] -pub(crate) fn run_monitor( - project_root: &Path, - phase: PhaseId, - stage: Option, - workdir: &Path, - prompt_file: &Path, - idle_timeout_secs: u64, - agent: AgentKind, - argv: &[String], -) -> Result<(), CliError> { - supervise_monitor( - project_root, - phase, - stage, - workdir, - prompt_file, - idle_timeout_secs, - agent, - argv, - true, - ) +/// Runs in the detached `__monitor` process, never in the operator's CLI. +pub(crate) fn run_monitor(invocation: &MonitorInvocation) -> Result<(), CliError> { + supervise_monitor(invocation, true) } /// [`run_monitor`]'s body. `advance` is `true` on every production path; the /// test build's supervise-only monitor harness entry (50-10) passes `false` /// so an in-crate launch supervises its agent without moving the stage /// machine on. -#[allow( - clippy::too_many_arguments, - reason = "run_monitor's independently parsed monitor arguments plus the advance switch" -)] pub(crate) fn supervise_monitor( - project_root: &Path, - phase: PhaseId, - stage: Option, - workdir: &Path, - prompt_file: &Path, - idle_timeout_secs: u64, - agent: AgentKind, - argv: &[String], + invocation: &MonitorInvocation, advance: bool, ) -> Result<(), CliError> { - let prompt = std::fs::read_to_string(prompt_file).map_err(|err| { - CliError::Message(format!( - "monitor could not read the prompt file {}: {err}", - prompt_file.display() - )) - })?; - let Some((program, args)) = argv.split_first() else { - return Err(CliError::Message( - "monitor was given no child program to supervise".to_string(), - )); - }; + let MonitorInvocation { + project_root, + phase, + stage, + workdir, + launch, + .. + } = invocation; + let (project_root, phase, stage) = (project_root.as_path(), *phase, *stage); + let spec = stage_spec(invocation)?; // D-01/D-06: hold GSD's `workflow._auto_chain_active` at the value this - // launch requires for as long as the child runs, and return it to `false` - // when this function returns — by `?` on the monitor's `Err` OR by falling - // through to `advance` below. Both exits are covered because the guard is - // bound to a named variable in THIS scope; `let _ = ...` would drop it - // immediately and the flag's true-window would collapse to nothing. + // launch requires for as long as the agent runs, and return it to + // `false` before anything else happens: on the `?` below when the + // supervisor fails, and right after the agent's run otherwise. // - // The target is `workdir`, not `project_root`: `.planning/config.json` is a - // tracked file inside the worktree the agent's cwd is set to, and that copy - // is the one GSD's `check auto-mode` reads. + // The target is `workdir`, not `project_root`: `.planning/config.json` is + // a tracked file inside the worktree the agent's cwd is set to, and that + // copy is the one GSD's `check auto-mode` reads. // - // F-4 — no agent or launch-shape condition belongs in the predicate. - // `run_monitor` is the body of the hidden `__monitor` subcommand, which - // `monitor::spawn_monitor` re-execs ONLY on its `MonitorLaunch::PipeOwning` - // arm. Being inside this function already implies a stream launch (Claude - // and Antigravity today, round-3 D-10) — the old "Claude + stream launch" - // claim became false when the predicate widened — so re-checking - // `state.agent` or `state.legacy_claude_launch` here would be a second, - // driftable notion of the same fact. The consequence — a Legacy-arm or - // non-stream-agent launch never gets the flag — is accepted and is turned - // into a loud preflight refusal by plan `35.1-03`, not left silent. + // Conditioned on the launch shape explicitly: since 50-11 both shapes + // reach this function, and only a stream launch can bound the flag's + // lifetime to the agent's (35.1-03's preflight refuses the capture shape + // in `Mode::Auto` for exactly that reason). A capture launch never gets + // the flag. + // + // The guard is dropped BEFORE the advance, not after it: the advance + // launches the next stage and waits for that stage's supervisor to + // publish, and that supervisor has already set the flag for its own + // agent. Clearing it after the advance would clear the next stage's + // value. // // A state that will not load is NOT fatal here: warn, skip the guard, and // let `advance` surface the real state error afterwards with its own // context. - let _auto_chain_guard = match workflow::load_state(project_root, phase) { - Ok(state) => Some(AutoChainGuard::engage( - workdir, - auto_chain_flag_eligible(state.stage, state.mode), - )), - Err(err) => { - warn!( - "monitor could not load state for phase {phase} ({err}) — running \ - without the GSD chain-flag guard" - ); - None + let auto_chain_guard = match launch { + LaunchShape::Capture => None, + LaunchShape::Stream => match workflow::load_state(project_root, phase) { + Ok(state) => Some(AutoChainGuard::engage( + workdir, + auto_chain_flag_eligible(state.stage, state.mode), + )), + Err(err) => { + warn!( + "monitor could not load state for phase {phase} ({err}) — running \ + without the GSD chain-flag guard" + ); + None + } + }, + }; + + let run = monitor::run_supervised_stage(spec) + .map_err(|err| CliError::Message(format!("stage supervisor failed: {err}")))?; + drop(auto_chain_guard); + + let result = match run.outcome { + AgentOutcome::Completed(_) if advance => { + advance_for_stage(project_root, Some(phase), stage) + } + AgentOutcome::Completed(_) => { + wait_for_launch_record(project_root, phase, &invocation.launch_id); + Ok(()) } + _ => record_supervisor_stop(project_root, phase), }; + // Never a `stopped` line for a completed run: `finish` decides the final + // line from the recorded outcome, so a shutdown that arrived while + // advancing is answered `handed_off`. + run.supervisor.finish(); + result +} - monitor::run_pipe_owning_monitor( - project_root, - phase, - workdir, - &prompt, - std::time::Duration::from_secs(idle_timeout_secs), - program, - args, - &[], - agent, - ) - .map_err(|err| CliError::Message(format!("pipe-owning monitor failed: {err}")))?; +/// The [`monitor::StageSpec`] for `invocation`: the stream shape reads its +/// prompt file, and each `--keep-env` name takes its value from this +/// process's environment. +fn stage_spec(invocation: &MonitorInvocation) -> Result { + let Some((program, args)) = invocation.argv.split_first() else { + return Err(CliError::Message( + "monitor was given no child program to supervise".to_string(), + )); + }; + let io = match invocation.launch { + LaunchShape::Capture => monitor::StageIo::Capture, + LaunchShape::Stream => { + let (Some(prompt_file), Some(idle_secs)) = + (&invocation.prompt_file, invocation.idle_timeout_secs) + else { + return Err(CliError::Message( + "a stream launch needs --prompt-file and --idle-timeout-secs".to_string(), + )); + }; + let prompt = std::fs::read_to_string(prompt_file).map_err(|err| { + CliError::Message(format!( + "monitor could not read the prompt file {}: {err}", + prompt_file.display() + )) + })?; + monitor::StageIo::Stream { + prompt, + idle_timeout: Duration::from_secs(idle_secs), + } + } + }; + let socket = devflow_core::supervisor::SocketPath::new(invocation.socket.clone()) + .map_err(|err| CliError::Message(format!("monitor socket path refused: {err}")))?; + let mut spec = monitor::StageSpec::new( + invocation.project_root.clone(), + invocation.phase, + invocation.agent, + program.clone(), + io, + socket, + invocation.launch_id.clone(), + ); + spec.stage = invocation.stage; + spec.workdir = invocation.workdir.clone(); + spec.args = args.to_vec(); + spec.envs = kept_env(&invocation.keep_env); + Ok(spec) +} - if !advance { - return Ok(()); +/// Each named variable with its value from this process's environment, +/// where `spawn_monitor` put it. A name that is unset or not UTF-8 is +/// skipped with a warning. +fn kept_env(names: &[String]) -> Vec<(String, String)> { + names + .iter() + .filter_map( + |name| match std::env::var_os(name).map(|value| value.into_string()) { + Some(Ok(value)) => Some((name.clone(), value)), + _ => { + warn!( + "--keep-env {name}: not set to a UTF-8 value in the monitor's environment" + ); + None + } + }, + ) + .collect() +} + +/// Record a run its supervisor stopped: under the per-phase lock (bounded +/// wait), `state.stopped = true` with [`SUPERVISOR_STOP_REASON`]. Never +/// advances. +fn record_supervisor_stop(project_root: &Path, phase: PhaseId) -> Result<(), CliError> { + let _lock = lock::acquire_blocking(project_root, phase, STOP_RECORD_LOCK_WAIT) + .map_err(|err| CliError::Message(format!("could not record the stop: {err}")))?; + let mut state = workflow::load_state(project_root, phase)?; + state.stopped = true; + state.stop_reason = Some(SUPERVISOR_STOP_REASON.to_string()); + workflow::save_state(&state)?; + info!("phase {phase}: {SUPERVISOR_STOP_REASON}"); + Ok(()) +} + +/// How long the supervise-only entry waits for its launcher's record. +const LAUNCH_RECORD_WAIT: Duration = Duration::from_secs(15); + +/// The supervise-only path (test build): keep serving until the launcher has +/// recorded THIS launch in the state file (bounded), so a launcher still +/// waiting for the publish cannot miss a socket that finished first. A +/// production supervisor gets the same ordering from `advance`, which takes +/// the per-phase lock its launcher holds until after the publish. +fn wait_for_launch_record(project_root: &Path, phase: PhaseId, launch_id: &str) { + let deadline = std::time::Instant::now() + LAUNCH_RECORD_WAIT; + while std::time::Instant::now() < deadline { + let recorded = workflow::load_state(project_root, phase) + .ok() + .and_then(|state| state.supervisor) + .is_some_and(|record| record.launch_id == launch_id); + if recorded { + return; + } + std::thread::sleep(Duration::from_millis(20)); } - advance_for_stage(project_root, Some(phase), stage) + warn!("launch {launch_id} was not recorded within {LAUNCH_RECORD_WAIT:?}"); } /// Resolve, once per launch, the cache directory and the monitor executable @@ -1165,12 +1288,23 @@ fn spawn_agent_and_record( ); } let ctx = launch_context()?; - let pid = monitor::spawn_monitor(state, program, args, extra_env, launch, &ctx) + let launch_label = match &launch { + monitor::MonitorLaunch::PipeOwning { .. } => "stream", + monitor::MonitorLaunch::Legacy => "capture", + }; + let published = monitor::spawn_monitor(state, program, args, extra_env, launch, &ctx) .map_err(|err| CliError::Message(format!("could not spawn monitor: {err}")))?; + let pid = published.supervisor_pid; + let socket_path = published.record.socket_path.display().to_string(); + let launch_id = published.record.launch_id.clone(); // `transition()` calls `workflow::save_state` BEFORE `launch_stage`, so a // pid recorded only in memory here is lost unless it is written again - // (18b). + // (18b). The ping-confirmed supervisor record replaces any previous one + // in this same save; it is never cleared first (WR-04's pre-clear above + // applies to `monitor_pid` only). `monitor_pid` names the supervisor + // until 50-22 removes it. state.monitor_pid = Some(pid); + state.supervisor = Some(published.record); workflow::save_state(state)?; match devflow_core::ship::consume_cron_instructions(&state.project_root, state.phase) { Ok(Some(path_kind)) => { @@ -1205,10 +1339,13 @@ fn spawn_agent_and_record( "stage": state.stage.to_string(), "agent": state.agent.to_string(), "monitor_pid": pid, + "socket_path": socket_path, + "launch_id": launch_id, + "launch": launch_label, }), ); println!( - "stage {} → launched {} (monitor pid {pid})", + "stage {} → launched {} (monitor pid {pid}, {launch_label} supervisor at {socket_path})", state.stage, agents::driver_for(state.agent).name() ); @@ -2410,15 +2547,15 @@ mod tests { exe: std::env::current_exe().unwrap(), }, ); - let control_pid = - monitor::spawn_monitor(&control, "claude", &[], &[], prompt(), &subcommand).unwrap(); - control.monitor_pid = Some(control_pid); - let _control_reap = ReapMonitorOnDrop::after_launch(&control); + // Since 50-11 `spawn_monitor` waits for the publish, so the control + // monitor (libtest given `__monitor`'s arguments) exiting by itself + // comes back as `NotReady` rather than as a pid to poll. + let control_result = + monitor::spawn_monitor(&control, "claude", &[], &[], prompt(), &subcommand); assert!( - poll_until(Duration::from_secs(30), || { - !devflow_core::agent::agent_running(control_pid) - }), - "the control monitor (libtest given __monitor's arguments) must exit by itself" + matches!(control_result, Err(monitor::MonitorError::NotReady(_))), + "the control monitor (libtest given __monitor's arguments) must exit by itself \ + without publishing: {control_result:?}" ); assert!( !control_root.join("agent-ran").exists(), @@ -3312,6 +3449,21 @@ mod tests { plan files, so the set is legitimately empty" ); + // 50-11: the relaunch also records its ping-confirmed supervisor, + // asserted rather than merely excluded for the same reason. + let supervisor_before = before + .as_object_mut() + .unwrap() + .remove("supervisor") + .unwrap(); + let supervisor_after = after.as_object_mut().unwrap().remove("supervisor").unwrap(); + assert_eq!(supervisor_before, serde_json::Value::Null); + assert!( + supervisor_after["socket_path"].is_string() + && supervisor_after["launch_id"].is_string(), + "the relaunch must record its published supervisor: {supervisor_after}" + ); + for value in [&mut before, &mut after] { value.as_object_mut().unwrap().remove("agent"); value.as_object_mut().unwrap().remove("monitor_pid"); diff --git a/crates/devflow-cli/src/test_support.rs b/crates/devflow-cli/src/test_support.rs index b217d298..24560a42 100644 --- a/crates/devflow-cli/src/test_support.rs +++ b/crates/devflow-cli/src/test_support.rs @@ -697,27 +697,34 @@ fn run_monitor_harness_entry(advance: bool) { phase, stage, workdir, + agent, + launch, + socket, + launch_id, + keep_env, prompt_file, idle_timeout_secs, - agent, argv, } = cli.command else { panic!("the monitor argv parsed as a command other than __monitor"); }; - let project_root = crate::project_root(project).expect("monitor project root"); - crate::pipeline_launch::supervise_monitor( - &project_root, + let invocation = crate::pipeline_launch::MonitorInvocation { + project_root: crate::project_root(project).expect("monitor project root"), phase, stage, - &workdir, - &prompt_file, - idle_timeout_secs, + workdir, agent, - &argv, - advance, - ) - .expect("the harness-entry monitor must supervise its agent"); + launch, + socket, + launch_id, + keep_env, + prompt_file, + idle_timeout_secs, + argv, + }; + crate::pipeline_launch::supervise_monitor(&invocation, advance) + .expect("the harness-entry monitor must supervise its agent"); } /// The test build's full monitor entry (50-10): `run_monitor` exactly as the diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index 77690cfe..dbea0a92 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -1,15 +1,16 @@ -//! Background monitor daemon. +//! Background monitor: the addressed Rust supervisor that owns a stage's +//! coding agent. //! -//! Spawns a detached child process that *owns* the coding agent: it launches -//! the agent, captures its stdout and exit code into `.devflow/`, and — when -//! the agent exits — runs `devflow advance` to advance the stage machine. +//! [`spawn_monitor`] re-execs the `devflow` binary as its hidden `__monitor` +//! subcommand, which runs [`run_supervised_stage`]: it launches the agent, +//! captures its stdout and exit code into `.devflow/`, publishes a socket that +//! answers liveness and shutdown requests, and — when the agent completes — +//! advances the stage machine in-process. //! //! Owning the agent is the key fix over a CLI-scoped capture thread: because -//! the monitor outlives `devflow start`, the agent's stdout keeps flowing into -//! the capture file and its exit code is still reaped after the CLI exits. -//! -//! This is the core automation primitive — no cron, no scheduler, -//! no agent cooperation needed. +//! the supervisor outlives `devflow start`, the agent's stdout keeps flowing +//! into the capture file and its exit code is still reaped after the CLI +//! exits. use crate::agent_result::{IdleTimeoutCommit, IdleTimeoutRecord}; use crate::git::hermetic_command; @@ -284,24 +285,25 @@ pub fn idle_timeout_setting_for(agent: AgentKind) -> IdleTimeoutSetting { /// Which supervision shape [`spawn_monitor`] should launch. /// -/// This is a MODE selection on one supervisor, not two monitors: both arms -/// write the same capture, exit-code and agent-pid files under `.devflow/`, -/// and both end by advancing the same stage machine. Nothing downstream needs -/// to know which arm ran. +/// Both arms launch the same Rust supervisor (the hidden `__monitor` +/// subcommand running [`run_supervised_stage`]): it owns the agent in its own +/// process group, publishes an addressed socket, writes the same capture, +/// exit-code and agent-pid files under `.devflow/`, and advances the stage +/// machine in-process once the agent completes (D-02). The arm only selects +/// how the agent's stdio is wired. pub enum MonitorLaunch { - /// Phase 31: a Rust supervisor that owns BOTH of the child's pipes, - /// delivers `prompt` as a JSON user turn on the child's stdin, and holds - /// that stdin open past the child's first turn so a task-notification turn - /// can still be delivered (constraint 4). + /// The stream shape ([`StageIo::Stream`]): the supervisor owns BOTH of + /// the agent's pipes, delivers `prompt` as a JSON user turn on its stdin, + /// and holds that stdin open past the first turn so a task-notification + /// turn can still be delivered (constraint 4). PipeOwning { /// The stage prompt, delivered on the child's stdin rather than argv. prompt: String, }, - /// The pre-31 detached `sh` script: stdin is `/dev/null`, stdout is - /// redirected to the capture file by the shell, and the script waits on - /// the agent then runs `devflow advance`. Every non-Claude adapter, every - /// stage not yet widened by D-09/D-10's rollout, and the checkpoint-resume - /// relaunch all run through here, unchanged. + /// The capture shape ([`StageIo::Capture`]): stdin is `/dev/null` and + /// stdout goes to the capture file. Every non-Claude adapter not on the + /// stream path, every stage not yet widened by D-09/D-10's rollout, and + /// the checkpoint-resume relaunch run through here. Legacy, } @@ -381,21 +383,38 @@ pub(crate) fn strip_notify_env(child_cmd: &mut std::process::Command) { child_cmd.env_remove(crate::gates::GATE_NOTIFY_CMD_ENV); } -/// Spawn a background monitor that owns the agent for the given workflow state. +/// A supervisor [`spawn_monitor`] started and saw publish: the record the +/// caller persists in [`State::supervisor`] and the supervisor's pid. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Published { + /// The ping-confirmed handle on the supervisor: its socket, launch id, + /// agent process group, agent start time and boot id. + pub record: crate::state::SupervisorRecord, + /// The pid of the `__monitor` process serving the socket. + pub supervisor_pid: u32, +} + +/// Start the addressed Rust supervisor that owns the agent for `state`'s +/// stage, and return once it has published its socket. /// -/// The monitor is a detached process that: -/// 1. Launches the agent (`program` + `args`) with stdout captured to the -/// phase stdout file, recording the agent PID to the agent-pid file -/// 2. Waits for the agent to exit and records its exit code to the exit file -/// 3. Runs `devflow advance --phase N` to advance the workflow through its -/// remaining stages +/// Both [`MonitorLaunch`] shapes re-exec the monitor executable +/// ([`LaunchContext::exec`]) as the hidden `__monitor` subcommand, in its own +/// process group, with this launch's socket and launch id on its argv. That +/// process runs [`run_supervised_stage`]: it spawns the agent (`program` + +/// `args`) in the agent's own process group, publishes the socket under +/// [`LaunchContext::cache_dir`], and advances the stage machine in-process +/// once the agent completes (D-02). Adapter env values (`envs`) travel on the +/// `__monitor` process's environment and only their names on its argv +/// (`--keep-env NAME`), so a value is never visible in the process table. /// -/// `launch` selects the supervision shape — see [`MonitorLaunch`]. `ctx` -/// names the monitor executable: the `PipeOwning` arm re-execs it as -/// [`LaunchContext::exec`] says, and the `Legacy` arm's `advance` tail runs -/// the same executable. +/// Returns only after the new supervisor answers a ping with this launch's +/// id, through [`wait_for_publish`]. A supervisor that exits first, or does +/// not publish within [`PUBLISH_WAIT`], is ended with what it started and +/// reported as [`MonitorError::NotReady`]. /// -/// Returns the PID of the spawned monitor. +/// The capture directory (and its `.gitignore`) is created before the spawn, +/// so it exists even when the launch fails. pub fn spawn_monitor( state: &State, program: &str, @@ -403,43 +422,11 @@ pub fn spawn_monitor( envs: &[(String, String)], launch: MonitorLaunch, ctx: &LaunchContext, -) -> Result { - spawn_monitor_inner( - state, - program, - args, - envs, - launch, - ctx, - MonitorTail::Advance, - ) -} - -/// What the Legacy monitor runs after it records the agent's exit code. -enum MonitorTail { - /// `devflow advance` for this phase and stage — every production launch. - Advance, - /// Test-only stand-in for `advance` (empty for none), so a test neither - /// re-execs the test binary as `devflow advance` nor loses the window in - /// which the monitor outlives its agent. - #[cfg(test)] - Script(String), -} - -fn spawn_monitor_inner( - state: &State, - program: &str, - args: &[String], - envs: &[(String, String)], - launch: MonitorLaunch, - ctx: &LaunchContext, - tail: MonitorTail, -) -> Result { +) -> Result { let project_root = state .project_root .to_str() .ok_or(MonitorError::NonUtf8Path)?; - let binary = ctx .exec .exe() @@ -453,274 +440,171 @@ fn spawn_monitor_inner( args.join(" ") ); - let stdout_file = crate::agent_result::stdout_path(&state.project_root, state.phase); - let stderr_file = crate::agent_result::stderr_path(&state.project_root, state.phase); - let exit_file = crate::agent_result::exit_code_path(&state.project_root, state.phase); - let pid_file = crate::agent_result::agent_pid_path(&state.project_root, state.phase); - // Ensure the capture directory exists before the detached process runs. + let stdout_file = crate::agent_result::stdout_path(&state.project_root, state.phase); if let Some(parent) = stdout_file.parent() { crate::workflow::ensure_devflow_dir(parent)?; } - let stdout_file = stdout_file.to_str().ok_or(MonitorError::NonUtf8Path)?; - let stderr_file = stderr_file.to_str().ok_or(MonitorError::NonUtf8Path)?; - let exit_file = exit_file.to_str().ok_or(MonitorError::NonUtf8Path)?; - let pid_file = pid_file.to_str().ok_or(MonitorError::NonUtf8Path)?; - - // The agent runs in its worktree when worktree mode is active; otherwise it - // runs in the project root. Capture/state files and the `devflow check` - // calls below always use the main project root, regardless of cwd. + // The agent runs in its worktree when worktree mode is active; otherwise + // it runs in the project root. Capture and state files always use the + // main project root, regardless of cwd. let workdir_path = state .worktree_path .as_deref() .unwrap_or(&state.project_root); let workdir = workdir_path.to_str().ok_or(MonitorError::NonUtf8Path)?; - if let MonitorLaunch::PipeOwning { prompt } = launch { - // `tail` is not consulted on this arm: the `__monitor` - // subcommand always advances after reaping, and `spawn_monitor` is the - // only caller of this function — it hardcodes `Advance`. Adding a - // `--no-advance` flag for a case nothing exercises would be an - // untested branch; add it when a caller actually needs it. - let _ = tail; - - // The adapter's extra env rides down by INHERITANCE here (set via - // `.envs(...)` on the `__monitor` process below), and that is only - // sufficient because the adapters routed through this arm — Claude - // and Antigravity (round-3) — declare no extra env at all - // (`codex_disables_signing_via_env_others_do_not` asserts this). - // Widening this arm to an adapter that DOES set env requires - // threading it explicitly to `run_pipe_owning_monitor`: the inner - // `hermetic_command` scrubs `GIT_CONFIG_COUNT`, which neutralises any - // inherited `GIT_CONFIG_KEY_n` pair (Codex's unsigned-commit - // override is exactly that shape). Loud rather than silent, and in - // the CLI process where an operator can actually see it. - if !envs.is_empty() { - warn!( - "pipe-owning monitor: {} adapter env var(s) will not survive the \ - inner hermetic_command scrub — thread them explicitly before \ - routing an env-setting adapter through this arm", - envs.len() - ); - } - - // D-04: resolve and clamp the idle timeout HERE, in the parent, and - // hand the monitor the already-resolved integer. - // - // The placement is the whole point. `spawn_monitor` runs inside - // `devflow start`, attached to the operator's terminal; the monitor is - // a detached process whose stdio is all `Stdio::null()`, so a warning - // logged there scrolls into nothing. A silent clamp is the exact - // failure class this project keeps paying for, so the notice goes to - // BOTH `tracing::warn!` and stdout — the log for the record, stdout - // for the human who is watching right now. - let idle = idle_timeout_setting_for(state.agent); - // The notice names the variable the operator actually set (D-08): the - // literal is deliberately spelled here so `doc_check` keeps BOTH - // variables visible to the operator-doc parity gate. - let idle_env = match state.agent { - AgentKind::Antigravity => "DEVFLOW_ANTIGRAVITY_IDLE_TIMEOUT_SECS", - _ => IDLE_TIMEOUT_ENV, - }; - if let Some(notice) = idle.notice_for(idle_env) { - warn!("{notice}"); - println!("{notice}"); + let launch_id = crate::supervisor::new_launch_id(); + let socket = + SocketPath::for_launch(&ctx.cache_dir, &state.project_root, state.phase, &launch_id)?; + let socket_str = socket.as_path().to_str().ok_or(MonitorError::NonUtf8Path)?; + + let stream = match launch { + MonitorLaunch::PipeOwning { prompt } => { + // D-04: resolve and clamp the idle timeout HERE, in the parent, + // and hand the monitor the already-resolved integer. The monitor's + // stdio is all `Stdio::null()`, so a warning logged there scrolls + // into nothing: the notice goes to BOTH `tracing::warn!` and + // stdout, for the human watching `devflow start` right now. + let idle = idle_timeout_setting_for(state.agent); + // The notice names the variable the operator actually set (D-08): + // the literal is deliberately spelled here so `doc_check` keeps + // BOTH variables visible to the operator-doc parity gate. + let idle_env = match state.agent { + AgentKind::Antigravity => "DEVFLOW_ANTIGRAVITY_IDLE_TIMEOUT_SECS", + _ => IDLE_TIMEOUT_ENV, + }; + if let Some(notice) = idle.notice_for(idle_env) { + warn!("{notice}"); + println!("{notice}"); + } + // The prompt travels as a FILE, not argv: argv has a hard length + // ceiling and DevFlow stage prompts routinely exceed it. + let prompt_file = crate::agent_result::prompt_path(&state.project_root, state.phase); + std::fs::write(&prompt_file, &prompt)?; + let prompt_file = prompt_file + .to_str() + .ok_or(MonitorError::NonUtf8Path)? + .to_string(); + Some((prompt_file, idle.timeout.as_secs())) } - - // The prompt travels as a FILE, not argv: argv has a hard length - // ceiling and DevFlow stage prompts routinely exceed what is safe to - // pass positionally. - let prompt_file = crate::agent_result::prompt_path(&state.project_root, state.phase); - std::fs::write(&prompt_file, &prompt)?; - let prompt_file = prompt_file.to_str().ok_or(MonitorError::NonUtf8Path)?; - - // Re-exec THIS binary as its hidden `__monitor` subcommand. The - // monitor must outlive `devflow start`/`advance`, so it has to be a - // distinct OS process; re-exec needs no daemonization primitive beyond - // `spawn()`-without-`wait()`, which is exactly what the `sh` monitor - // below already relies on. - // - // Ordering is load-bearing for the same reason the Legacy arm's - // comment gives: `hermetic_command` does its `env_remove`s at - // construction and `.envs(...)` runs after, so deliberate - // configuration survives while inherited pollution does not. - let mut monitor_argv = pipe_owning_monitor_args( - project_root, - state, - workdir, - prompt_file, - idle.timeout.as_secs(), - ); - monitor_argv.push(program.to_string()); - monitor_argv.extend(args.iter().cloned()); - let mut child_cmd = monitor_command(&ctx.exec, &binary, workdir_path, &monitor_argv)?; - child_cmd.envs(envs.iter().map(|(k, v)| (k.as_str(), v.as_str()))); - - strip_notify_env(&mut child_cmd); - - let child = child_cmd - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn()?; - - let pid = child.id(); - info!("pipe-owning monitor spawned with pid {pid}"); - return Ok(pid); - } - - // Shell script that launches the agent in the background, captures its - // stdout and exit code, then advances the workflow. Because this process - // is the agent's parent, capture survives the CLI exiting. - // - // stderr is captured to a separate file so it cannot corrupt the (possibly - // JSON) stdout capture that DevFlow parses for DEVFLOW_RESULT. Inspect - // .devflow/phase-NN-stderr.log for agent error output on failures. - // - // `devflow advance --phase N` evaluates the agent result, moves the stage - // machine forward, and (for an agent stage) spawns the next monitor - // itself. The phase is recorded here at spawn time so advance's identity - // never depends on a shared state singleton (13-DEFERRED-CR-03): under - // `devflow parallel`, each phase's monitor advances exactly its own - // stage machine. - // - // Traps SIGTERM and SIGINT for clean shutdown. WR-08 (13-REVIEW.md): - // the trap must also kill the backgrounded agent ($apid) — previously - // it only exited the monitor shell itself, orphaning the agent so it - // kept running/committing unsupervised with nothing left to call - // `devflow advance` once it finished. `apid` is initialized to empty - // before the trap is installed so a signal arriving before the agent is - // even backgrounded doesn't reference an unset variable. - // - // A single `kill` from the trap is NOT enough on its own - // (monitor-sigterm-orphans-agent, .planning/debug, 2026-09-22). Two - // windows lose it, and both orphaned the agent (PPid=1) in the tests - // that hold them open: - // - // - Window A (dash, i.e. Debian's /bin/sh and CI's): the forked agent - // child keeps the monitor's *caught* TERM handler until dash's - // post-fork trap reset. A TERM landing first is recorded, then - // discarded (the trap it would run is gone), and the child execs an - // agent that never saw it. bash does not lose it. - // - Window F (dash and bash): a TERM reaching the monitor after its fork - // but before `apid=$!` ran `cleanup` with an empty `apid`. - // - // So `cleanup` creates a stop marker BEFORE it kills, and the agent child - // checks that marker right before `exec`. The check runs after the - // child's trap reset, so a TERM it lost is always followed by a check - // that sees the marker — no timing assumption. `echo`, not `:`, creates - // the marker: a failed redirect on the special builtin `:` would exit the - // shell before the `kill`. `${apid:-$!}` closes window F, since `$!` is - // set as soon as the fork returns. The marker is removed in Rust before - // the spawn, never by the script, so the agent launch stays the monitor's - // first fork and a stale marker cannot stop the next agent. - // - // Once `wait` has reaped the agent, `cleanup` must leave it alone (WR-01, - // 48-REVIEW.md). The trap stays armed through the advance tail, and the - // shell defers it until that foreground command returns — after - // `advance` has launched the next stage's monitor, which removed the - // marker. Writing it then stops that next agent, and the `kill` targets a - // pid reaped long ago. `reaped` gates both; clearing `apid` would not, - // because `$!` still names the same pid. A TERM landing between `wait` - // returning and `reaped=1` still writes the marker, but the trap then - // exits before `advance` runs, so the next launch removes it as stale. - let stop_path = crate::agent_result::stop_marker_path(&state.project_root, state.phase); - if let Err(err) = std::fs::remove_file(&stop_path) - && err.kind() != std::io::ErrorKind::NotFound - { - return Err(err.into()); - } - let stop_file = stop_path.to_str().ok_or(MonitorError::NonUtf8Path)?; - let advance_tail = match tail { - MonitorTail::Advance => advance_tail(&binary, project_root, state), - #[cfg(test)] - MonitorTail::Script(script) if script.is_empty() => String::new(), - #[cfg(test)] - MonitorTail::Script(script) => format!("; {script}"), + MonitorLaunch::Legacy => None, }; - let script = format!( - "apid=''; reaped=''; cleanup() {{ if [ -z \"$reaped\" ]; then echo > {stop_file}; \ - [ -n \"${{apid:-$!}}\" ] && kill \"${{apid:-$!}}\" 2>/dev/null; fi; exit 0; }}; \ - trap cleanup TERM INT; \ - cd {workdir} || exit 1; \ - {{ [ -e {stop_file} ] && exit 143; exec \"$@\"; }} > {stdout_file} 2>{stderr_file} & \ - apid=$!; echo $apid > {pid_file}; \ - wait $apid; rc=$?; reaped=1; echo $rc > {exit_file}{advance_tail}", - stop_file = shell_escape(stop_file), - workdir = shell_escape(workdir), - stdout_file = shell_escape(stdout_file), - stderr_file = shell_escape(stderr_file), - exit_file = shell_escape(exit_file), - pid_file = shell_escape(pid_file), - ); - // 27-REVIEW WR-03: built through `hermetic_command`, not a bare - // `Command::new("sh")`. This is the spawn that launches the coding agent - // itself, and the comment below is precisely the hazard: whatever - // environment this `sh` carries rides down into the agent and into every - // git command the agent runs. An inherited `GIT_DIR` here would silently - // retarget the phase's real commits at a repository the operator never - // named — the worst case this phase exists to prevent, on its - // highest-consequence call site. - // - // Ordering is load-bearing: `hermetic_command` does its `env_remove`s at - // construction, and `.envs(...)` below runs after, so an adapter that - // deliberately sets one of these variables still wins. Deliberate - // configuration survives; inherited pollution does not. That is what - // keeps Codex's unsigned-commit override (`GIT_CONFIG_*`) working. - let mut child_cmd = hermetic_command("sh", workdir_path); - child_cmd - .arg("-c") - .arg(&script) - .arg("sh") - .arg(program) - .args(args) - // Adapter-scoped env (e.g. Codex's unsigned-commit override) rides - // the whole monitor chain: sh → agent → its git children (13-06). - .envs(envs.iter().map(|(k, v)| (k.as_str(), v.as_str()))); + let keep_env: Vec<&str> = envs.iter().map(|(name, _)| name.as_str()).collect(); + let mut monitor_argv = monitor_args(&MonitorArgs { + project_root, + state, + workdir, + socket: socket_str, + launch_id: &launch_id, + keep_env: &keep_env, + stream: stream + .as_ref() + .map(|(prompt_file, idle_secs)| (prompt_file.as_str(), *idle_secs)), + }); + monitor_argv.push(program.to_string()); + monitor_argv.extend(args.iter().cloned()); + // Ordering is load-bearing: `hermetic_command` does its `env_remove`s at + // construction and `.envs(...)` runs after, so deliberate adapter + // configuration survives while inherited pollution does not. The + // supervisor re-applies the names in `--keep-env` to the agent after its + // own `hermetic_command` scrub. + let mut child_cmd = monitor_command(&ctx.exec, &binary, workdir_path, &monitor_argv)?; + child_cmd.envs(envs.iter().map(|(k, v)| (k.as_str(), v.as_str()))); strip_notify_env(&mut child_cmd); - let child = child_cmd + // RESEARCH Finding 6: the supervisor leads its own process group, so it + // never sits in the operator's shell job and a terminal Ctrl-C to a + // lingering `devflow start` does not reach the unattended run. + let mut child = child_cmd .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()) + .process_group(0) .spawn()?; + let supervisor_pid = child.id(); + info!("monitor spawned with pid {supervisor_pid}; waiting for its socket"); - let pid = child.id(); - info!("monitor spawned with pid {pid}"); - Ok(pid) + let reply = wait_for_publish(&mut child, &socket, &launch_id, PUBLISH_WAIT)?; + info!( + "supervisor {supervisor_pid} published {} ({})", + socket.as_path().display(), + reply.launch + ); + Ok(Published { + record: crate::state::SupervisorRecord::new( + socket.as_path().to_path_buf(), + launch_id, + reply.agent_pgid, + reply.agent_start_time, + reply.boot_id.unwrap_or_default(), + ), + supervisor_pid, + }) } -fn pipe_owning_monitor_args( - project_root: &str, - state: &State, - workdir: &str, - prompt_file: &str, - idle_timeout_secs: u64, -) -> Vec { - vec![ - "__monitor".to_string(), - "--project".to_string(), - project_root.to_string(), - "--phase".to_string(), - state.phase.to_string(), - "--stage".to_string(), - state.stage.to_string(), - "--workdir".to_string(), - workdir.to_string(), - "--prompt-file".to_string(), - prompt_file.to_string(), - "--idle-timeout-secs".to_string(), - idle_timeout_secs.to_string(), - "--agent".to_string(), - state.agent.to_string(), - "--".to_string(), +/// The inputs [`monitor_args`] turns into a `__monitor` argv. +struct MonitorArgs<'a> { + project_root: &'a str, + state: &'a State, + workdir: &'a str, + socket: &'a str, + launch_id: &'a str, + /// The adapter env NAMES the supervisor re-applies to the agent. + keep_env: &'a [&'a str], + /// The stream shape's prompt file and idle timeout in seconds; `None` + /// for the capture shape. + stream: Option<(&'a str, u64)>, +} + +/// The `__monitor` argv up to and including `--`; the caller appends the +/// agent's program and argv. +fn monitor_args(args: &MonitorArgs<'_>) -> Vec { + let launch = if args.stream.is_some() { + "stream" + } else { + "capture" + }; + let mut argv: Vec = [ + "__monitor", + "--project", + args.project_root, + "--phase", + &args.state.phase.to_string(), + "--stage", + &args.state.stage.to_string(), + "--workdir", + args.workdir, + "--agent", + &args.state.agent.to_string(), + "--launch", + launch, + "--socket", + args.socket, + "--launch-id", + args.launch_id, ] + .iter() + .map(ToString::to_string) + .collect(); + for name in args.keep_env { + argv.push("--keep-env".to_string()); + argv.push((*name).to_string()); + } + if let Some((prompt_file, idle_secs)) = args.stream { + argv.push("--prompt-file".to_string()); + argv.push(prompt_file.to_string()); + argv.push("--idle-timeout-secs".to_string()); + argv.push(idle_secs.to_string()); + } + argv.push("--".to_string()); + argv } -/// The `PipeOwning` re-exec command for `exec`, before the adapter env is +/// The `__monitor` re-exec command for `exec`, before the adapter env is /// applied. `monitor_argv` starts with `__monitor` and ends with the child's /// program and argv; `binary` is `exec`'s executable as UTF-8. fn monitor_command( @@ -752,16 +636,6 @@ fn monitor_command( Ok(command) } -fn advance_tail(binary: &str, project_root: &str, state: &State) -> String { - format!( - "; {binary} advance {project_root} --phase {phase} --stage {stage}", - binary = shell_escape(binary), - project_root = shell_escape(project_root), - phase = state.phase, - stage = state.stage, - ) -} - /// Constraint 4's close rule as a pure, line-fed state machine: stdin may be /// released only once a `DEVFLOW_RESULT` marker has appeared inside a /// TOP-LEVEL `result` event **and** the background-task list has drained. @@ -1063,16 +937,26 @@ pub fn user_turn_line_for(agent: AgentKind, prompt: &str) -> String { } } -/// Supervise a `stream-json` child, owning both of its pipes, until the close -/// rule is satisfied and the child exits. Returns the child's exit code, which -/// is also written to the phase exit file. +/// The stream shape's writer and reader threads, joined once the agent has +/// been reaped. +struct StreamThreads { + writer: std::thread::JoinHandle<()>, + reader: std::thread::JoinHandle<()>, +} + +impl StreamThreads { + fn join(self) { + let _ = self.writer.join(); + let _ = self.reader.join(); + } +} + +/// Drive a `stream-json` child's two pipes until its stream is over. /// /// Agent-aware transport (round-3): the first turn is written via /// [`user_turn_line_for`] and the close rule via [`CloseRule::for_agent`], so /// the write/read/close triple matches whichever CLI is being supervised. /// -/// This runs INSIDE the detached `__monitor` process, not in the CLI. -/// /// Threading model (constraint 4 / T-31-04). Three participants: /// - a **writer thread** owning the child's stdin: it writes the initial user /// turn, then BLOCKS on a channel rather than returning. It drops stdin only @@ -1080,129 +964,20 @@ pub fn user_turn_line_for(agent: AgentKind, prompt: &str) -> String { /// is already gone — a task-notification turn arriving after the child's /// first turn would have nowhere to be delivered. /// - a **reader thread** owning the child's stdout: it tees each line verbatim -/// to the capture file and forwards it to the supervisor. Dropping its -/// sender at EOF is what surfaces `Disconnected` below. -/// - the **supervisor** (this function's own thread), which applies the close -/// rule and reaps. +/// to `capture` and forwards it to this thread. Dropping its sender at EOF +/// is what surfaces `Disconnected`. +/// - this thread, which applies the close rule and the idle timeout. /// /// The write and the read MUST be on independent threads. Writing the prompt /// synchronously before reading stdout is the textbook two-pipe deadlock: it -/// passes every short-prompt smoke test and hangs on exactly the context-heavy -/// production stages that matter (the Linux pipe buffer is commonly 64KiB and -/// a DevFlow stage prompt can exceed that in one write). -#[allow(clippy::too_many_arguments)] -pub fn run_pipe_owning_monitor( - project_root: &Path, - phase: PhaseId, - workdir: &Path, - prompt: &str, - idle_timeout: Duration, - program: &str, - args: &[String], - envs: &[(String, String)], - agent: AgentKind, -) -> Result { - let stdout_file = crate::agent_result::stdout_path(project_root, phase); - let stderr_file = crate::agent_result::stderr_path(project_root, phase); - let exit_file = crate::agent_result::exit_code_path(project_root, phase); - let pid_file = crate::agent_result::agent_pid_path(project_root, phase); - if let Some(parent) = stdout_file.parent() { - crate::workflow::ensure_devflow_dir(parent)?; - } - - // stderr goes to its own file so it cannot corrupt the JSONL stdout - // capture DevFlow parses — the same separation the Legacy script's - // `2>{stderr_file}` provides. - let stderr_handle = std::fs::File::create(&stderr_file)?; - // One handle, opened once, truncating at open and appending line by line. - // Truncate-at-open reproduces the Legacy arm's `>` redirection exactly, so - // a capture from a previous attempt can never be mixed into this one's - // (the launch path archives the prior capture first, but relying on that - // to make an append-mode open safe would be an unstated coupling). - let capture = std::fs::File::create(&stdout_file)?; - - let mut agent_cmd = hermetic_command(program, workdir); - // The coding agent never needs the gate notify command (WR-07). - strip_notify_env(&mut agent_cmd); - let mut child = agent_cmd - .args(args) - .envs(envs.iter().map(|(k, v)| (k.as_str(), v.as_str()))) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::from(stderr_handle)) - // T-31-05: make the child its own process-group leader so a later - // group signal cannot reach this monitor's own ancestors. Verified - // source shows the pre-31 `spawn_monitor` had NO session or group - // configuration at all — detachment came only from the parent not - // waiting — so this closes a gap rather than preserving one. - // Full `setsid()` session detachment is deliberately NOT done: no - // forensics record cites a SIGHUP-related monitor loss, so there is - // no evidence it buys anything. `pre_exec` calling `libc::setsid()` - // is the one-line follow-on if such a loss ever surfaces. - .process_group(0) - .spawn()?; - - // Recorded immediately, before any pipe work: `wait_for_agent_pid` polls - // for this and the rest of DevFlow's liveness reporting depends on it. - let child_pid = child.id(); - std::fs::write(&pid_file, format!("{child_pid}\n"))?; - - let threads = drive_stream_io(&mut child, capture, agent, prompt, idle_timeout, || { - fire_idle_timeout(project_root, phase, workdir, child_pid, idle_timeout); - })?; - - let status = child.wait()?; - // A signal-killed child has NO exit code — `status.code()` is `None`, and - // the previous `unwrap_or(-1)` threw the signal away (peer review - // 2026-08-03, found independently by both reviewers and by the 31-04 plan - // review as W1). That silently defeated the classification 31-04 took care - // to preserve: `evaluate_layer2` and - // `reconcile_stream_success_against_exit_code` map **137** to - // `ResourceKilled` (routed to `GateInfra` — an infrastructure fault) and - // **127** to `AgentUnavailable`. Recording `-1` matched neither, so a real - // OOM kill arrived as a generic `Failed` and routed to `GateReview`, asking - // an operator to code-review a stage that was killed by the kernel. - // - // `128 + signal` is the shell convention those constants already encode: - // SIGKILL(9) -> 137, SIGTERM(15) -> 143. `-1` is now reachable only when a - // status is neither exited nor signalled, which POSIX does not define. - let code = status.code().unwrap_or_else(|| { - use std::os::unix::process::ExitStatusExt; - status.signal().map_or(-1, |signal| 128 + signal) - }); - std::fs::write(&exit_file, format!("{code}\n"))?; - - threads.join(); - - info!("supervised child {child_pid} exited with code {code}"); - Ok(code) -} - -/// The stream shape's writer and reader threads, joined once the agent has -/// been reaped. -struct StreamThreads { - writer: std::thread::JoinHandle<()>, - reader: std::thread::JoinHandle<()>, -} - -impl StreamThreads { - fn join(self) { - let _ = self.writer.join(); - let _ = self.reader.join(); - } -} - -/// Drive a `stream-json` child's two pipes until its stream is over, with -/// the threading model [`run_pipe_owning_monitor`]'s doc describes: a writer -/// thread that delivers the first turn and holds stdin open, a reader thread -/// that tees stdout into `capture`, and this thread applying the close rule -/// and the idle timeout. +/// passes every short-prompt smoke test and hangs on exactly the +/// context-heavy production stages that matter (the Linux pipe buffer is +/// commonly 64KiB and a DevFlow stage prompt can exceed that in one write). /// /// Returns once stdout reaches EOF, the idle timeout has fired (after /// `on_idle_timeout` ran), or the stream stayed quiet after the close rule /// released stdin. Stdin is released before it returns, so the caller may -/// then wait for the child. Shared by [`run_pipe_owning_monitor`] and -/// [`run_supervised_stage`]'s stream shape, so the two cannot drift apart. +/// then wait for the child. Used by [`run_supervised_stage`]'s stream shape. fn drive_stream_io( child: &mut std::process::Child, mut capture: std::fs::File, @@ -1502,9 +1277,9 @@ const PGID_SETTLE_WAIT: Duration = Duration::from_millis(100); /// sidecar exists before the socket name appears (R-7), and it is the /// launcher's only record of the agent if this process is SIGKILLed /// before it publishes (RV-03; `serve` rewrites it with the same content). -/// 4. The stream shape only: drive the agent's pipes through the same -/// helper `run_pipe_owning_monitor` uses (first turn, capture tee, close -/// rule, idle timeout). An idle-timeout group kill is a natural outcome, +/// 4. The stream shape only: drive the agent's pipes through +/// `drive_stream_io` (first turn, capture tee, close rule, idle +/// timeout). An idle-timeout group kill is a natural outcome, /// `Completed(128 + n)` with the idle-timeout verdict file, never a stop. /// 5. [`Supervisor::wait_agent`], then the exit file: the agent's code for /// [`AgentOutcome::Completed`], `143` for [`AgentOutcome::Stopped`]. @@ -1774,34 +1549,18 @@ fn stage_sidecar(spec: &StageSpec, agent_pid: u32) -> crate::supervisor::Sidecar /// How long a launcher waits for a new supervisor to answer a ping with its /// launch id before it ends what it started (50-24). -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] const PUBLISH_WAIT: Duration = Duration::from_secs(10); /// How often [`wait_for_publish`] polls the child and probes the socket. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] const PUBLISH_POLL: Duration = Duration::from_millis(20); /// How long the not-ready teardown waits for its SIGSTOP to show in /// `/proc` before it lists the child's children. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] const STOP_SETTLE_WAIT: Duration = Duration::from_millis(100); /// The `/proc//stat` fields the not-ready teardown picks its targets /// by, from one read. 50-09's `agent::ProcStat` carries no parent pid, so /// this is a separate, private parse. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct ChildStat { /// Field 4, the parent pid. @@ -1818,10 +1577,6 @@ struct ChildStat { /// `)`: index 0 is then field 3, so the parent pid (field 4) is index 1, /// the process group (field 5) index 2 and the start time (field 22) /// index 19. A missing or unparsable field is `None`, never a guess. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn parse_proc_stat_line(line: &str) -> Option { let rest = &line[line.rfind(')')? + 1..]; let fields: Vec<&str> = rest.split_whitespace().collect(); @@ -1834,20 +1589,12 @@ fn parse_proc_stat_line(line: &str) -> Option { /// [`ChildStat`] for `pid` from one read of `/proc//stat`, or `None` /// when it cannot be read or parsed. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn read_child_stat(pid: u32) -> Option { parse_proc_stat_line(&std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?) } /// Every process whose parent pid is `parent`, each with the stat it was /// chosen by. An entry that cannot be read is skipped. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn direct_children(parent: u32) -> Vec<(u32, ChildStat)> { proc_table() .into_iter() @@ -1857,10 +1604,6 @@ fn direct_children(parent: u32) -> Vec<(u32, ChildStat)> { /// Every readable `/proc//stat`, each from one read. An unlistable /// `/proc` is an empty table (logged); an unreadable entry is skipped. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn proc_table() -> Vec<(u32, ChildStat)> { let Ok(entries) = std::fs::read_dir("/proc") else { warn!("could not list /proc"); @@ -1900,10 +1643,6 @@ fn signal_guarded_group(pgid: u32, signal: libc::c_int) -> bool { /// Send `signal` to `child` alone, by the pid of the `Child` this process /// spawned and has not reaped (so the pid cannot name another process). -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn signal_own_child(child: &std::process::Child, signal: libc::c_int) -> bool { match libc::pid_t::try_from(child.id()) { // SAFETY: `pid > 1` names one unreaped child of this process. @@ -1913,10 +1652,6 @@ fn signal_own_child(child: &std::process::Child, signal: libc::c_int) -> bool { } /// Poll `child` until it exits (reaping it) or `wait` elapses. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn exit_within( child: &mut std::process::Child, wait: Duration, @@ -1946,10 +1681,6 @@ fn exit_within( /// 4. SIGKILL the group of each listed child that leads its own group /// (R-3's agent shape) and still has the listed start time. A bare kill /// of the child's group would orphan such an agent. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn end_unpublished_child(child: &mut std::process::Child, wait: Duration) -> String { signal_own_child(child, libc::SIGTERM); if let Some(status) = exit_within(child, 2 * crate::agent::TERMINATE_VERIFY_WAIT) { @@ -1998,10 +1729,6 @@ fn end_unpublished_child(child: &mut std::process::Child, wait: Duration) -> Str /// launch's address any more (its only server was the reaped child), so its /// launch files are removed, and the result is [`MonitorError::NotReady`] /// naming the child, how it ended and how many child groups were ended. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn wait_for_publish( child: &mut std::process::Child, socket: &SocketPath, @@ -2050,18 +1777,10 @@ fn wait_for_publish( /// How long the ending step waits for a SIGKILLed recorded agent group to /// empty. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] const RECORDED_GROUP_GONE_WAIT: Duration = Duration::from_secs(2); /// What the ending step did about the agent group a launch's sidecar /// records. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] struct RecordedGroupEnd { /// The sentence the `NotReady` message carries. text: String, @@ -2085,10 +1804,6 @@ struct RecordedGroupEnd { /// [`agent::is_same_process`](crate::agent::is_same_process) re-confirms /// the leader immediately before the SIGKILL. No sidecar, another launch /// id or any other decision signals nothing, and the text says which. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn end_recorded_agent_group(socket: &SocketPath, launch_id: &str) -> RecordedGroupEnd { use crate::supervisor::backstop::BackstopDecision; let done = |text: String| RecordedGroupEnd { @@ -2164,10 +1879,6 @@ fn end_recorded_agent_group(socket: &SocketPath, launch_id: &str) -> RecordedGro } /// The variant name of a backstop decision, for the `NotReady` text. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn decision_name(decision: &crate::supervisor::backstop::BackstopDecision) -> &'static str { use crate::supervisor::backstop::BackstopDecision; match decision { @@ -2183,10 +1894,6 @@ fn decision_name(decision: &crate::supervisor::backstop::BackstopDecision) -> &' } /// How many processes `/proc` lists in group `pgid` right now. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn group_member_count(pgid: u32) -> usize { proc_table() .iter() @@ -2197,10 +1904,6 @@ fn group_member_count(pgid: u32) -> usize { /// Poll until no process is left in group `pgid` (`kill(-pgid, 0)` fails /// with ESRCH), up to `wait`. A pgid that is not a safe group target never /// counts as empty. -#[cfg_attr( - not(test), - expect(dead_code, reason = "spawn_monitor calls it from 50-11") -)] fn group_empties_within(pgid: u32, wait: Duration) -> bool { let Ok(signed) = libc::pid_t::try_from(pgid) else { return false; @@ -2386,9 +2089,9 @@ fn enumerate_phase_commits( // `workdir`, which is a different bug and left this one open. // // The state file is the same source of truth `git_flow_for_run` reads; - // consulting it here rather than threading a base through - // `run_pipe_owning_monitor`'s public signature keeps the fix local to - // the site that needs it. A missing or unreadable state degrades to the + // consulting it here rather than threading a base through the + // supervisor's `StageSpec` keeps the fix local to the site that needs + // it. A missing or unreadable state degrades to the // ambient resolution — the pre-existing behaviour, never an abort, // because losing the verdict is worse than losing the commit names. let persisted_base = crate::workflow::load_state(project_root, phase) @@ -2571,11 +2274,6 @@ pub fn wait_for_agent_pid(project_root: &Path, phase: PhaseId) -> Option { None } -/// Escape a string for safe use in a single-quoted shell context. -fn shell_escape(s: &str) -> String { - format!("'{}'", s.replace('\'', "'\\''")) -} - #[cfg(test)] mod tests { use super::*; @@ -2583,29 +2281,76 @@ mod tests { use crate::stage::Stage; use crate::state::{AgentKind, State}; + /// The `__monitor` argv carries the stage, the launch shape, the socket and + /// launch id, and adapter env NAMES only (T-50-41): values travel on the + /// process environment, never on an argv every local user can read. The + /// stream-only flags appear only for the stream shape. #[test] - fn legacy_advance_tail_passes_the_launched_stage() { + fn monitor_args_carry_the_launch_and_only_adapter_env_names() { let dir = tempfile::tempdir().unwrap(); - let mut state = state_in(dir.path()); - state.stage = Stage::Code; - let root = dir.path().to_str().unwrap(); - - let tail = advance_tail("devflow", root, &state); - assert!(tail.contains(" advance ")); - assert!(tail.contains(&format!("--phase {} --stage code", state.phase))); - } + let state = state_in(dir.path()); + let keep_env = ["GIT_CONFIG_COUNT", "GIT_CONFIG_KEY_0"]; + let args = |stream| { + monitor_args(&MonitorArgs { + project_root: "/project", + state: &state, + workdir: "/workdir", + socket: "/cache/sock/x.sock", + launch_id: "0123456789abcdef", + keep_env: &keep_env, + stream, + }) + }; + let value_after = |argv: &[String], flag: &str| { + let at = argv.iter().position(|arg| arg == flag); + at.map(|at| argv[at + 1].clone()) + }; - #[test] - fn pipe_owning_monitor_args_pass_the_stage() { - let dir = tempfile::tempdir().unwrap(); - let mut state = state_in(dir.path()); - state.stage = Stage::Code; - let args = pipe_owning_monitor_args("/project", &state, "/workdir", "/prompt", 60); - let stage = args.iter().position(|arg| arg == "--stage").unwrap(); - let separator = args.iter().position(|arg| arg == "--").unwrap(); + let stream = args(Some(("/prompt", 60))); + let separator = stream.iter().position(|arg| arg == "--").unwrap(); + assert_eq!( + separator, + stream.len() - 1, + "`--` ends the monitor's own argv" + ); + assert_eq!(stream[0], "__monitor"); + assert_eq!(value_after(&stream, "--stage").as_deref(), Some("code")); + assert_eq!(value_after(&stream, "--launch").as_deref(), Some("stream")); + assert_eq!( + value_after(&stream, "--socket").as_deref(), + Some("/cache/sock/x.sock") + ); + assert_eq!( + value_after(&stream, "--launch-id").as_deref(), + Some("0123456789abcdef") + ); + assert_eq!( + value_after(&stream, "--prompt-file").as_deref(), + Some("/prompt") + ); + assert_eq!( + value_after(&stream, "--idle-timeout-secs").as_deref(), + Some("60") + ); + let kept: Vec<&str> = stream + .windows(2) + .filter(|pair| pair[0] == "--keep-env") + .map(|pair| pair[1].as_str()) + .collect(); + assert_eq!(kept, keep_env, "one --keep-env per adapter env name"); + assert!( + !stream.iter().any(|arg| arg.contains('=')), + "no NAME=VALUE pair may reach the argv: {stream:?}" + ); - assert_eq!(args[stage + 1], "code"); - assert!(stage < separator, "stage must be parsed before child argv"); + // Control: the capture shape names itself and carries no stream flags. + let capture = args(None); + assert_eq!( + value_after(&capture, "--launch").as_deref(), + Some("capture") + ); + assert_eq!(value_after(&capture, "--prompt-file"), None); + assert_eq!(value_after(&capture, "--idle-timeout-secs"), None); } /// WR-07 (49-REVIEW.md): the chain's command never carries the variable. @@ -2637,19 +2382,6 @@ mod tests { state } - /// The launch context every pre-50-10 `spawn_monitor` test passes: this - /// test binary as the monitor executable, exactly what `current_exe()` - /// resolved to inside `spawn_monitor` before 50-10, and a cache dir under - /// the test's own project tempdir. - fn test_launch_context(state: &State) -> LaunchContext { - LaunchContext::new( - state.project_root.join("cache"), - MonitorExec::Subcommand { - exe: std::env::current_exe().unwrap(), - }, - ) - } - /// T-50-26's source scan: whether the first `HarnessEntry {` after /// `enum MonitorExec` in `source` carries `cfg(feature = "test-support")` /// on its own line or on one of the two lines above it. The source is cut @@ -3274,13 +3006,6 @@ mod tests { ); } - #[test] - fn shell_escape_wraps_basic_strings() { - assert_eq!(shell_escape("hello"), "'hello'"); - assert_eq!(shell_escape("hello world"), "'hello world'"); - assert_eq!(shell_escape("/tmp/devflow"), "'/tmp/devflow'"); - } - /// The Phase 31 tracer: ONE Claude-shaped stage driven end to end through /// the pipe-owning supervisor. /// @@ -3365,7 +3090,7 @@ exit 0 early = early_file.display(), ); - let code = run_pipe_owning_monitor( + let code = supervise_stream_child( root, phase, root, @@ -3473,7 +3198,7 @@ exit 0 let phase = PhaseId::new(51); std::fs::create_dir_all(root.join(".devflow")).unwrap(); - let code = run_pipe_owning_monitor( + let code = supervise_stream_child( root, phase, root, @@ -3527,7 +3252,7 @@ exit 0 "# ); - let _ = run_pipe_owning_monitor( + let _ = supervise_stream_child( root, phase, root, @@ -3557,7 +3282,7 @@ exit 0 let phase = PhaseId::new(52); std::fs::create_dir_all(root.join(".devflow")).unwrap(); - let _ = run_pipe_owning_monitor( + let _ = supervise_stream_child( root, phase, root, @@ -3604,7 +3329,7 @@ printf '%s\n' '{"type":"result","subtype":"success","is_error":false,"session_id exit 0 "#; - let code = run_pipe_owning_monitor( + let code = supervise_stream_child( root, phase, root, @@ -3669,7 +3394,7 @@ sleep 3 exit 0 "#; - let code = run_pipe_owning_monitor( + let code = supervise_stream_child( root, phase, root, @@ -3725,7 +3450,7 @@ printf '%s\n' '{"type":"system","subtype":"init","session_id":"sig-1"}' kill -9 $$ "#; - let code = run_pipe_owning_monitor( + let code = supervise_stream_child( root, phase, root, @@ -3749,17 +3474,6 @@ kill -9 $$ assert_eq!(exit.trim(), "137", "exit file contents: {exit:?}"); } - #[test] - fn shell_escape_handles_single_quotes() { - assert_eq!(shell_escape("can't"), "'can'\\''t'"); - assert_eq!(shell_escape("a'b'c"), "'a'\\''b'\\''c'"); - } - - #[test] - fn shell_escape_handles_empty_string() { - assert_eq!(shell_escape(""), "''"); - } - #[test] fn wait_for_agent_pid_returns_pid_when_file_exists() { let dir = tempfile::tempdir().unwrap(); @@ -3793,53 +3507,6 @@ kill -9 $$ assert_eq!(wait_for_agent_pid(dir.path(), PhaseId::new(4)), None); } - #[test] - fn spawn_monitor_captures_agent_pid_and_output() { - let dir = tempfile::tempdir().unwrap(); - let state = state_in(dir.path()); - // Stub agent: write a known marker to stdout, then exit cleanly. - let args = vec!["-c".to_string(), "echo MONITOR_READY".to_string()]; - - let monitor_pid = spawn_monitor( - &state, - "sh", - &args, - &[], - MonitorLaunch::Legacy, - &test_launch_context(&state), - ) - .unwrap(); - assert!(monitor_pid > 0); - - // Observable side effect #1: the monitor records the agent PID to its - // pid file with valid numeric content. - let agent_pid = wait_for_agent_pid(dir.path(), state.phase) - .expect("monitor should record the agent pid"); - assert!(agent_pid > 0); - - // Observable side effect #2: the agent's stdout is captured to the - // phase stdout file (proving the monitor actually ran the agent). - let stdout_path = crate::agent_result::stdout_path(dir.path(), state.phase); - let mut captured = String::new(); - for _ in 0..100 { - if let Ok(contents) = std::fs::read_to_string(&stdout_path) - && contents.contains("MONITOR_READY") - { - captured = contents; - break; - } - std::thread::sleep(Duration::from_millis(20)); - } - assert!( - captured.contains("MONITOR_READY"), - "expected MONITOR_READY in captured stdout, got {captured:?}" - ); - } - - /// WR-08 (13-REVIEW.md): sending SIGTERM/SIGINT to the monitor must also - /// terminate the agent it owns. Before the fix, `cleanup()` only exited - /// the monitor shell, leaving the agent orphaned and running/committing - /// unsupervised with nothing left to call `devflow advance` for it. /// A one-line identity/state summary of a pid, for failure diagnostics. /// `Name`/`State`/`PPid` come from `/proc//status`; the cmdline /// distinguishes a shell that exec'd its command from one that forked it. @@ -3878,374 +3545,85 @@ kill -9 $$ ) } - #[test] - fn sigterm_to_monitor_also_kills_the_agent() { - let dir = tempfile::tempdir().unwrap(); - let state = state_in(dir.path()); - // Stub agent that runs long enough to observe: sleeps well past the - // window this test needs to send SIGTERM and check liveness. - let args = vec!["-c".to_string(), "sleep 30".to_string()]; - - let monitor_pid = spawn_monitor( - &state, - "sh", - &args, - &[], - MonitorLaunch::Legacy, - &test_launch_context(&state), - ) - .unwrap(); - let agent_pid = wait_for_agent_pid(dir.path(), state.phase) - .expect("monitor should record the agent pid"); - assert!( - crate::agent::agent_running(agent_pid), - "agent should be running before SIGTERM" - ); - - // Snapshot both processes before signalling. This assertion fails in - // containerised CI and cannot be reproduced locally, and a bare - // "still running" message discards everything that could explain it - // — the same antipattern that made 999.47 expensive to diagnose. - let monitor_before = proc_snapshot(monitor_pid); - let agent_before = proc_snapshot(agent_pid); - - // SIGTERM the monitor, as an operator (or lock.rs's stale-holder - // reclaim path) would to abort a run. - let kill_rc = unsafe { libc::kill(monitor_pid as libc::pid_t, libc::SIGTERM) }; - let kill_err = if kill_rc == 0 { - "ok".to_string() - } else { - format!("errno {}", std::io::Error::last_os_error()) - }; - - // The agent should be killed promptly by the monitor's trap — - // poll rather than sleep a fixed amount to keep this fast and - // avoid flaking under load. (Window widened to 5s: at 2s this - // still flaked under a fully parallel workspace test run.) - // - // 2026-07-26: this was widened 5s -> 15s for the containerised CI - // job and STILL failed, then reverted to 5s. That widening was a - // mistake: 15s is far beyond any plausible trap-and-kill latency, - // so the agent is not being reaped SLOWLY, it is not being reaped. - // Buying silence with a bigger number would have hidden a real - // defect behind a green check — the exact false negative this - // repository keeps getting bitten by. - // - // 2026-09-22 diagnosis (monitor-sigterm-orphans-agent, - // .planning/debug): under dash (the container's /bin/sh; the Fedora - // host's is bash), a TERM that lands in the agent child after its - // fork but before dash's post-fork trap reset is caught by the - // inherited handler and then discarded, and the child execs an agent - // that never saw it. The CI failures' `agent before` snapshot shows - // the child still pre-exec, consistent with that window; this test - // only races it. The `sigterm_before_*` tests below hold it (and the - // monitor-side `apid=$!` window) open deterministically; the Legacy - // script's stop marker closes the first and `${apid:-$!}` the second. - // - // Do NOT widen this window: a failure here is a lost TERM, not a slow - // one. - let mut still_running = true; - for _ in 0..250 { - if !crate::agent::agent_running(agent_pid) { - still_running = false; - break; - } - std::thread::sleep(Duration::from_millis(20)); - } - let monitor_after = proc_snapshot(monitor_pid); - let agent_after = proc_snapshot(agent_pid); - let pidfile = - std::fs::read_to_string(crate::agent_result::agent_pid_path(dir.path(), state.phase)) - .unwrap_or_else(|e| format!("")); - - assert!( - !still_running, - "agent (pid {agent_pid}) was orphaned — still running after monitor SIGTERM\n\ - \x20 monitor pid: {monitor_pid}\n\ - \x20 kill(TERM) rc: {kill_rc} ({kill_err})\n\ - \x20 monitor before: {monitor_before}\n\ - \x20 monitor after: {monitor_after}\n\ - \x20 agent pid: {agent_pid}\n\ - \x20 agent before: {agent_before}\n\ - \x20 agent after: {agent_after}\n\ - \x20 pidfile contents: {}\n\ - Read the monitor's `after` line first. GONE means the shell died \ - without running its trap — most likely SIGTERM arrived before \ - `trap` was installed, or it was killed rather than handling the \ - signal, either way leaving the agent unreaped. STILL ALIVE means \ - the trap never fired or `kill $apid` failed, so compare the agent \ - pid against the pidfile and check the agent's PPid: if PPid is not \ - the monitor, `$!` did not name the process we are polling. If the \ - agent's Name is `sh` rather than `sleep`, the agent shell forked \ - rather than exec'd, so killing it leaves its own child behind.", - pidfile.trim() - ); - } - - // ---- monitor-sigterm-orphans-agent (.planning/debug, 2026-09-22): a TERM - // that lands inside one of the Legacy monitor script's spawn windows. - // - // Window A (dash only; Debian's /bin/sh, so CI's): the forked agent child - // keeps the monitor's *caught* TERM handler until dash's fork-reset clears - // the trap. A TERM that lands first is recorded by that handler and then - // discarded (`dotrap` skips a signal whose trap is now NULL), and the child - // goes on to exec the agent, which never sees a TERM and is orphaned when - // the monitor exits. bash does not lose it, which is why the Fedora host - // never reproduced `sigterm_to_monitor_also_kills_the_agent`'s CI failures. - // - // Window F (dash and bash): a TERM that reaches the monitor after its fork - // but before `apid=$!` runs `cleanup` with an empty `apid`, so it kills - // nothing. - // - // Both windows are microseconds wide, so these tests hold them open - // deterministically instead of racing them. An LD_PRELOAD `fork()` shim, - // compiled here with `cc` and handed to the monitor through - // `spawn_monitor`'s own `envs`, holds ONE side of the monitor's first fork - // (the agent launch) until the test creates a release file. The shim - // blocks every signal for the hold, so a TERM sent inside it stays pending - // (kernel-visible in /proc) until the hold ends, and each test asserts that - // the TERM really landed inside the window instead of assuming it. - - /// The `fork()` interposer. Only the first fork, on the configured side, - /// in a process carrying these variables holds: that side unlinks the - /// `HOLDFORK_ONCE` file, and every later fork (the exec'd agent inherits - /// `LD_PRELOAD`) finds it gone. - #[cfg(target_os = "linux")] - const HOLDFORK_SHIM_C: &str = r#" -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include - -static void record_pending_term(const char *path) { - sigset_t pending; - if (sigpending(&pending) == 0 && sigismember(&pending, SIGTERM)) { - int fd = open(path, O_WRONLY | O_CREAT, 0600); - if (fd >= 0) close(fd); - } -} - -pid_t fork(void) { - pid_t (*real_fork)(void) = (pid_t (*)(void))dlsym(RTLD_NEXT, "fork"); - pid_t pid = real_fork(); - if (pid < 0) return pid; - int saved_errno = errno; - const char *side = getenv("HOLDFORK_SIDE"); - const char *once = getenv("HOLDFORK_ONCE"); - const char *release = getenv("HOLDFORK_RELEASE"); - const char *term_seen = getenv("HOLDFORK_TERM_SEEN"); - if (side && once && release && term_seen - && strcmp(side, pid == 0 ? "child" : "parent") == 0 - && unlink(once) == 0) { - /* Every signal is blocked for the hold, so one sent inside it stays - pending instead of running a handler whose only trace would be a - cut-short sleep, which a process that is descheduled when the - signal lands never sees. A pending TERM is recorded each tick and - once more at release; restoring the mask then delivers it to the - handler this process already had, still before any of the shell's - own post-fork code. Hold until released: 10 ms ticks, capped at - 20 s so a crashed test cannot wedge a process forever. */ - sigset_t all, saved; - sigfillset(&all); - sigprocmask(SIG_BLOCK, &all, &saved); - for (int tick = 0; tick < 2000 && access(release, F_OK) != 0; tick++) { - struct timespec step = {0, 10 * 1000 * 1000}; - nanosleep(&step, NULL); - record_pending_term(term_seen); - } - record_pending_term(term_seen); - sigprocmask(SIG_SETMASK, &saved, NULL); - } - errno = saved_errno; - return pid; -} -"#; - - /// Which side of the monitor's first `fork()` the shim holds. - #[cfg(target_os = "linux")] - #[derive(Clone, Copy)] - enum HoldSide { - /// The agent child, before any of the shell's post-fork code runs. - Child, - /// The monitor, after the fork returned and before `apid=$!`. - Parent, - } - - /// A Legacy monitor whose first fork is held open by the shim. Dropping it - /// releases the hold and SIGKILLs every still-tracked process that is - /// still recognisably ours, so a failing assertion leaks neither a held - /// monitor nor a 30-second agent. - #[cfg(target_os = "linux")] - struct HeldMonitor { - state: State, - monitor_pid: u32, - release: std::path::PathBuf, - term_seen: std::path::PathBuf, - tracked: Vec, - } - - #[cfg(target_os = "linux")] - impl HeldMonitor { - fn spawn(root: &Path, side: HoldSide) -> Self { - let shim = compile_holdfork_shim(root); - let once = root.join("holdfork-once"); - std::fs::write(&once, "").unwrap(); - let release = root.join("holdfork-release"); - let term_seen = root.join("holdfork-term-seen"); - let side = match side { - HoldSide::Child => "child", - HoldSide::Parent => "parent", - }; - let envs = [ - ("LD_PRELOAD", shim.to_str().unwrap()), - ("HOLDFORK_SIDE", side), - ("HOLDFORK_ONCE", once.to_str().unwrap()), - ("HOLDFORK_RELEASE", release.to_str().unwrap()), - ("HOLDFORK_TERM_SEEN", term_seen.to_str().unwrap()), - ] - .map(|(key, value)| (key.to_string(), value.to_string())); - let state = state_in(root); - let args = vec!["-c".to_string(), "sleep 30".to_string()]; - let monitor_pid = spawn_monitor( - &state, - "sh", - &args, - &envs, - MonitorLaunch::Legacy, - &test_launch_context(&state), - ) - .unwrap(); - Self { - state, - monitor_pid, - release, - term_seen, - tracked: vec![monitor_pid], - } - } - - fn release(&self) { - std::fs::write(&self.release, "").unwrap(); - } - - /// Whether the shim saw a TERM pending during its hold. - fn hold_saw_a_pending_term(&self) -> bool { - self.term_seen.exists() - } - - fn sigterm_monitor(&self) { - let rc = unsafe { libc::kill(self.monitor_pid as libc::pid_t, libc::SIGTERM) }; - assert_eq!( - rc, - 0, - "kill(monitor, TERM): {}", - std::io::Error::last_os_error() - ); - } - - /// Reap the monitor (this test process is its parent) and require that - /// it left through its trap: `cleanup` ends in `exit 0`, whereas a - /// shell killed by the TERM itself reports a signal. Once reaped, its - /// pid may be reused, so it is no longer tracked. - fn reap_monitor_expecting_trap_exit(&mut self) { - let pid = self.monitor_pid as libc::pid_t; - let mut reaped = None; - for _ in 0..250 { - let mut status: libc::c_int = 0; - let rc = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; - if rc == pid { - reaped = Some(status); - break; - } - assert_eq!( - rc, - 0, - "waitpid(monitor): {}", - std::io::Error::last_os_error() - ); - std::thread::sleep(Duration::from_millis(20)); - } - let status = reaped.expect("the monitor did not exit within 5s of SIGTERM"); - self.tracked.retain(|&tracked| tracked != self.monitor_pid); - assert!( - libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, - "the monitor must leave through its trap (`cleanup` → `exit 0`); \ - wait status {status:#x}" - ); - } - } - - #[cfg(target_os = "linux")] - impl Drop for HeldMonitor { - fn drop(&mut self) { - let _ = std::fs::write(&self.release, ""); - for &pid in &self.tracked { - let cmd = cmdline_of(pid); - let ours = cmd.contains("trap cleanup TERM INT") || cmd.contains("sleep 30"); - if ours && crate::agent::agent_running(pid) { - unsafe { libc::kill(pid as libc::pid_t, libc::SIGKILL) }; - } - } + /// The stream shape's pipe driving without the published socket: spawn + /// `program` through `spawn_stage_agent` (its own process group, the + /// adapter env, the truncating capture and stderr files), record its pid, + /// drive it through `drive_stream_io` with `fire_idle_timeout` as the idle + /// action, reap it, and record `runtime::exit_code` in the exit file. + /// That is the sequence `run_supervised_stage` runs for its stream shape, + /// minus the signal pipe and the socket, which need a child test process + /// (the `*_arm_runs_the_agent_under_an_addressed_supervisor` tests cover + /// those). Returns the recorded exit code. + #[allow( + clippy::too_many_arguments, + reason = "mirrors the stream shape's independently varied inputs" + )] + fn supervise_stream_child( + project_root: &Path, + phase: PhaseId, + workdir: &Path, + prompt: &str, + idle_timeout: Duration, + program: &str, + args: &[String], + envs: &[(String, String)], + agent: AgentKind, + ) -> Result { + let stdout_file = crate::agent_result::stdout_path(project_root, phase); + if let Some(parent) = stdout_file.parent() { + crate::workflow::ensure_devflow_dir(parent)?; } - } - - #[cfg(target_os = "linux")] - fn compile_holdfork_shim(dir: &Path) -> std::path::PathBuf { - let source = dir.join("holdfork.c"); - std::fs::write(&source, HOLDFORK_SHIM_C).unwrap(); - let shim = dir.join("holdfork.so"); - let out = std::process::Command::new("cc") - .args(["-shared", "-fPIC", "-o"]) - .arg(&shim) - .arg(&source) - .arg("-ldl") - .output() - .expect( - "these tests compile an LD_PRELOAD fork() shim and need `cc`, \ - the C toolchain cargo already links with on Linux", - ); - assert!( - out.status.success(), - "compiling the fork() shim failed:\n{}", - String::from_utf8_lossy(&out.stderr) + let never_bound = SocketPath::new(PathBuf::from("/nonexistent/never-bound.sock")) + .expect("a short socket path"); + let mut spec = StageSpec::new( + project_root.to_path_buf(), + phase, + agent, + program.to_string(), + StageIo::Stream { + prompt: prompt.to_string(), + idle_timeout, + }, + never_bound, + "never-bound".to_string(), ); - shim - } - - /// `/proc//cmdline` with its NULs as spaces; empty once the pid is - /// gone or a zombie. - #[cfg(target_os = "linux")] - fn cmdline_of(pid: u32) -> String { - std::fs::read(format!("/proc/{pid}/cmdline")) - .map(|raw| { - raw.split(|&byte| byte == 0) - .filter(|arg| !arg.is_empty()) - .map(|arg| String::from_utf8_lossy(arg).into_owned()) - .collect::>() - .join(" ") - }) - .unwrap_or_default() + spec.workdir = workdir.to_path_buf(); + spec.args = args.to_vec(); + spec.envs = envs.to_vec(); + let (mut child, capture) = spawn_stage_agent(&spec)?; + let pid = child.id(); + std::fs::write( + crate::agent_result::agent_pid_path(project_root, phase), + format!("{pid}\n"), + )?; + let capture = capture.expect("the stream shape pipes the agent's stdout"); + let threads = drive_stream_io(&mut child, capture, agent, prompt, idle_timeout, || { + fire_idle_timeout(project_root, phase, workdir, pid, idle_timeout); + })?; + let code = crate::supervisor::runtime::exit_code(child.wait()?); + std::fs::write( + crate::agent_result::exit_code_path(project_root, phase), + format!("{code}\n"), + )?; + threads.join(); + Ok(code) } - /// Whether `pid` has exec'd the stub agent (`sh -c 'sleep 30'`, or - /// `sleep 30` itself where the shell execs its last command). - #[cfg(target_os = "linux")] - fn runs_the_agent(pid: u32) -> bool { - let cmd = cmdline_of(pid); - !cmd.contains("trap cleanup TERM INT") && cmd.contains("sleep 30") - } + // ---- the three capture-shape launch tests 50-11 moved off the deleted + // `sh` monitor onto `run_supervised_stage`, each in a child test + // process (`launch_stage_with`, below), keeping its name and its + // assertions. - /// Whether SIGTERM is set in the named signal mask of `pid`'s - /// `/proc//status` (`SigCgt`, `SigPnd`, `ShdPnd`, ...). Panics when - /// the field cannot be read: an absent field must not read as "clear". - #[cfg(target_os = "linux")] - fn sigterm_in_mask(pid: u32, field: &str) -> bool { - signal_mask(pid, field) & (1 << (libc::SIGTERM - 1)) != 0 + /// A capture-shape fake agent `codex` running `body`, then waiting for + /// `go` so the parent sees its socket answer before it exits. + fn released_agent(fixture: &StageFixture, body: &str, go: &Path) -> PathBuf { + fixture.agent( + "codex", + &format!( + "{body}\nwhile [ ! -e '{go}' ]; do sleep 0.05; done\nexit 0", + go = go.display() + ), + ) } /// The named signal mask of `pid`'s `/proc//status` (`SigCgt`, @@ -4263,379 +3641,46 @@ pid_t fork(void) { u64::from_str_radix(hex, 16).expect("signal masks are hex") } - #[cfg(target_os = "linux")] - fn sigterm_is_caught(pid: u32) -> bool { - sigterm_in_mask(pid, "SigCgt") - } - - /// Whether a TERM has reached `pid` while it was held. The shim blocks - /// signals for the hold, so the TERM sits pending (`ShdPnd` for a - /// process-directed `kill`) until the hold ends, and the shim records it. - #[cfg(target_os = "linux")] - fn held_process_received_sigterm(held: &HeldMonitor, pid: u32) -> bool { - held.hold_saw_a_pending_term() - || sigterm_in_mask(pid, "ShdPnd") - || sigterm_in_mask(pid, "SigPnd") - } - - #[cfg(target_os = "linux")] - fn within_5s(mut done: impl FnMut() -> bool) -> bool { - for _ in 0..250 { - if done() { - return true; - } - std::thread::sleep(Duration::from_millis(20)); - } - done() - } - - /// Window A. Fails under dash before the fix (bash does not lose the TERM, - /// so on a bash `/bin/sh` this passes either way). - #[cfg(target_os = "linux")] - #[test] - fn sigterm_before_the_agent_child_resets_its_traps_still_kills_the_agent() { - let dir = tempfile::tempdir().unwrap(); - let mut held = HeldMonitor::spawn(dir.path(), HoldSide::Child); - let agent_pid = wait_for_agent_pid(dir.path(), held.state.phase) - .expect("the monitor records the agent pid right after its fork"); - held.tracked.push(agent_pid); - - // The injection landed where this test claims it did: the agent child - // is held before its trap reset, still running the monitor's script - // and still catching TERM with the handler it inherited. - let held_cmd = cmdline_of(agent_pid); - assert!( - held_cmd.contains("trap cleanup TERM INT"), - "the hold did not land on the agent fork; child already runs [{held_cmd}]" - ); - assert!( - sigterm_is_caught(agent_pid), - "the held agent child does not catch TERM, so it is not in the pre-reset window" - ); - - held.sigterm_monitor(); - held.reap_monitor_expecting_trap_exit(); - // `cleanup` has run, so its `kill` was sent while the child was held. - // It is pending there, and the handler the child inherited catches it - // when the hold ends, before the child's trap reset. - assert!( - within_5s(|| held_process_received_sigterm(&held, agent_pid)), - "the held agent child never received the monitor's TERM, so the kill did \ - not land inside the window" - ); - - held.release(); - let died = within_5s(|| !crate::agent::agent_running(agent_pid)); - assert!( - died, - "agent (pid {agent_pid}) survived a TERM that reached it before its trap \ - reset, orphaned: {}", - proc_snapshot(agent_pid) - ); - } - - /// Opposite-result control for the test above: the same shim and the same - /// hold, but released BEFORE the TERM, so the TERM finds an exec'd agent. - /// Passes before and after the fix; if it ever fails, the harness, not the - /// monitor, is what keeps the agent alive. - #[cfg(target_os = "linux")] - #[test] - fn sigterm_after_a_held_agent_child_execs_kills_the_agent() { - let dir = tempfile::tempdir().unwrap(); - let mut held = HeldMonitor::spawn(dir.path(), HoldSide::Child); - let agent_pid = wait_for_agent_pid(dir.path(), held.state.phase) - .expect("the monitor records the agent pid right after its fork"); - held.tracked.push(agent_pid); - let held_cmd = cmdline_of(agent_pid); - assert!( - held_cmd.contains("trap cleanup TERM INT"), - "the hold did not land on the agent fork; child already runs [{held_cmd}]" - ); - - held.release(); - assert!( - within_5s(|| runs_the_agent(agent_pid)), - "the released agent child never exec'd the agent: [{}]", - cmdline_of(agent_pid) - ); - held.sigterm_monitor(); - held.reap_monitor_expecting_trap_exit(); - - let died = within_5s(|| !crate::agent::agent_running(agent_pid)); - assert!( - died, - "agent (pid {agent_pid}) survived a TERM sent after it exec'd: {}", - proc_snapshot(agent_pid) - ); - assert!( - !held.hold_saw_a_pending_term(), - "a TERM landed inside the hold; this control must only TERM after exec" - ); - } - - /// Window F. Fails under dash and bash before the fix. - #[cfg(target_os = "linux")] - #[test] - fn sigterm_before_the_monitor_records_the_agent_pid_still_kills_the_agent() { - let dir = tempfile::tempdir().unwrap(); - let mut held = HeldMonitor::spawn(dir.path(), HoldSide::Parent); - // The monitor is held right after its fork, so it has not written the - // pidfile; the agent is the monitor's only child. - let children = format!("/proc/{0}/task/{0}/children", held.monitor_pid); - let mut agent_pid = None; - let found = within_5s(|| { - agent_pid = std::fs::read_to_string(&children) - .expect("/proc//task//children must be readable") - .split_whitespace() - .next() - .map(|pid| pid.parse::().expect("children lists numeric pids")); - agent_pid.is_some() - }); - assert!(found, "the held monitor never forked its agent"); - let agent_pid = agent_pid.unwrap(); - held.tracked.push(agent_pid); - assert!( - within_5s(|| runs_the_agent(agent_pid)), - "the agent child of the held monitor never exec'd: [{}]", - cmdline_of(agent_pid) - ); - let pidfile = crate::agent_result::agent_pid_path(dir.path(), held.state.phase); - assert!( - !pidfile.exists(), - "the monitor already wrote the pidfile, so the hold did not land between \ - its fork and `apid=$!`" - ); - - held.sigterm_monitor(); - let monitor_pid = held.monitor_pid; - assert!( - within_5s(|| held_process_received_sigterm(&held, monitor_pid)), - "the held monitor never received the TERM" - ); - held.release(); - held.reap_monitor_expecting_trap_exit(); - - let died = within_5s(|| !crate::agent::agent_running(agent_pid)); - assert!( - died, - "agent (pid {agent_pid}) survived a TERM that reached the monitor before \ - it recorded the agent pid, orphaned: {}", - proc_snapshot(agent_pid) - ); - } - - /// The stop marker a TERM'd monitor leaves behind must not stop the NEXT - /// agent for that phase: the Legacy launch removes a stale marker before - /// it spawns. Without that, the agent child finds the marker, exits 143 - /// without ever running, and the stage records a kill nobody sent. - /// - /// The tail is empty so the monitor does not re-exec this test binary as - /// `devflow advance`; the marker handling does not depend on it. - #[test] - fn a_stale_stop_marker_does_not_stop_the_next_agent() { - let dir = tempfile::tempdir().unwrap(); - let state = state_in(dir.path()); - let marker = crate::agent_result::stop_marker_path(dir.path(), state.phase); - std::fs::create_dir_all(marker.parent().unwrap()).unwrap(); - std::fs::write(&marker, "\n").unwrap(); - let args = vec![ - "-c".to_string(), - "echo STALE_MARKER_AGENT_RAN; exit 7".to_string(), - ]; - - spawn_monitor_inner( - &state, - "sh", - &args, - &[], - MonitorLaunch::Legacy, - &test_launch_context(&state), - MonitorTail::Script(String::new()), - ) - .unwrap(); - - let exit_path = crate::agent_result::exit_code_path(dir.path(), state.phase); - let mut recorded = None; - for _ in 0..250 { - recorded = std::fs::read_to_string(&exit_path) - .ok() - .filter(|contents| contents.ends_with('\n')); - if recorded.is_some() { - break; - } - std::thread::sleep(Duration::from_millis(20)); - } - let recorded = recorded.expect("the monitor never recorded the agent's exit code"); - assert_eq!( - recorded.trim(), - "7", - "a stale stop marker stopped the next agent: its exit code is {:?}, not the \ - agent's own 7 (143 means the agent child found the marker and never exec'd)", - recorded.trim() - ); - let captured = - std::fs::read_to_string(crate::agent_result::stdout_path(dir.path(), state.phase)) - .unwrap(); - assert!( - captured.contains("STALE_MARKER_AGENT_RAN"), - "the agent did not run under a stale stop marker; captured stdout: {captured:?}" - ); - assert!( - !marker.exists(), - "the stale stop marker survived the launch that should have removed it" - ); - } - - /// WR-01 (48-REVIEW.md): a TERM that reaches the monitor after its agent - /// has been reaped — while the `advance` tail runs — must not act on the - /// agent. The shell defers the trap until the foreground tail returns, and - /// by then `advance` has launched the next stage's monitor, which removed - /// the phase's stop marker. A `cleanup` that writes the marker anyway stops - /// that next agent (exit 143) and `kill`s a pid reaped long ago. - /// - /// The tail stands in for `advance`: it announces itself, blocks until - /// released (capped at 10 s so a failed test cannot leak it), then exits 5. - /// `kill()` queues the TERM before it returns and the release comes after, - /// so the TERM always lands while the tail runs; and the monitor exiting 0 - /// rather than the tail's 5 proves `cleanup` ran rather than being missed. - #[test] - fn a_term_after_the_agent_exits_leaves_no_stop_marker() { - let dir = tempfile::tempdir().unwrap(); - let state = state_in(dir.path()); - let marker = crate::agent_result::stop_marker_path(dir.path(), state.phase); - let started = dir.path().join("tail-started"); - let release = dir.path().join("tail-release"); - let tail = format!( - "sh -c 'echo > \"$1\"; i=0; while [ ! -e \"$2\" ] && [ $i -lt 500 ]; do \ - sleep 0.02; i=$((i+1)); done; exit 5' tail {} {}", - shell_escape(started.to_str().unwrap()), - shell_escape(release.to_str().unwrap()), - ); - let args = vec!["-c".to_string(), "exit 7".to_string()]; - let monitor_pid = spawn_monitor_inner( - &state, - "sh", - &args, - &[], - MonitorLaunch::Legacy, - &test_launch_context(&state), - MonitorTail::Script(tail), - ) - .unwrap(); - - let mut tail_running = false; - for _ in 0..250 { - if started.exists() { - tail_running = true; - break; - } - std::thread::sleep(Duration::from_millis(20)); - } - assert!(tail_running, "the monitor never reached its tail"); - let recorded = - std::fs::read_to_string(crate::agent_result::exit_code_path(dir.path(), state.phase)) - .unwrap(); - assert_eq!( - recorded.trim(), - "7", - "the tail runs only after the agent was reaped and its exit recorded" - ); - let monitor_cmd = std::fs::read(format!("/proc/{monitor_pid}/cmdline")).unwrap(); - assert!( - String::from_utf8_lossy(&monitor_cmd).contains("trap cleanup TERM INT"), - "the monitor shell exec'd its tail, so no trap is left to test" - ); - - let rc = unsafe { libc::kill(monitor_pid as libc::pid_t, libc::SIGTERM) }; - assert_eq!( - rc, - 0, - "kill(monitor, TERM): {}", - std::io::Error::last_os_error() - ); - std::fs::write(&release, "").unwrap(); - - let pid = monitor_pid as libc::pid_t; - let mut status = None; - for _ in 0..600 { - let mut raw: libc::c_int = 0; - let waited = unsafe { libc::waitpid(pid, &mut raw, libc::WNOHANG) }; - if waited == pid { - status = Some(raw); - break; - } - assert_eq!( - waited, - 0, - "waitpid(monitor): {}", - std::io::Error::last_os_error() - ); - std::thread::sleep(Duration::from_millis(20)); - } - let status = status.expect("the monitor did not exit within 12s of its release"); - assert!( - libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, - "the monitor must leave through its trap (`cleanup` → `exit 0`), not finish \ - its tail (exit 5); wait status {status:#x}" - ); - assert!( - !marker.exists(), - "a TERM after the agent was reaped wrote the phase's stop marker, which stops \ - the next stage's agent before it runs" - ); - } + const WORKTREE_CAPTURE_TEST: &str = + "monitor::tests::spawn_monitor_runs_agent_in_worktree_but_captures_in_project_root"; #[test] fn spawn_monitor_runs_agent_in_worktree_but_captures_in_project_root() { - let dir = tempfile::tempdir().unwrap(); - let worktree = dir.path().join(".worktrees/phase-04"); - std::fs::create_dir_all(&worktree).unwrap(); - let mut state = state_in(dir.path()); - state.worktree_path = Some(worktree.clone()); - - // Stub agent: print its cwd so the test proves the monitor changed - // directories before launching the agent. - let args = vec!["-c".to_string(), "pwd; echo WORKTREE_READY".to_string()]; - - let monitor_pid = spawn_monitor( - &state, - "sh", - &args, - &[], - MonitorLaunch::Legacy, - &test_launch_context(&state), - ) - .unwrap(); - assert!(monitor_pid > 0); - - let agent_pid = wait_for_agent_pid(dir.path(), state.phase) - .expect("monitor should record the agent pid in the main project"); - assert!(agent_pid > 0); - - let stdout_path = crate::agent_result::stdout_path(dir.path(), state.phase); - let mut captured = String::new(); - for _ in 0..100 { - if let Ok(contents) = std::fs::read_to_string(&stdout_path) - && contents.contains("WORKTREE_READY") - { - captured = contents; - break; - } - std::thread::sleep(Duration::from_millis(20)); + if crate::test_support::in_child_test(WORKTREE_CAPTURE_TEST) { + supervise_stage_in_child(); + return; } + let fixture = StageFixture::new(); + let go = fixture.path("go"); + // Stub agent: print its cwd so the test proves the supervisor started + // it in the worktree. + let program = released_agent(&fixture, "pwd; echo WORKTREE_READY", &go); + let mut run = launch_stage( + WORKTREE_CAPTURE_TEST, + &fixture, + Shape::Capture, + AgentKind::Codex, + &program, + "", + ); + std::fs::write(&go, "").expect("release the agent"); + assert_eq!(run.finish(WORKTREE_CAPTURE_TEST), "Completed(0) advancing"); + let captured = fixture.stdout(); assert!( - captured.contains(&worktree.display().to_string()), + captured.contains("WORKTREE_READY") + && captured.contains(&fixture.workdir.display().to_string()), "agent did not run in worktree cwd; captured stdout: {captured:?}" ); assert!( - stdout_path.exists(), + crate::agent_result::stdout_path(&fixture.root, STAGE_PHASE).exists(), "stdout capture missing in main .devflow" ); assert!( - !crate::agent_result::stdout_path(&worktree, state.phase).exists(), + !crate::agent_result::stdout_path(&fixture.workdir, STAGE_PHASE).exists(), "stdout capture should not be written under the worktree" ); + drop(run.group); } /// Build the fixture repositories through the scrubbing constructor, as @@ -4663,152 +3708,109 @@ pid_t fork(void) { git(root, &["config", "user.name", "Test"]); } - /// 27-REVIEW WR-03: the `sh` this function spawns owns the coding - /// agent, and whatever environment rides down with it reaches every git - /// command the agent runs (`sh` -> agent -> agent's git children). This - /// proves the scrub with a real spawned agent process, not by - /// inspecting the `Command` object: the agent shells out to - /// `git rev-parse --absolute-git-dir`, and the resolved path must be - /// the caller's own workdir, never a hostile `GIT_DIR` pointed at an - /// unrelated foreign repository. + const HOSTILE_GIT_DIR_TEST: &str = + "monitor::tests::spawn_monitor_agent_git_calls_resolve_workdir_not_a_hostile_git_dir"; + + /// 27-REVIEW WR-03: whatever environment the supervisor carries rides + /// down to the agent and every git command it runs. This proves the + /// scrub with a real spawned agent process: the supervisor (a child test + /// process) runs with `GIT_DIR` pointed at an unrelated foreign + /// repository, the agent shells out to `git rev-parse --absolute-git-dir`, + /// and the resolved path must be the agent's own workdir. /// - /// Mirrors `tag_reads_resolve_caller_root_under_a_hostile_git_dir` - /// (version.rs, 27-03/WR-01): `GIT_DIR` is never set on this test - /// process itself (Rust 2024 `unsafe`, unsound under threaded tests — - /// Phase 25 D-14), only on one freshly spawned child re-invoking this - /// binary filtered to this test. + /// `GIT_DIR` is never set on this test process itself (Rust 2024 + /// `unsafe`, unsound under threaded tests — Phase 25 D-14), only on the + /// child supervisor. Control: the child's own environment is read from + /// `/proc` and must carry the hostile value, so a pass cannot come from a + /// `GIT_DIR` that never reached the supervisor. #[test] fn spawn_monitor_agent_git_calls_resolve_workdir_not_a_hostile_git_dir() { - const INNER_ROOT: &str = "DEVFLOW_27_MONITOR_INNER_ROOT"; - - if let Ok(root) = std::env::var(INNER_ROOT) { - // Inner mode: GIT_DIR points at a foreign repository unrelated - // to `root`, scoped to this child process only. - let root = std::path::PathBuf::from(root); - let state = state_in(&root); - let args = vec![ - "-c".to_string(), - "git rev-parse --absolute-git-dir".to_string(), - ]; - - spawn_monitor( - &state, - "sh", - &args, - &[], - MonitorLaunch::Legacy, - &test_launch_context(&state), - ) - .unwrap(); - wait_for_agent_pid(&root, state.phase).expect("monitor should record the agent pid"); - - let stdout_path = crate::agent_result::stdout_path(&root, state.phase); - let mut captured = String::new(); - for _ in 0..100 { - if let Ok(contents) = std::fs::read_to_string(&stdout_path) - && !contents.trim().is_empty() - { - captured = contents; - break; - } - std::thread::sleep(Duration::from_millis(20)); - } - - let resolved = std::fs::canonicalize(captured.trim()) - .expect("agent's reported git-dir must exist on disk"); - let expected = - std::fs::canonicalize(root.join(".git")).expect("caller repo .git must exist"); - assert_eq!( - resolved, expected, - "agent's git call resolved to a hostile GIT_DIR's \ - repository instead of the caller's own workdir: \ - got {resolved:?}, want {expected:?}" - ); + if crate::test_support::in_child_test(HOSTILE_GIT_DIR_TEST) { + supervise_stage_in_child(); return; } - - // Outer mode: a real repository at `root`, and an unrelated - // foreign repository whose .git must never leak into the agent's - // environment. - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().join("caller-repo"); - std::fs::create_dir_all(&root).unwrap(); - init_repo(&root); - + let fixture = StageFixture::new(); + init_repo(&fixture.workdir); let foreign = tempfile::tempdir().unwrap(); init_repo(foreign.path()); - - let exe = std::env::current_exe().expect("current_exe for child re-invocation"); - let out = std::process::Command::new(&exe) - // Substring filter, NOT `--exact`: the binary's real test name - // is module-qualified (`monitor::tests::spawn_monitor_...`), so - // `--exact` against the bare name matches nothing, runs zero - // tests, and still exits 0 — a false green. - .arg("spawn_monitor_agent_git_calls_resolve_workdir_not_a_hostile_git_dir") - .arg("--test-threads=1") - .env(INNER_ROOT, root.to_str().unwrap()) - .env("GIT_DIR", foreign.path().join(".git")) - .output() - .expect("spawn hostile child test process"); - - let stdout = String::from_utf8_lossy(&out.stdout); - // Assert the child actually RAN the test, not merely that it - // exited 0. A filter that matches nothing exits 0 with "0 passed". - assert!( - stdout.contains("1 passed"), - "child test process must have run exactly the inner test; \ - stdout:\n{stdout}" + let foreign_git = foreign.path().join(".git"); + let go = fixture.path("go"); + let program = released_agent(&fixture, "git rev-parse --absolute-git-dir", &go); + let mut run = launch_stage_with( + HOSTILE_GIT_DIR_TEST, + &fixture, + StageLaunchArgs { + shape: Shape::Capture, + agent: AgentKind::Codex, + program: &program, + adapter_env: "", + args: &[], + extra_env: &[("GIT_DIR", foreign_git.as_os_str())], + }, ); + let environ = std::fs::read(format!("/proc/{}/environ", run.child.id())) + .expect("read the child supervisor's environment"); + let hostile = format!("GIT_DIR={}", foreign_git.display()); assert!( - out.status.success(), - "monitor-spawned agent (hostile GIT_DIR pointed at an \ - unrelated foreign repository) must still resolve its git \ - calls against the caller's own workdir; child exit status \ - {:?}\nstdout:\n{stdout}", - out.status + environ + .split(|byte| *byte == 0) + .any(|entry| entry == hostile.as_bytes()), + "control: the supervisor must run with the hostile {hostile}" + ); + std::fs::write(&go, "").expect("release the agent"); + assert_eq!(run.finish(HOSTILE_GIT_DIR_TEST), "Completed(0) advancing"); + + let captured = fixture.stdout(); + let resolved = std::fs::canonicalize(captured.trim()) + .expect("agent's reported git-dir must exist on disk"); + let expected = std::fs::canonicalize(fixture.workdir.join(".git")) + .expect("caller repo .git must exist"); + assert_eq!( + resolved, expected, + "agent's git call resolved to a hostile GIT_DIR's repository instead of the \ + caller's own workdir: got {resolved:?}, want {expected:?}" ); + drop(run.group); } + const LITERAL_ARGV_TEST: &str = + "monitor::tests::spawn_monitor_treats_agent_args_as_literal_argv"; + #[test] fn spawn_monitor_treats_agent_args_as_literal_argv() { - let dir = tempfile::tempdir().unwrap(); - let state = state_in(dir.path()); - let payload = "value; touch INJECTED"; - let args = vec![ - "-c".to_string(), - "printf '%s\\n' \"$0\"; echo ARGV_SAFE".to_string(), - payload.to_string(), - ]; - - spawn_monitor( - &state, - "sh", - &args, - &[], - MonitorLaunch::Legacy, - &test_launch_context(&state), - ) - .unwrap(); - wait_for_agent_pid(dir.path(), state.phase).expect("monitor should record the agent pid"); - - let stdout_path = crate::agent_result::stdout_path(dir.path(), state.phase); - let mut captured = String::new(); - for _ in 0..100 { - if let Ok(contents) = std::fs::read_to_string(&stdout_path) - && contents.contains("ARGV_SAFE") - { - captured = contents; - break; - } - std::thread::sleep(Duration::from_millis(20)); + if crate::test_support::in_child_test(LITERAL_ARGV_TEST) { + supervise_stage_in_child(); + return; } + let fixture = StageFixture::new(); + let go = fixture.path("go"); + let payload = "value; touch INJECTED"; + let program = released_agent(&fixture, "printf '%s\\n' \"$1\"; echo ARGV_SAFE", &go); + let args = [payload.to_string()]; + let mut run = launch_stage_with( + LITERAL_ARGV_TEST, + &fixture, + StageLaunchArgs { + shape: Shape::Capture, + agent: AgentKind::Codex, + program: &program, + adapter_env: "", + args: &args, + extra_env: &[], + }, + ); + std::fs::write(&go, "").expect("release the agent"); + assert_eq!(run.finish(LITERAL_ARGV_TEST), "Completed(0) advancing"); + let captured = fixture.stdout(); assert!( captured.contains(payload), "literal argv missing: {captured:?}" ); assert!(captured.contains("ARGV_SAFE")); - assert!(!dir.path().join("INJECTED").exists()); + assert!(!fixture.workdir.join("INJECTED").exists()); + assert!(!fixture.root.join("INJECTED").exists()); + drop(run.group); } // ---- idle timeout (31-02, D-01..D-08) -------------------------------- @@ -4924,7 +3926,7 @@ exit 0 "#; let started = std::time::Instant::now(); - let code = run_pipe_owning_monitor( + let code = supervise_stream_child( root, phase, root, @@ -5012,7 +4014,7 @@ sleep 120 None }); - let code = run_pipe_owning_monitor( + let code = supervise_stream_child( &root, phase, &root, @@ -5136,7 +4138,7 @@ printf '%s\n' '{"type":"system","subtype":"init","session_id":"idle-3"}' sleep 120 "#; - run_pipe_owning_monitor( + supervise_stream_child( root, phase, root, @@ -5388,7 +4390,7 @@ exit 0 eof = eof_file.display(), ); - let code = run_pipe_owning_monitor( + let code = supervise_stream_child( root, phase, root, @@ -5464,6 +4466,8 @@ exit 0 const STAGE_AGENT_ENV: &str = "DEVFLOW_TEST_STAGE_AGENT"; /// One adapter env pair as `NAME=VALUE`, or empty for none. const STAGE_ADAPTER_ENV: &str = "DEVFLOW_TEST_STAGE_ADAPTER_ENV"; + /// The agent's argv after its program, as a JSON array of strings. + const STAGE_ARGS_ENV: &str = "DEVFLOW_TEST_STAGE_ARGS"; /// The phase every supervised-stage test runs. const STAGE_PHASE: PhaseId = PhaseId::new(50); @@ -5607,6 +4611,10 @@ exit 0 if let Some((name, value)) = adapter_env.split_once('=') { spec.envs.push((name.to_string(), value.to_string())); } + if let Some(args) = std::env::var_os(STAGE_ARGS_ENV) { + spec.args = serde_json::from_str(args.to_str().expect("utf-8 agent args")) + .expect("the parent passes the agent args as a JSON array"); + } let run = run_supervised_stage(spec).expect("supervise the stage"); let record = format!( "{:?} {}", @@ -5805,9 +4813,19 @@ exit 0 } } - /// Re-run `test_name` as a child supervisor of `program` in `shape`, and - /// wait until its socket answers a ping. At that first Alive the sidecar - /// must already be readable (R-7: the sidecar precedes the socket name). + /// What [`launch_stage_with`] launches beyond the fixture: the shape, + /// the agent, its program and argv, one adapter env pair, and extra + /// environment for the child supervisor itself. + struct StageLaunchArgs<'a> { + shape: Shape, + agent: AgentKind, + program: &'a Path, + adapter_env: &'a str, + args: &'a [String], + extra_env: &'a [(&'a str, &'a std::ffi::OsStr)], + } + + /// [`launch_stage_with`] with no agent argv and no extra environment. fn launch_stage( test_name: &str, fixture: &StageFixture, @@ -5816,6 +4834,37 @@ exit 0 program: &Path, adapter_env: &str, ) -> StageLaunch { + launch_stage_with( + test_name, + fixture, + StageLaunchArgs { + shape, + agent, + program, + adapter_env, + args: &[], + extra_env: &[], + }, + ) + } + + /// Re-run `test_name` as a child supervisor of `launch.program` in + /// `launch.shape`, and wait until its socket answers a ping. At that first + /// Alive the sidecar must already be readable (R-7: the sidecar precedes + /// the socket name). + fn launch_stage_with( + test_name: &str, + fixture: &StageFixture, + launch: StageLaunchArgs<'_>, + ) -> StageLaunch { + let StageLaunchArgs { + shape, + agent, + program, + adapter_env, + args, + extra_env, + } = launch; let socket = SocketPath::for_launch( fixture.cache.path(), &fixture.root, @@ -5825,19 +4874,23 @@ exit 0 .expect("socket path fits sun_path"); let outcome = fixture.path("outcome"); let agent_label = agent.to_string(); + let args_json = serde_json::to_string(args).expect("encode the agent args"); + let mut env: Vec<(&str, &std::ffi::OsStr)> = vec![ + (STAGE_SOCKET_ENV, socket.as_path().as_os_str()), + (STAGE_OUTCOME_ENV, outcome.as_os_str()), + (STAGE_SHAPE_ENV, std::ffi::OsStr::new(shape.label())), + (STAGE_PROGRAM_ENV, program.as_os_str()), + (STAGE_ROOT_ENV, fixture.root.as_os_str()), + (STAGE_WORKDIR_ENV, fixture.workdir.as_os_str()), + (STAGE_AGENT_ENV, std::ffi::OsStr::new(&agent_label)), + (STAGE_ADAPTER_ENV, std::ffi::OsStr::new(adapter_env)), + (STAGE_ARGS_ENV, std::ffi::OsStr::new(&args_json)), + ]; + env.extend_from_slice(extra_env); let mut child = StageChild(Some(crate::test_support::spawn_test_in_child( test_name, fixture.cache.path(), - &[ - (STAGE_SOCKET_ENV, socket.as_path().as_os_str()), - (STAGE_OUTCOME_ENV, outcome.as_os_str()), - (STAGE_SHAPE_ENV, std::ffi::OsStr::new(shape.label())), - (STAGE_PROGRAM_ENV, program.as_os_str()), - (STAGE_ROOT_ENV, fixture.root.as_os_str()), - (STAGE_WORKDIR_ENV, fixture.workdir.as_os_str()), - (STAGE_AGENT_ENV, std::ffi::OsStr::new(&agent_label)), - (STAGE_ADAPTER_ENV, std::ffi::OsStr::new(adapter_env)), - ], + &env, ))); let deadline = std::time::Instant::now() + STAGE_CHILD_WAIT; let reply = loop { diff --git a/crates/devflow-core/src/supervisor/runtime.rs b/crates/devflow-core/src/supervisor/runtime.rs index 8ccf9f1e..5c088d1e 100644 --- a/crates/devflow-core/src/supervisor/runtime.rs +++ b/crates/devflow-core/src/supervisor/runtime.rs @@ -331,7 +331,7 @@ fn wait_for_exit_without_reaping(pid: u32) -> io::Result<()> { /// An exit status as a code: the exit code, or `128 + n` for a death by /// signal `n` (the `monitor` convention: SIGTERM is 143, SIGKILL 137). -fn exit_code(status: std::process::ExitStatus) -> i32 { +pub(crate) fn exit_code(status: std::process::ExitStatus) -> i32 { status.code().unwrap_or_else(|| { use std::os::unix::process::ExitStatusExt; status.signal().map_or(-1, |signal| 128 + signal) diff --git a/crates/devflow-core/tests/devflow_dir_gitignore.rs b/crates/devflow-core/tests/devflow_dir_gitignore.rs index 4dba93be..cdbaabc8 100644 --- a/crates/devflow-core/tests/devflow_dir_gitignore.rs +++ b/crates/devflow-core/tests/devflow_dir_gitignore.rs @@ -27,7 +27,6 @@ use devflow_core::ship; use devflow_core::workflow; use devflow_core::{AgentKind, Mode, Stage, State}; use std::path::Path; -use std::time::{Duration, Instant}; /// `true` iff `root/.devflow/.gitignore` exists with trimmed content `*`. fn gitignore_is_star(root: &Path) -> bool { @@ -36,62 +35,34 @@ fn gitignore_is_star(root: &Path) -> bool { .unwrap_or(false) } -/// Poll `path` until it exists (or `timeout` elapses, at which point this -/// panics naming the path — a monitor that never writes its exit-code -/// capture file is itself a bug, not a thing to silently tolerate). -fn wait_for_file(path: &Path, timeout: Duration) { - let start = Instant::now(); - while !path.exists() { - assert!( - start.elapsed() < timeout, - "expected {} to appear within {timeout:?}", - path.display() - ); - std::thread::sleep(Duration::from_millis(20)); - } -} - -/// Reap `pid`, then poll until it is no longer a live process (or `timeout` -/// elapses). -/// -/// Codex review: `monitor::spawn_monitor` returns only a detached child's -/// `u32` pid, not a `Child` handle the test could `.wait()` on. But this test -/// binary is still the OS-level parent of that process (it called -/// `Command::spawn()` internally), so it never gets reparented to init — -/// without an explicit `waitpid`, the exited monitor shell would sit as a -/// zombie (which still answers `kill(pid, 0)` successfully) for the rest of -/// the test binary's lifetime. -fn wait_for_pid_to_die(pid: u32, timeout: Duration) { - unsafe { - let mut status: libc::c_int = 0; - libc::waitpid(pid as libc::pid_t, &mut status, 0); - } - let start = Instant::now(); - while devflow_core::agent::agent_running(pid) { - assert!( - start.elapsed() < timeout, - "monitor process pid {pid} did not terminate within {timeout:?}" - ); - std::thread::sleep(Duration::from_millis(20)); - } -} - /// Every one of the 7 production `.devflow/`-creating entry points, driven /// through its real public API (not `ensure_devflow_dir` directly — that /// would prove nothing about whether the site was actually converted), /// against its own fresh temp project root. Failures accumulate into a /// vector so one regressed site doesn't hide another (mirrors the /// accumulate-then-assert shape `gitignore_coverage.rs` already uses). -/// The launch context case 3 passes `spawn_monitor` (50-10): this test binary -/// as the monitor executable, which is what `spawn_monitor` resolved itself -/// before it took a context, and a cache dir under the case's own tempdir. -fn launch_context(root: &Path) -> monitor::LaunchContext { - monitor::LaunchContext::new( - root.join("cache"), - monitor::MonitorExec::Subcommand { - exe: std::env::current_exe().unwrap(), - }, - ) +/// The launch context case 3 passes `spawn_monitor` (50-11): a monitor +/// executable that exits at once (a 0755 `#!/bin/sh` / `exit 0` script) and +/// a short cache dir under `/tmp`, both in a tempdir the case owns. +struct ExitingMonitor { + _dir: tempfile::TempDir, + context: monitor::LaunchContext, +} + +fn launch_context() -> ExitingMonitor { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::Builder::new() + .prefix("dfs") + .tempdir_in("/tmp") + .expect("create a short monitor tempdir under /tmp"); + let exe = dir.path().join("monitor"); + std::fs::write(&exe, "#!/bin/sh\nexit 0\n").unwrap(); + std::fs::set_permissions(&exe, std::fs::Permissions::from_mode(0o755)).unwrap(); + let context = monitor::LaunchContext::new( + dir.path().join("cache"), + monitor::MonitorExec::Subcommand { exe }, + ); + ExitingMonitor { _dir: dir, context } } #[test] @@ -128,20 +99,12 @@ fn all_seven_devflow_constructors_produce_the_gitignore() { // creates the capture directory without going through save_state — // the state passed here is synthetic and never persisted, D-14). // - // spawn_monitor appends a `devflow advance` tail that the deleted - // no-advance variant omitted (23-08). That tail's `binary` resolves - // via `std::env::current_exe()`, which inside `cargo test` is this - // test binary itself, not the real `devflow` CLI — invoking it with - // `advance --phase 1` hits the Rust test harness's own - // argument parser, which rejects the unrecognized `--phase` flag and - // exits immediately (~milliseconds; independently confirmed via a - // direct invocation of the built test binary). It never reaches - // `devflow advance`'s real state-loading or gate logic, so there is - // no risk of blocking on a multi-day gate wait here — the same - // fail-fast tail already runs, unbounded and unremarked, in every - // other `spawn_monitor`-based test in `monitor.rs`. No additional - // timeout override is needed; `wait_for_file` and `wait_for_pid_to_die` - // below are the test's real (and sufficient) synchronisation points. + // The monitor executable is a `#!/bin/sh` script that exits at once: + // a core test binary has no `__monitor` subcommand to re-exec, so no + // supervisor can publish here. `spawn_monitor` therefore reports + // `NotReady` once the script has exited, and the case proves the + // capture directory (and its `.gitignore`) was created BEFORE the + // spawn, so it exists even when the launch fails. { let dir = tempfile::tempdir().unwrap(); let root = dir.path(); @@ -151,21 +114,19 @@ fn all_seven_devflow_constructors_produce_the_gitignore() { Mode::Auto, root.to_path_buf(), ); - let pid = monitor::spawn_monitor( + let ctx = launch_context(); + let result = monitor::spawn_monitor( &state, "sh", &["-c".to_string(), "exit 0".to_string()], &[], monitor::MonitorLaunch::Legacy, - &launch_context(root), - ) - .expect("spawn_monitor"); - - wait_for_file( - &agent_result::exit_code_path(root, PhaseId::new(1)), - Duration::from_secs(5), + &ctx.context, + ); + assert!( + matches!(result, Err(monitor::MonitorError::NotReady(_))), + "a monitor executable that exits at once must be reported NotReady, got {result:?}" ); - wait_for_pid_to_die(pid, Duration::from_secs(5)); if !gitignore_is_star(root) { failures.push("monitor::spawn_monitor (monitor.rs)"); From 033bd34b21d2fcd363e1099627c4ba05b60a9ed6 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 16:18:51 -0400 Subject: [PATCH 050/112] test(50-11): end launched runs through the supervisor socket - ReapMonitorOnDrop, reap_spawned_monitor and LaunchedRunReaper send a socket shutdown, then SIGKILL the recorded agent group only after agent::is_same_process confirms its leader; none reads monitor_pid - the reaper tests' fixture records a SupervisorRecord for a subject in its own process group, so they drive the identity-confirmed fallback - auto_chain_flag_e2e and auto_chain_leak_repair_e2e pass the 3.x __monitor argv (--launch stream, --launch-id, --socket) - libc becomes a devflow dev-dependency for the group signal Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- Cargo.lock | 1 + crates/devflow-cli/Cargo.toml | 2 + crates/devflow-cli/src/test_support.rs | 175 +++++++++++------- .../devflow-cli/tests/auto_chain_flag_e2e.rs | 18 +- .../tests/auto_chain_leak_repair_e2e.rs | 16 ++ crates/devflow-cli/tests/common/mod.rs | 62 +++++-- 6 files changed, 188 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f707ba5c..93d6cf37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,7 @@ dependencies = [ "clap", "devflow-core", "insta", + "libc", "serde_json", "tempfile", "thiserror", diff --git a/crates/devflow-cli/Cargo.toml b/crates/devflow-cli/Cargo.toml index 2c60a8be..a7ea4bbe 100644 --- a/crates/devflow-cli/Cargo.toml +++ b/crates/devflow-cli/Cargo.toml @@ -23,6 +23,8 @@ thiserror.workspace = true [dev-dependencies] tempfile = "3" +# Process-group signals in the test reapers' identity-confirmed fallback (50-11). +libc = "0.2" # `test_support::git_command` for this crate's test targets (999.37). devflow-core = { workspace = true, features = ["test-support"] } # `insta::assert_snapshot!` for the prompt-text drift guard (47-CONTEXT.md D-14/D-15). diff --git a/crates/devflow-cli/src/test_support.rs b/crates/devflow-cli/src/test_support.rs index 24560a42..6667d0d6 100644 --- a/crates/devflow-cli/src/test_support.rs +++ b/crates/devflow-cli/src/test_support.rs @@ -15,7 +15,7 @@ use devflow_core::agents; use devflow_core::phase_id::PhaseId; -use devflow_core::state::State; +use devflow_core::state::{State, SupervisorRecord}; use std::path::{Path, PathBuf}; use std::sync::Mutex; @@ -531,28 +531,60 @@ pub(crate) fn stage_launched_count(root: &Path, phase: PhaseId) -> usize { .count() } -/// The single reap mechanism: escalate through `terminate_and_verify` -/// (bounded TERM-then-KILL — 999.44 measured 15 of 15 orphaned monitor -/// wrappers surviving a bare `SIGTERM` against this exact process shape, so -/// an unescalated signal would leave the leak in place while appearing to -/// fix it), then return a VERIFIED liveness answer. +/// How long a reaper waits for a supervisor's final shutdown line. +const REAP_SHUTDOWN_WAIT: std::time::Duration = std::time::Duration::from_secs(15); + +/// The single reap mechanism (D-07): end a launched run through its +/// supervisor's socket, then return a VERIFIED answer about its agent group. +/// +/// 1. `supervisor::request_shutdown` on the recorded socket, bounded by +/// [`REAP_SHUTDOWN_WAIT`]. A supervisor that answers ends its agent group +/// itself. +/// 2. If the recorded agent group's leader is still the process the record +/// names (`agent::is_same_process` against its start time), SIGKILL the +/// whole group. This covers a socket that does not answer (a SIGKILLed +/// supervisor, or a record whose socket never existed); the identity +/// check keeps a recycled pid from ever being signalled. +/// 3. Poll until the leader is gone (a zombie counts as gone) or +/// `agent::TERMINATE_VERIFY_WAIT` passes. /// /// Asserts nothing and panics on no path — the caller decides what a `false` -/// means. This matters because one of its two callers -/// ([`ReapMonitorOnDrop::drop`]) may run during a stack unwind, where a -/// panic would call `abort()` and kill the whole test binary instead of -/// merely failing the one test in flight. -fn reap_monitor_pid(pid: u32) -> bool { - devflow_core::agent::terminate_and_verify( - pid, - devflow_core::agent::TERMINATE_VERIFY_WAIT, - devflow_core::agent::TERMINATE_VERIFY_POLL, - ); - !devflow_core::agent::agent_running(pid) +/// means. This matters because one of its callers ([`ReapMonitorOnDrop::drop`]) +/// may run during a stack unwind, where a panic would call `abort()` and kill +/// the whole test binary instead of merely failing the one test in flight. +fn reap_supervised_run(record: &SupervisorRecord) -> bool { + let _ = devflow_core::supervisor::request_shutdown(&record.socket_path, REAP_SHUTDOWN_WAIT); + let leader = record.agent_pgid; + let alive = || { + devflow_core::agent::agent_running(leader) + && devflow_core::agent::is_same_process(leader, record.agent_start_time) + }; + if alive() { + signal_group(leader, libc::SIGKILL); + } + let deadline = std::time::Instant::now() + devflow_core::agent::TERMINATE_VERIFY_WAIT; + while alive() && std::time::Instant::now() < deadline { + std::thread::sleep(devflow_core::agent::TERMINATE_VERIFY_POLL); + } + !alive() +} + +/// Send `signal` to process group `pgid`. Refuses pgid 0 or 1, a value that +/// does not fit `pid_t`, and this process's own group. +fn signal_group(pgid: u32, signal: libc::c_int) { + let Ok(signed) = libc::pid_t::try_from(pgid) else { + return; + }; + // SAFETY: getpgrp has no preconditions and cannot fail. + if signed > 1 && signed != unsafe { libc::getpgrp() } { + // SAFETY: `signed > 1` and not our own group: this names one process + // group whose leader the caller has just identity-checked. + unsafe { libc::kill(-signed, signal) }; + } } -/// Reap the detached monitor wrapper a real `launch_stage_inner` call just -/// spawned, verifying its death rather than assuming it (WR-03, 999.46). +/// Reap the supervised run a real `launch_stage_inner` call just launched, +/// verifying its agent group's end rather than assuming it (WR-03, 999.46). /// /// This plain-call form runs ONLY if every preceding statement in its caller /// returned normally — a panic anywhere between the launch and this call @@ -564,61 +596,47 @@ fn reap_monitor_pid(pid: u32) -> bool { /// `tests::trailing_reap_call_is_skipped_when_a_later_assertion_panics`, /// which exists precisely to demonstrate this failure mode. /// -/// Any test that drives a real `launch_stage_inner` causes it to spawn a -/// detached monitor wrapper (`monitor::spawn_monitor`) that is DESIGNED to -/// outlive the call that spawned it — that is exactly what lets a real -/// `devflow start` invocation return while the monitor keeps watching the -/// agent. A test that drives the same call path gets the same detached -/// wrapper, and since nothing else in a test's lifetime ever signals it, the -/// TEST is the only thing that ever could — so the test owns reaping it. -/// -/// The pid to reap comes from `state.monitor_pid` on the same `&mut State` -/// the caller handed to `launch_stage_inner` — the only on-disk-free handle a -/// test has on what it just caused to exist (`pipeline_launch.rs` writes the -/// spawned pid there immediately after `monitor::spawn_monitor` returns). -/// This function never scans `/proc`, never guesses, and never signals a pid -/// it did not read from that same handle. -/// -/// Must be called BEFORE the caller's `TempDir` guard drops — reaping after -/// the project root has already been deleted is 999.44's reproduction shape -/// with extra steps, not a fix for it. -/// -/// Tolerates `state.monitor_pid == None` by returning quietly, with no -/// `unwrap`, no `expect` and no panic: an early-failing `launch_stage_inner` -/// clears the field before any fallible step (`pipeline_launch.rs:70`), and a -/// teardown helper that panics on that path would mask the launch's own -/// failure rather than merely fail to clean up after it. +/// Any test that drives a real `launch_stage_inner` causes it to start a +/// detached supervisor that is DESIGNED to outlive the call that started it; +/// the TEST is the only thing that ever could end it, so the test owns +/// reaping it. +/// +/// The handle comes from `state.supervisor` on the same `&State` the caller +/// handed to the launch (`spawn_agent_and_record` records the ping-confirmed +/// record there). This function never scans `/proc`, never guesses, and +/// signals only the group that record names, after its identity check. +/// +/// Must be called BEFORE the caller's `TempDir` guard drops. Tolerates a +/// state with no supervisor record by returning quietly: an early-failing +/// launch never recorded one, and a teardown helper that panics on that path +/// would mask the launch's own failure. pub(crate) fn reap_spawned_monitor(state: &State) { - let Some(pid) = state.monitor_pid else { + let Some(record) = state.supervisor.as_ref() else { return; }; assert!( - reap_monitor_pid(pid), - "monitor wrapper pid {pid}, spawned by this test's own launch_stage_inner call, must be \ - verified dead after reaping — not merely assumed dead" + reap_supervised_run(record), + "agent group {}, launched by this test's own launch_stage_inner call, must be \ + verified gone after reaping through {} — not merely assumed gone", + record.agent_pgid, + record.socket_path.display() ); } -/// RAII guard that reaps a spawned monitor wrapper on EVERY exit path of the -/// scope it is bound in — including a path on which a later assertion -/// panics. This is the fix for G-25-2 / WINDOWS.md item 3: a plain trailing -/// call to [`reap_spawned_monitor`] only runs on the success path, since Rust +/// RAII guard that reaps a launched run on EVERY exit path of the scope it +/// is bound in — including a path on which a later assertion panics. This is +/// the fix for G-25-2 / WINDOWS.md item 3: a plain trailing call to +/// [`reap_spawned_monitor`] only runs on the success path, since Rust /// abandons the rest of a function's statements the instant a panic begins /// unwinding, so it is the language's own `Drop` guarantee — not a call /// ordering convention — that makes the reap unconditional. pub(crate) struct ReapMonitorOnDrop { - pid: Option, + record: Option, } impl ReapMonitorOnDrop { - /// Capture `state.monitor_pid` BY VALUE, not `&'a State`. - /// - /// By value: at every call site this guard is bound strictly AFTER the - /// test's final `&mut state` use, so a borrow would not conflict with - /// anything at that point — but a `&'a State` guard would still tie the - /// guard's lifetime to the `State` binding's own scope, which is exactly - /// the kind of incidental coupling a teardown guard should not have. A - /// bare `Option` keeps the guard independent of `State` entirely. + /// Capture `state.supervisor` BY VALUE, not `&'a State`, so the guard + /// stays independent of the `State` binding's own scope. /// /// Named `after_launch`, not `new`: bound before the launch call /// returns, it captures `None` and silently does nothing — the @@ -626,17 +644,17 @@ impl ReapMonitorOnDrop { /// rather than leaving it to a doc comment alone. pub(crate) fn after_launch(state: &State) -> Self { Self { - pid: state.monitor_pid, + record: state.supervisor.clone(), } } } impl Drop for ReapMonitorOnDrop { fn drop(&mut self) { - let Some(pid) = self.pid else { + let Some(record) = self.record.as_ref() else { return; }; - if !reap_monitor_pid(pid) { + if !reap_supervised_run(record) { // The check below is the double-panic interlock — the ONLY // thing standing between "an assertion failed" and "the whole // test binary aborted". A `Drop` that panics while an unwind is @@ -657,13 +675,16 @@ impl Drop for ReapMonitorOnDrop { use std::io::Write as _; let _ = writeln!( std::io::stderr(), - "ReapMonitorOnDrop: monitor wrapper pid {pid} still alive after reap \ - during an unwind — not re-panicking because a panic is already in flight" + "ReapMonitorOnDrop: agent group {} still alive after reap \ + during an unwind — not re-panicking because a panic is already in flight", + record.agent_pgid ); } else { panic!( - "monitor wrapper pid {pid}, spawned by this test's own launch call, must be \ - verified dead after reaping — not merely assumed dead" + "agent group {}, launched by this test's own launch call, must be \ + verified gone after reaping through {} — not merely assumed gone", + record.agent_pgid, + record.socket_path.display() ); } } @@ -865,9 +886,13 @@ mod tests { impl ChildGuard { fn spawn() -> Self { + use std::os::unix::process::CommandExt as _; + // Its own process group, as a launched agent is (R-3), so the + // reapers' identity-confirmed group fallback has a real subject. Self( Command::new("sleep") .arg("300") + .process_group(0) .spawn() .expect("sleep must be spawnable to run this test"), ) @@ -885,9 +910,11 @@ mod tests { } } - /// A `State` whose only relevant field is `monitor_pid`. No I/O occurs — - /// the project root is never read, since nothing in the reap path - /// touches disk. + /// A `State` whose only relevant field is `supervisor`: a record whose + /// socket does not exist, naming `pid` (its own group leader) with its + /// real start time, so the reapers' shutdown finds nothing to ask and + /// they reach the identity-confirmed group fallback. No I/O occurs + /// beyond reading `pid`'s start time and boot id. fn state_holding(pid: u32) -> State { let mut state = State::new( PhaseId::new(0), @@ -895,7 +922,13 @@ mod tests { Mode::Auto, PathBuf::from("/nonexistent"), ); - state.monitor_pid = Some(pid); + state.supervisor = Some(SupervisorRecord::new( + PathBuf::from("/nonexistent/reap-fixture.sock"), + "reapfixture00000".to_string(), + pid, + devflow_core::agent::process_start_time(pid).expect("the subject's start time"), + devflow_core::supervisor::boot_id().unwrap_or_default(), + )); state } diff --git a/crates/devflow-cli/tests/auto_chain_flag_e2e.rs b/crates/devflow-cli/tests/auto_chain_flag_e2e.rs index d1487f9f..9dcc5d06 100644 --- a/crates/devflow-cli/tests/auto_chain_flag_e2e.rs +++ b/crates/devflow-cli/tests/auto_chain_flag_e2e.rs @@ -202,6 +202,16 @@ impl Fixture { /// Run the REAL binary's hidden `__monitor` subcommand over `argv`. fn run_monitor(&self, argv: &[&str]) -> Output { + // The 3.x `__monitor` argv (50-11): a per-launch id and a socket + // under the hermetic cache dir this file's child process gets. + let launch_id = devflow_core::supervisor::new_launch_id(); + let socket = devflow_core::supervisor::SocketPath::for_launch( + &self.home.cache_dir(), + &self.root, + self.phase, + &launch_id, + ) + .expect("the hermetic socket path fits sun_path"); common::hermetic_devflow(&self.home) .arg("__monitor") .arg("--project") @@ -216,6 +226,12 @@ impl Fixture { .arg("30") .arg("--agent") .arg("claude") + .arg("--launch") + .arg("stream") + .arg("--launch-id") + .arg(&launch_id) + .arg("--socket") + .arg(socket.as_path()) .arg("--") .args(argv) .output() @@ -325,7 +341,7 @@ fn supervise_mode_code_stage_child_observes_the_flag_clear() { } /// The in-process half of criterion 2: the flag is cleared even when the run -/// dies before `advance` is ever reached — `run_pipe_owning_monitor`'s +/// dies before `advance` is ever reached — `run_supervised_stage`'s /// early-return `Err` path, driven here by a child program that cannot be /// spawned at all. (`35.1-02` owns the kill half.) /// diff --git a/crates/devflow-cli/tests/auto_chain_leak_repair_e2e.rs b/crates/devflow-cli/tests/auto_chain_leak_repair_e2e.rs index 24765ad4..c60090f5 100644 --- a/crates/devflow-cli/tests/auto_chain_leak_repair_e2e.rs +++ b/crates/devflow-cli/tests/auto_chain_leak_repair_e2e.rs @@ -254,6 +254,16 @@ impl Fixture { /// sleeper, as a child of this test process so it can be signalled /// directly. fn spawn_monitor(&self) -> Child { + // The 3.x `__monitor` argv (50-11): a per-launch id and a socket + // under the hermetic cache dir this file's child process gets. + let launch_id = devflow_core::supervisor::new_launch_id(); + let socket = devflow_core::supervisor::SocketPath::for_launch( + &self.home.cache_dir(), + &self.root, + self.phase, + &launch_id, + ) + .expect("the hermetic socket path fits sun_path"); let child = common::hermetic_devflow(&self.home) .arg("__monitor") .arg("--project") @@ -270,6 +280,12 @@ impl Fixture { .arg("120") .arg("--agent") .arg("claude") + .arg("--launch") + .arg("stream") + .arg("--launch-id") + .arg(&launch_id) + .arg("--socket") + .arg(socket.as_path()) .arg("--") .arg("sh") .arg(&self.script) diff --git a/crates/devflow-cli/tests/common/mod.rs b/crates/devflow-cli/tests/common/mod.rs index df7d9515..6727f967 100644 --- a/crates/devflow-cli/tests/common/mod.rs +++ b/crates/devflow-cli/tests/common/mod.rs @@ -144,10 +144,17 @@ pub fn kill_and_reap(child: &mut Child) { let _ = child.wait(); } -/// Terminates, on drop, the monitor recorded in the phase state and the -/// agent named in the agent-pid file, unless its pid is in `spare`. Bind it -/// before (or right after) the spawn that launches a run, and keep the -/// run's [`HermeticHome`] alive until this has dropped. +/// How long [`LaunchedRunReaper`] waits for a supervisor's final shutdown line. +const REAP_SHUTDOWN_WAIT: Duration = Duration::from_secs(15); + +/// Ends, on drop, the run recorded in the phase state, through its +/// supervisor's socket (D-07): a `shutdown` request first; then, if the +/// recorded agent group's leader is still the process the record names +/// (`agent::is_same_process` against its start time), a SIGKILL to that +/// group. Only when the state holds no `supervisor` record does it fall back +/// to the agent named in the agent-pid file. A pid in `spare` is never +/// signalled. Bind it before (or right after) the spawn that launches a run, +/// and keep the run's [`HermeticHome`] alive until this has dropped. pub struct LaunchedRunReaper { root: PathBuf, phase: PhaseId, @@ -166,26 +173,53 @@ impl LaunchedRunReaper { impl Drop for LaunchedRunReaper { fn drop(&mut self) { - let monitor = devflow_core::workflow::load_state(&self.root, self.phase) + let record = devflow_core::workflow::load_state(&self.root, self.phase) .ok() - .and_then(|state| state.monitor_pid); + .and_then(|state| state.supervisor); + if let Some(record) = record { + let _ = + devflow_core::supervisor::request_shutdown(&record.socket_path, REAP_SHUTDOWN_WAIT); + let leader = record.agent_pgid; + if !self.spare.contains(&leader) + && devflow_core::agent::agent_running(leader) + && devflow_core::agent::is_same_process(leader, record.agent_start_time) + { + signal_group(leader, libc::SIGKILL); + } + return; + } let agent = fs::read_to_string(devflow_core::agent_result::agent_pid_path( &self.root, self.phase, )) .ok() .and_then(|pid| pid.trim().parse::().ok()); - for pid in [monitor, agent].into_iter().flatten() { - if !self.spare.contains(&pid) && devflow_core::agent::agent_running(pid) { - let _ = devflow_core::agent::terminate_and_verify( - pid, - devflow_core::agent::TERMINATE_VERIFY_WAIT, - devflow_core::agent::TERMINATE_VERIFY_POLL, - ); - } + if let Some(pid) = agent + && !self.spare.contains(&pid) + && devflow_core::agent::agent_running(pid) + { + let _ = devflow_core::agent::terminate_and_verify( + pid, + devflow_core::agent::TERMINATE_VERIFY_WAIT, + devflow_core::agent::TERMINATE_VERIFY_POLL, + ); } } } +/// Send `signal` to process group `pgid`. Refuses pgid 0 or 1, a value that +/// does not fit `pid_t`, and this process's own group. +pub fn signal_group(pgid: u32, signal: libc::c_int) { + let Ok(signed) = libc::pid_t::try_from(pgid) else { + return; + }; + // SAFETY: getpgrp has no preconditions and cannot fail. + if signed > 1 && signed != unsafe { libc::getpgrp() } { + // SAFETY: `signed > 1` and not our own group: one process group + // whose leader the caller has just identity-checked. + unsafe { libc::kill(-signed, signal) }; + } +} + /// Every file below `dir`, recursively, sorted. Empty when `dir` is missing. pub fn entries_under(dir: &Path) -> Vec { let mut files = Vec::new(); From b82f138c5a1bf8081933dc3de2b7441e6cca95fb Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 16:26:41 -0400 Subject: [PATCH 051/112] test(50-11): prove the cache-dir socket, CLI SIGTERM stop and capture result - V-07: a launch records state.supervisor.socket_path under its own DEVFLOW_CACHE_DIR/sock (control: a second cache dir), and the first child runs monitor_entry_full, whose advance names the launched stage - e2e: SIGTERM to the supervisor named in its sidecar ends the run for both shapes (group gone, state.stopped, no further stage_launched, socket Gone); control: a finishing agent advances to Plan - re-home the capture-result check from devflow-core's monitor_e2e onto the real binary; monitor_e2e keeps only its non-spawning test - the --legacy-claude-launch doc no longer names the deleted sh monitor Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/main.rs | 4 +- crates/devflow-cli/src/pipeline_launch.rs | 176 +++++++++++++ crates/devflow-cli/tests/supervisor_e2e.rs | 289 ++++++++++++++++++++- crates/devflow-core/tests/monitor_e2e.rs | 140 +--------- 4 files changed, 474 insertions(+), 135 deletions(-) diff --git a/crates/devflow-cli/src/main.rs b/crates/devflow-cli/src/main.rs index bdbe7f41..82795e5d 100644 --- a/crates/devflow-cli/src/main.rs +++ b/crates/devflow-cli/src/main.rs @@ -90,8 +90,8 @@ enum Command { #[arg(long)] yes_ship: bool, /// Force the pre-31 single-document Claude launch: the prompt - /// positionally in argv, `--output-format json`, and the `sh` monitor - /// (D-11, `31-CONTEXT.md`). Off by default. + /// positionally in argv, `--output-format json`, and the capture-shape + /// supervisor (D-11, `31-CONTEXT.md`). Off by default. /// /// It exists so an operator can recover from a defect in the /// `stream-json` transport without waiting for a release. Its use is diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index 02627941..1902525d 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -2452,6 +2452,182 @@ mod tests { } } + /// Names the file a V-07 child writes its recorded socket path to. + const V07_OUT_ENV: &str = "DEVFLOW_TEST_V07_OUT"; + + /// A `claude` stand-in that answers `--version`, then waits for `go` in + /// its working directory (the project root) and exits 0, with `sleep` + /// linked beside it so the wait works under the replaced PATH. + fn go_waiting_claude_dir() -> tempfile::TempDir { + let path_dir = agent_free_dir_with_agent_stub("claude"); + std::fs::write( + path_dir.path().join("claude"), + "#!/bin/sh\ncase \"$1\" in --version) exit 0 ;; esac\n\ + while [ ! -e go ]; do sleep 0.05; done\nexit 0\n", + ) + .unwrap(); + let real_sleep = ["/usr/bin/sleep", "/bin/sleep"] + .into_iter() + .map(Path::new) + .find(|path| path.is_file()) + .expect("sleep must exist to run this test"); + std::os::unix::fs::symlink(real_sleep, path_dir.path().join("sleep")).unwrap(); + path_dir + } + + /// V-07 (criterion 3): a launch persists `state.supervisor.socket_path` + /// under its context's cache dir (`DEVFLOW_CACHE_DIR/sock/`), never under + /// the project root, and the reloaded state holds exactly the published + /// record. Control: the same launch in a child with another cache dir + /// records a path under THAT dir, so the path follows the context. + /// + /// The first child also runs the advance-running harness entry + /// (`test_support::monitor_entry_full`, `DEVFLOW_TEST_MONITOR_FULL=1`) and + /// proves it is reached: with the state moved on to Plan while the agent + /// runs, its in-process advance records `advance_failed` naming the + /// launched stage (`define`). The second child, on the supervise-only + /// entry, must record no advance event at all. + #[test] + fn launch_publishes_the_socket_path_under_the_cache_dir_in_state() { + const NAME: &str = + "pipeline_launch::tests::launch_publishes_the_socket_path_under_the_cache_dir_in_state"; + if !devflow_core::test_support::in_child_test(NAME) { + let mut recorded = Vec::new(); + for full in [true, false] { + let path_dir = go_waiting_claude_dir(); + let cache = crate::test_support::hermetic_cache_dir(); + let out = path_dir.path().join("v07-socket-path"); + let mut env: Vec<(&str, &std::ffi::OsStr)> = vec![ + ("DEVFLOW_CACHE_DIR", cache.path().as_os_str()), + (V07_OUT_ENV, out.as_os_str()), + ]; + if full { + env.push((TEST_MONITOR_FULL_ENV, std::ffi::OsStr::new("1"))); + } + let output = + devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &env); + devflow_core::test_support::assert_child_ran_exactly_one_passing_test( + &output, NAME, + ); + let socket = PathBuf::from(std::fs::read_to_string(&out).unwrap()); + assert!( + socket.starts_with(cache.path().join("sock")), + "{} must be under {}", + socket.display(), + cache.path().display() + ); + recorded.push((cache, socket)); + } + let (a, b) = (&recorded[0], &recorded[1]); + assert_ne!(a.0.path(), b.0.path(), "two distinct cache dirs"); + assert!( + !a.1.starts_with(b.0.path()) && !b.1.starts_with(a.0.path()), + "control: each launch's socket follows its own cache dir: {} and {}", + a.1.display(), + b.1.display() + ); + return; + } + let _guard = env_lock(); + let cache = PathBuf::from( + std::env::var_os("DEVFLOW_CACHE_DIR").expect("the parent passes DEVFLOW_CACHE_DIR"), + ); + let out = + PathBuf::from(std::env::var_os(V07_OUT_ENV).expect("the parent passes the out file")); + let full = std::env::var_os(TEST_MONITOR_FULL_ENV).is_some_and(|value| value == "1"); + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + init_repo(root); + let phase = PhaseId::new(1107); + let mut state = State::new( + phase, + AgentKind::Claude, + Mode::Supervise, + root.to_path_buf(), + ); + // The capture shape, for the same reason as + // `launch_stage_persists_monitor_pid_for_reload`: no delivery canary. + state.legacy_claude_launch = true; + workflow::save_state(&state).unwrap(); + + let result = launch_stage(&mut state, None, None); + let _reap_guard = ReapMonitorOnDrop::after_launch(&state); + result.unwrap(); + + let record = state + .supervisor + .clone() + .expect("the launch records its supervisor"); + assert!( + record.socket_path.starts_with(cache.join("sock")), + "{} must be under {}/sock", + record.socket_path.display(), + cache.display() + ); + assert!( + !record.socket_path.starts_with(root), + "the socket must not live in the project tree" + ); + assert!(record.socket_path.as_os_str().len() <= 107); + let reloaded = workflow::load_state(root, phase).unwrap(); + assert_eq!( + reloaded.supervisor.as_ref(), + Some(&record), + "the reloaded state holds exactly the published record" + ); + match devflow_core::supervisor::probe(&record.socket_path).unwrap() { + devflow_core::supervisor::Liveness::Alive(reply) => { + assert_eq!(reply.launch_id, record.launch_id); + assert_eq!(reply.launch, "capture"); + } + other => panic!("the recorded socket must answer ALIVE, got {other:?}"), + } + std::fs::write(&out, record.socket_path.display().to_string()).unwrap(); + + let events_path = devflow_core::events::events_path(root); + let advance_events = || { + std::fs::read_to_string(&events_path) + .unwrap_or_default() + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|event| { + event["event"] + .as_str() + .is_some_and(|name| name.starts_with("advance_")) + }) + .collect::>() + }; + if full { + let mut moved = workflow::load_state(root, phase).unwrap(); + moved.stage = Stage::Plan; + workflow::save_state(&moved).unwrap(); + } + std::fs::write(root.join("go"), "").unwrap(); + let gone = poll_until(Duration::from_secs(30), || { + devflow_core::supervisor::probe(&record.socket_path) + .is_ok_and(|liveness| liveness == devflow_core::supervisor::Liveness::Gone) + }); + assert!(gone, "the supervisor finishes once its agent exits"); + let events = advance_events(); + if full { + let failed = events + .iter() + .find(|event| event["event"] == "advance_failed") + .unwrap_or_else(|| panic!("monitor_entry_full must run the advance: {events:?}")); + assert_eq!(failed["reason"], "stage mismatch", "{failed}"); + assert_eq!( + failed["expected"], "define", + "the advance is passed the launched stage: {failed}" + ); + } else { + assert!( + events.is_empty(), + "control: the supervise-only entry never advances: {events:?}" + ); + } + } + /// A `claude` stand-in that leaves `agent-ran` in its working directory /// (the project root) and exits 0 without reading its stdin. fn marker_writing_claude_dir() -> tempfile::TempDir { diff --git a/crates/devflow-cli/tests/supervisor_e2e.rs b/crates/devflow-cli/tests/supervisor_e2e.rs index 03069e5e..e69e6975 100644 --- a/crates/devflow-cli/tests/supervisor_e2e.rs +++ b/crates/devflow-cli/tests/supervisor_e2e.rs @@ -20,7 +20,7 @@ use devflow_core::supervisor::{self, Liveness, ShutdownOutcome}; use std::fs; use std::path::Path; use std::process::Output; -use std::time::Duration; +use std::time::{Duration, Instant}; /// How long a shutdown waits for the supervisor's final line. const SHUTDOWN_WAIT: Duration = Duration::from_secs(30); @@ -105,6 +105,18 @@ fn run_start( phase: PhaseId, fake_bin: &common::FakeBin, shape: Shape, +) -> Output { + run_start_with(home, root, phase, fake_bin, shape, &[]) +} + +/// [`run_start`] with `extra` start flags (for example `--until plan`). +fn run_start_with( + home: &common::HermeticHome, + root: &Path, + phase: PhaseId, + fake_bin: &common::FakeBin, + shape: Shape, + extra: &[&str], ) -> Output { let existing = std::env::var_os("PATH").unwrap_or_default(); let path = format!("{}:{}", fake_bin.path.display(), existing.to_string_lossy()); @@ -120,6 +132,7 @@ fn run_start( "--no-worktree", ]) .args(shape.start_flags()) + .args(extra) .arg(root) .current_dir(root) .env("PATH", path) @@ -216,3 +229,277 @@ fn start_publishes_a_live_supervisor_socket_for_each_launch_shape() { start_publishes_for(Shape::Stream, PhaseId::new(61)); start_publishes_for(Shape::Capture, PhaseId::new(62)); } + +/// How long the tests below wait for a supervisor to act on a signal, or a +/// stage chain to reach its stop point. +const RUN_WAIT: Duration = Duration::from_secs(60); + +/// The stop reason `run_monitor` records for a run its supervisor stopped. +const SUPERVISOR_STOP_REASON: &str = + "stopped through the supervisor (socket shutdown or SIGTERM/SIGINT)"; + +/// A fake `claude` that answers `--version` and the canary like +/// [`WAITING_CLAUDE`], commits one empty commit as its "work", prints the +/// success marker and exits 0 on its own. +const FINISHING_CLAUDE: &str = r#"if [ "$1" = "--version" ]; then echo "2.1.220 (Claude Code)"; exit 0; fi +read -r turn +case "$turn" in + *DEVFLOW_DELIVERY_CANARY_*) + token=$(printf '%s' "$turn" | grep -o 'DEVFLOW_DELIVERY_CANARY_[0-9a-f]*' | head -1) + printf '{"type":"result","subtype":"success","is_error":false,"session_id":"s-fake","result":"%s"}\n' "$token" + exit 0 + ;; +esac +git commit -q --allow-empty -m "fake stage work" +printf 'work done\nDEVFLOW_RESULT: {"status":"success","commits":1}\n' +exit 0"#; + +/// The `stage_launched` events recorded for `phase`. +fn stage_launched_count(root: &Path, phase: PhaseId) -> usize { + fs::read_to_string(devflow_core::events::events_path(root)) + .unwrap_or_default() + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|event| { + phase.matches_json(event.get("phase")) + && event["event"].as_str() == Some("stage_launched") + }) + .count() +} + +/// The pids whose process group (`/proc//stat` field 5) is `pgid`. +fn group_members(pgid: u32) -> Vec { + let Ok(entries) = fs::read_dir("/proc") else { + return Vec::new(); + }; + entries + .filter_map(|entry| entry.ok()?.file_name().to_str()?.parse::().ok()) + .filter(|pid| { + fs::read_to_string(format!("/proc/{pid}/stat")) + .ok() + .and_then(|stat| { + let rest = &stat[stat.rfind(')')? + 1..]; + rest.split_whitespace().nth(2)?.parse::().ok() + }) + == Some(pgid) + }) + .collect() +} + +/// Poll `done` every 25 ms until it holds or `within` elapses. +fn holds_within(within: Duration, mut done: impl FnMut() -> bool) -> bool { + let deadline = Instant::now() + within; + loop { + if done() { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(25)); + } +} + +/// D-05 at the CLI level (999.134) for one shape: SIGTERM to the live +/// supervisor named in its sidecar ends the run. The agent group is gone, +/// `state.stopped` carries the supervisor stop reason, no further +/// `stage_launched` is recorded, and the socket is Gone. +fn sigterm_stops_for(shape: Shape, phase: PhaseId) { + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path().canonicalize().unwrap(); + init_repo(&root, phase); + let fake_bin = common::fake_bin_dir(WAITING_CLAUDE); + let _reaper = common::LaunchedRunReaper::new(&root, phase, vec![]); + + let output = run_start(&home, &root, phase, &fake_bin, shape); + assert!( + output.status.success(), + "{shape:?}: start failed\nstderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let record = devflow_core::workflow::load_state(&root, phase) + .expect("start saved state") + .supervisor + .unwrap_or_else(|| panic!("{shape:?}: start must record state.supervisor")); + let socket = record.socket_path.clone(); + let sidecar = supervisor::read_sidecar(&socket.with_extension("json")).expect("sidecar"); + assert_eq!( + stage_launched_count(&root, phase), + 1, + "{shape:?}: one launch" + ); + assert!( + !group_members(record.agent_pgid).is_empty(), + "{shape:?}: the agent group is running before the signal" + ); + + // The pid to signal comes from the sidecar and is confirmed against its + // recorded start time first, so a recycled pid is never signalled. + let pid = sidecar.supervisor_pid; + assert!( + devflow_core::agent::is_same_process(pid, sidecar.supervisor_start_time), + "{shape:?}: the sidecar's supervisor {pid} is the recorded process" + ); + let signed = libc::pid_t::try_from(pid).expect("pid fits pid_t"); + assert!(signed > 1, "refusing to signal pid {signed}"); + // SAFETY: `signed > 1` names the supervisor just identity-checked. + assert_eq!( + unsafe { libc::kill(signed, libc::SIGTERM) }, + 0, + "SIGTERM {pid}" + ); + + assert!( + holds_within(RUN_WAIT, || !devflow_core::agent::agent_running(pid)), + "{shape:?}: the supervisor {pid} exits after SIGTERM" + ); + assert_eq!( + supervisor::probe(&socket).expect("probe after the signal"), + Liveness::Gone, + "{shape:?}: the socket is gone" + ); + assert_eq!( + group_members(record.agent_pgid), + Vec::::new(), + "{shape:?}: the agent group {} is gone", + record.agent_pgid + ); + let state = devflow_core::workflow::load_state(&root, phase).expect("reload state"); + assert!(state.stopped, "{shape:?}: state.stopped after the signal"); + assert_eq!( + state.stop_reason.as_deref(), + Some(SUPERVISOR_STOP_REASON), + "{shape:?}: the supervisor stop reason" + ); + assert_eq!( + stage_launched_count(&root, phase), + 1, + "{shape:?}: no stage_launched after the signal" + ); +} + +/// The control for [`sigterm_stops_for`]: the same launch, whose fake agent +/// exits 0 on its own, advances in-process and launches the next stage, so a +/// following `stage_launched` names Plan (the launched stage was passed to the +/// advance). `--until plan` stops the chain at the Plan boundary. +fn finishing_run_advances_for(shape: Shape, phase: PhaseId) { + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path().canonicalize().unwrap(); + init_repo(&root, phase); + let fake_bin = common::fake_bin_dir(FINISHING_CLAUDE); + let _reaper = common::LaunchedRunReaper::new(&root, phase, vec![]); + + let output = run_start_with(&home, &root, phase, &fake_bin, shape, &["--until", "plan"]); + assert!( + output.status.success(), + "{shape:?}: start failed\nstderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + holds_within(RUN_WAIT, || stage_launched_count(&root, phase) >= 2), + "{shape:?} control: a completed agent must advance and launch the next stage\n{}", + fs::read_to_string(devflow_core::events::events_path(&root)).unwrap_or_default() + ); + let events = fs::read_to_string(devflow_core::events::events_path(&root)).unwrap(); + let launched: Vec = events + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|event| event["event"].as_str() == Some("stage_launched")) + .collect(); + assert_eq!(launched[0]["stage"], "define", "{shape:?}: {launched:?}"); + assert_eq!(launched[1]["stage"], "plan", "{shape:?}: {launched:?}"); + assert_eq!( + launched[1]["launch"], + shape.label(), + "{shape:?}: {launched:?}" + ); + assert!( + holds_within(RUN_WAIT, || devflow_core::workflow::load_state( + &root, phase + ) + .is_ok_and(|state| state.stopped)), + "{shape:?} control: the chain stops at --until plan" + ); +} + +#[test] +fn sigterm_to_a_live_supervisor_ends_the_run_without_advancing() { + sigterm_stops_for(Shape::Stream, PhaseId::new(63)); + sigterm_stops_for(Shape::Capture, PhaseId::new(64)); + finishing_run_advances_for(Shape::Stream, PhaseId::new(65)); + finishing_run_advances_for(Shape::Capture, PhaseId::new(66)); +} + +/// Re-homed from `devflow-core/tests/monitor_e2e.rs` (50-11): a library test +/// binary cannot re-exec `__monitor`, so the capture-result check runs on the +/// real binary. A capture-shape run whose fake agent prints a +/// `DEVFLOW_RESULT` marker and exits 0: Layers 1–2 evaluate to Success from +/// the files the supervisor wrote, and the supervisor's own advance recorded +/// the same verdict. +#[test] +fn capture_supervisor_owns_a_fake_agent_and_records_devflow_result() { + let phase = PhaseId::new(67); + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path().canonicalize().unwrap(); + init_repo(&root, phase); + let fake_bin = common::fake_bin_dir(FINISHING_CLAUDE); + let _reaper = common::LaunchedRunReaper::new(&root, phase, vec![]); + + let output = run_start_with( + &home, + &root, + phase, + &fake_bin, + Shape::Capture, + &["--until", "define"], + ); + assert!( + output.status.success(), + "start failed\nstderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + holds_within(RUN_WAIT, || devflow_core::workflow::load_state( + &root, phase + ) + .is_ok_and(|state| state.stopped)), + "the run stops at --until define once its supervisor advanced" + ); + + let agent_pid: u32 = + fs::read_to_string(devflow_core::agent_result::agent_pid_path(&root, phase)) + .expect("the supervisor records the agent pid") + .trim() + .parse() + .expect("a pid"); + assert!(agent_pid > 0); + let stdout = fs::read_to_string(devflow_core::agent_result::stdout_path(&root, phase)).unwrap(); + assert!( + stdout.contains("DEVFLOW_RESULT"), + "captured stdout missing marker: {stdout:?}" + ); + let exit = fs::read_to_string(devflow_core::agent_result::exit_code_path(&root, phase)) + .expect("the supervisor records the exit code"); + assert_eq!(exit.trim(), "0"); + + let state = devflow_core::workflow::load_state(&root, phase).unwrap(); + let git_flow = devflow_core::config::git_flow_for_run(&root, state.base_branch.as_deref()); + let result = devflow_core::agent_result::evaluate_agent_result(&root, &state, &git_flow) + .expect("evaluate the supervisor's files"); + assert_eq!( + result.status, + devflow_core::agent_result::AgentStatus::Success, + "{result:?}" + ); + let events = fs::read_to_string(devflow_core::events::events_path(&root)).unwrap(); + assert!( + events + .lines() + .any(|line| line.contains("\"advance_evaluated\"") + && line.contains("\"status\":\"success\"")), + "the supervisor's in-process advance evaluated the stage as success:\n{events}" + ); +} diff --git a/crates/devflow-core/tests/monitor_e2e.rs b/crates/devflow-core/tests/monitor_e2e.rs index f4a17f38..403348e6 100644 --- a/crates/devflow-core/tests/monitor_e2e.rs +++ b/crates/devflow-core/tests/monitor_e2e.rs @@ -1,139 +1,15 @@ -//! End-to-end monitor integration test. +//! Monitor integration tests that need no running supervisor. //! -//! Spawns a real background monitor that owns a *fake* agent (a `sh` script -//! that prints a `DEVFLOW_RESULT` marker and exits 0), then asserts the monitor -//! captured everything `devflow check` needs to advance the state machine. -//! -//! Boundary note: the real `devflow` binary is not built inside the library's -//! integration-test harness, so the monitor's tail `devflow check` self-call is -//! a no-op here (it re-invokes this test binary with a non-matching filter). -//! We therefore assert the **capture/result** side — that Layers 1–2 evaluate -//! to `Success` from the files the monitor wrote — rather than the state -//! transition itself. +//! Boundary note (50-11): a library integration-test binary cannot re-exec +//! the `devflow` binary's hidden `__monitor` subcommand, so no test here can +//! start a supervisor. The end-to-end capture-result check that lived here +//! (`monitor_owns_fake_agent_and_records_devflow_result`) moved to +//! `crates/devflow-cli/tests/supervisor_e2e.rs` as +//! `capture_supervisor_owns_a_fake_agent_and_records_devflow_result`, which +//! drives the real binary. What remains is the advance-side state loading. -use devflow_core::agent_result::{ - AgentStatus, agent_pid_path, evaluate_agent_result, exit_code_path, stdout_path, -}; -use devflow_core::config::GitFlowConfig; -use devflow_core::mode::Mode; -use devflow_core::monitor::{ - LaunchContext, MonitorExec, MonitorLaunch, spawn_monitor, wait_for_agent_pid, -}; use devflow_core::phase_id::PhaseId; -use devflow_core::stage::Stage; -use devflow_core::state::{AgentKind, State}; use devflow_core::workflow::{WorkflowError, load_state}; -use std::path::Path; -use std::time::Duration; - -fn git(root: &Path, args: &[&str]) { - // Hermetic: pinning cwd alone does not stop an inherited GIT_DIR from - // retargeting the real repository (999.37). - let output = devflow_core::test_support::git_command(root) - .args(args) - .output() - .expect("spawn git"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); -} - -/// Init a temp repo with `develop` and a `feature/phase-NN` branch holding one -/// commit, hooks disabled for isolation. -fn init_repo(root: &Path, phase: PhaseId) { - git(root, &["init", "-q"]); - git(root, &["config", "user.email", "devflow@example.com"]); - git(root, &["config", "user.name", "DevFlow Tests"]); - git(root, &["config", "commit.gpgsign", "false"]); - git(root, &["config", "core.hooksPath", "/dev/null"]); - git(root, &["checkout", "-q", "-b", "develop"]); - std::fs::write(root.join("README.md"), "base\n").unwrap(); - git(root, &["add", "README.md"]); - git(root, &["commit", "-q", "-m", "base"]); - - let branch = format!("feature/phase-{padded}", padded = phase.padded()); - git(root, &["checkout", "-q", "-b", &branch]); - std::fs::write(root.join("work.txt"), "agent work\n").unwrap(); - git(root, &["add", "work.txt"]); - git(root, &["commit", "-q", "-m", "agent work"]); -} - -/// The launch context this suite passes `spawn_monitor` (50-10): this test -/// binary as the monitor executable, which is what `spawn_monitor` resolved -/// itself before it took a context, and a cache dir under the test's tempdir. -fn launch_context(root: &Path) -> LaunchContext { - LaunchContext::new( - root.join("cache"), - MonitorExec::Subcommand { - exe: std::env::current_exe().unwrap(), - }, - ) -} - -#[test] -fn monitor_owns_fake_agent_and_records_devflow_result() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path(); - let phase = PhaseId::new(7); - init_repo(root, phase); - - let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); - state.stage = Stage::Code; - - // Fake agent: emit the success marker on stdout, then exit 0. - let args = vec![ - "-c".to_string(), - "printf 'work done\\nDEVFLOW_RESULT: {\"status\":\"success\",\"commits\":1}\\n'" - .to_string(), - ]; - - spawn_monitor( - &state, - "sh", - &args, - &[], - MonitorLaunch::Legacy, - &launch_context(root), - ) - .expect("spawn monitor"); - - // The monitor records the agent PID. - let agent_pid = wait_for_agent_pid(root, phase).expect("agent pid recorded"); - assert!(agent_pid > 0); - - // Wait until the monitor writes the exit-code file (agent finished). - let exit_path = exit_code_path(root, phase); - let mut exit_seen = false; - for _ in 0..200 { - if exit_path.exists() { - exit_seen = true; - break; - } - std::thread::sleep(Duration::from_millis(20)); - } - assert!(exit_seen, "monitor never wrote the exit-code file"); - - // Captured stdout contains the marker. - let stdout = std::fs::read_to_string(stdout_path(root, phase)).unwrap(); - assert!( - stdout.contains("DEVFLOW_RESULT"), - "captured stdout missing marker: {stdout:?}" - ); - - // Exit code recorded as 0. - let exit = std::fs::read_to_string(&exit_path).unwrap(); - assert_eq!(exit.trim(), "0"); - - // The pid file exists (the monitor created the capture directory + files). - assert!(agent_pid_path(root, phase).exists()); - - // End-to-end: the layered evaluator reads the monitor's files and reports - // Success — exactly what `devflow check` would act on to advance state. - let result = evaluate_agent_result(root, &state, &GitFlowConfig::default()).unwrap(); - assert_eq!(result.status, AgentStatus::Success); -} #[test] fn advance_state_loading_fails_cleanly_for_missing_and_corrupt_state() { From ab5845f67408238dccb731a6bb5cba591edf3243 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 16:42:49 -0400 Subject: [PATCH 052/112] test(50-12): add failing tests for probe-answered status liveness - liveness_matrix_covers_every_probe_outcome over a stub phase_liveness - status_reports_each_liveness_state_with_its_repair_verb e2e: ALIVE, STALE after a SIGKILLed supervisor, GONE after the socket is removed Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 119 +++++++++++++++++++++ crates/devflow-cli/tests/supervisor_e2e.rs | 107 ++++++++++++++++++ 2 files changed, 226 insertions(+) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 09e315a4..f08b4556 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -1026,6 +1026,41 @@ fn liveness(monitor_pid: Option, monitor_alive: bool, agent_alive: bool) -> } } +/// A phase's supervisor liveness as `status`, `cleanup` and `doctor` report +/// it (50-12): answered from the socket recorded in `state.supervisor` +/// through [`supervisor::probe`], never from a pid. +#[derive(Debug, Clone, PartialEq, Eq)] +#[allow(dead_code)] // RED stub (50-12 Task 1) +pub(crate) enum PhaseLiveness { + /// The recorded socket answered a ping. + Alive { + /// Who owns the run right now, from the ping reply. + owner: String, + /// The launch shape, from the ping reply. + launch: String, + }, + /// The socket file remains but nothing listens on it. + Stale, + /// No socket at the recorded path, and the phase is stopped. + Gone, + /// No socket at the recorded path, but the phase is not stopped. + GoneWhileActive, + /// The state records no supervisor. + NotLaunched, + /// The probe could not answer; a path-free summary of why. + ProbeFailed(String), +} + +/// RED stub (50-12 Task 1): answers `NotLaunched` for every input. +#[allow(dead_code)] +fn phase_liveness( + _record: Option<&devflow_core::state::SupervisorRecord>, + _probe: Option>, + _stopped: bool, +) -> PhaseLiveness { + PhaseLiveness::NotLaunched +} + /// Recovery verbs discoverable from a phase's liveness (21a, D-03) — the /// pure, testable counterpart to `status`'s old inline Stuck `println!`. /// Always includes `devflow resume` for `Stuck`; additionally includes @@ -5478,6 +5513,90 @@ mod tests { assert_eq!(liveness(pid, false, true), Liveness::Stuck); } + /// A ping reply as a supervisor sends it; built through serde because + /// `PingReply` is `#[non_exhaustive]`. + fn ping_reply(owner: &str, launch: &str) -> devflow_core::supervisor::protocol::PingReply { + serde_json::from_value(serde_json::json!({ + "launch_id": "0123456789abcdef", + "phase": "7", + "stage": "code", + "launch": launch, + "owner": owner, + "agent_pid": 10, + "agent_pgid": 10, + "agent_start_time": 20, + "boot_id": null, + "supervisor_pid": 11, + })) + .expect("a complete ping reply") + } + + /// 50-12: the pure mapping over every probe outcome, with and without + /// `stopped`. Each row of one `stopped` value is a distinct answer, and + /// GONE splits by `stopped`: an abandoned active phase is not a stopped + /// one. + #[test] + fn liveness_matrix_covers_every_probe_outcome() { + use devflow_core::supervisor::{Liveness as Probe, ProbeError}; + let record = devflow_core::state::SupervisorRecord::new( + PathBuf::from("/cache/sock/x.sock"), + "0123456789abcdef".to_string(), + 10, + 20, + "boot".to_string(), + ); + for stopped in [false, true] { + let rows = vec![ + phase_liveness(None, None, stopped), + phase_liveness(Some(&record), Some(Ok(Probe::Gone)), stopped), + phase_liveness(Some(&record), Some(Ok(Probe::Stale)), stopped), + phase_liveness( + Some(&record), + Some(Ok(Probe::Alive(ping_reply("agent_running", "stream")))), + stopped, + ), + phase_liveness( + Some(&record), + Some(Err(ProbeError::NoReply("timed out".to_string()))), + stopped, + ), + ]; + assert_eq!(rows[0], PhaseLiveness::NotLaunched, "stopped={stopped}"); + let gone = if stopped { + PhaseLiveness::Gone + } else { + PhaseLiveness::GoneWhileActive + }; + assert_eq!(rows[1], gone, "stopped={stopped}"); + assert_eq!(rows[2], PhaseLiveness::Stale, "stopped={stopped}"); + assert_eq!( + rows[3], + PhaseLiveness::Alive { + owner: "agent_running".to_string(), + launch: "stream".to_string(), + }, + "stopped={stopped}" + ); + match &rows[4] { + PhaseLiveness::ProbeFailed(why) => assert!( + why.contains("no valid ping reply") && !why.contains('/'), + "a path-free probe failure, got {why:?}" + ), + other => panic!("a probe error must be ProbeFailed, got {other:?}"), + } + for (at, row) in rows.iter().enumerate() { + for other in &rows[at + 1..] { + assert_ne!(row, other, "stopped={stopped}: two outcomes share one answer"); + } + } + } + assert_ne!( + phase_liveness(Some(&record), Some(Ok(Probe::Gone)), true), + phase_liveness(Some(&record), Some(Ok(Probe::Gone)), false), + "GONE while stopped must differ from GONE while active" + ); + } + /// 18b: a corrupt pid (0, or above `i32::MAX`) must never read as alive /// — `liveness` relies entirely on `agent::agent_running`'s existing /// hardening (no second probe is written), so it can only ever produce diff --git a/crates/devflow-cli/tests/supervisor_e2e.rs b/crates/devflow-cli/tests/supervisor_e2e.rs index e69e6975..982936cd 100644 --- a/crates/devflow-cli/tests/supervisor_e2e.rs +++ b/crates/devflow-cli/tests/supervisor_e2e.rs @@ -503,3 +503,110 @@ fn capture_supervisor_owns_a_fake_agent_and_records_devflow_result() { "the supervisor's in-process advance evaluated the stage as success:\n{events}" ); } + +/// `devflow status` for `root`, as an operator runs it; its stdout. +fn run_status(home: &common::HermeticHome, root: &Path) -> String { + let output = common::hermetic_devflow(home) + .arg("status") + .arg(root) + .current_dir(root) + .output() + .expect("run devflow status"); + assert!( + output.status.success(), + "status failed\nstderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +/// 50-12 criterion 1: `devflow status` answers ALIVE, STALE and GONE from the +/// recorded socket, and each non-ALIVE answer names the verb that repairs it. +/// The run is made STALE by a SIGKILL of its supervisor (the pid comes from +/// the sidecar and is identity-checked first), and GONE by removing the stale +/// socket file. The reaper ends the agent group the SIGKILL orphaned. +#[test] +fn status_reports_each_liveness_state_with_its_repair_verb() { + let phase = PhaseId::new(68); + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path().canonicalize().unwrap(); + init_repo(&root, phase); + let fake_bin = common::fake_bin_dir(WAITING_CLAUDE); + let _reaper = common::LaunchedRunReaper::new(&root, phase, vec![]); + + let output = run_start(&home, &root, phase, &fake_bin, Shape::Stream); + assert!( + output.status.success(), + "start failed\nstderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let record = devflow_core::workflow::load_state(&root, phase) + .expect("start saved state") + .supervisor + .expect("start must record state.supervisor"); + let socket = record.socket_path.clone(); + let stop_verb = format!("devflow stop --phase {phase}"); + let resume_verb = format!("devflow resume --phase {phase}"); + + let alive = run_status(&home, &root); + assert!( + alive.contains("ALIVE") && alive.contains("owner: agent_running"), + "a live run reads ALIVE with its owner:\n{alive}" + ); + assert!( + alive.contains(&format!("supervisor: {}", socket.display())), + "status names the recorded socket:\n{alive}" + ); + assert!( + !alive.contains(&stop_verb) && !alive.contains(&resume_verb), + "an ALIVE run needs no repair:\n{alive}" + ); + + let sidecar = supervisor::read_sidecar(&socket.with_extension("json")).expect("sidecar"); + let pid = sidecar.supervisor_pid; + assert!( + devflow_core::agent::is_same_process(pid, sidecar.supervisor_start_time), + "the sidecar's supervisor {pid} is the recorded process" + ); + let signed = libc::pid_t::try_from(pid).expect("pid fits pid_t"); + assert!(signed > 1, "refusing to signal pid {signed}"); + // SAFETY: `signed > 1` names the supervisor just identity-checked. + assert_eq!( + unsafe { libc::kill(signed, libc::SIGKILL) }, + 0, + "SIGKILL {pid}" + ); + assert!( + holds_within(RUN_WAIT, || !devflow_core::agent::agent_running(pid)), + "the supervisor {pid} is gone after SIGKILL" + ); + assert_eq!( + supervisor::probe(&socket).expect("probe the orphaned socket"), + Liveness::Stale, + "a SIGKILLed supervisor leaves its socket STALE" + ); + + let stale = run_status(&home, &root); + assert!(stale.contains("STALE"), "the run reads STALE:\n{stale}"); + let stop_at = stale + .find(&stop_verb) + .unwrap_or_else(|| panic!("STALE names `{stop_verb}`:\n{stale}")); + let resume_at = stale + .find(&resume_verb) + .unwrap_or_else(|| panic!("STALE names `{resume_verb}` after the stop:\n{stale}")); + assert!(stop_at < resume_at, "stop comes before resume:\n{stale}"); + assert!(!stale.contains("ALIVE"), "a STALE run is not ALIVE:\n{stale}"); + + fs::remove_file(&socket).expect("remove the stale socket"); + let gone = run_status(&home, &root); + assert!(gone.contains("GONE"), "the run reads GONE:\n{gone}"); + assert!( + gone.contains(&resume_verb), + "GONE while active names `{resume_verb}`:\n{gone}" + ); + assert!( + !gone.contains(&stop_verb), + "GONE names no stop, there is nothing left to stop:\n{gone}" + ); +} From c4b6e6cc9fdd1732c8e2c9f506ffeeaea60da689 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 16:46:15 -0400 Subject: [PATCH 053/112] feat(50-12): answer status and cleanup liveness from the supervisor socket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - PhaseLiveness (ALIVE/STALE/GONE/GONE-while-active/not launched/probe failed), describe, the pure phase_liveness and the one-probe probe_phase replace the pid-based Liveness/liveness - status prints 'supervisor: — ' and names stop then resume for STALE, resume for GONE while active; no monitor_pid read - cleanup refuses an ALIVE run, keeps a STALE run's worktree unless --force, and applies its existing rules otherwise Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 378 ++++++++++++--------- crates/devflow-cli/tests/supervisor_e2e.rs | 5 +- 2 files changed, 227 insertions(+), 156 deletions(-) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index f08b4556..4af322b9 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -36,7 +36,8 @@ use devflow_core::recover; use devflow_core::registry; use devflow_core::ship_evidence; use devflow_core::stage::Stage; -use devflow_core::state::{AgentKind, State}; +use devflow_core::state::{AgentKind, State, SupervisorRecord}; +use devflow_core::supervisor; use devflow_core::version; use devflow_core::workflow; use devflow_core::worktree; @@ -871,10 +872,12 @@ fn remove_worktree_with_retry( /// associated feature branches, then prune and clean up merged branches. /// /// Hard-refuses (D-06, no override flag) removal of any worktree whose owning -/// phase has a live agent (any monitor state, including Unknown/Stuck) or an -/// active monitor (Healthy/BetweenStages) — closing the race where a real -/// `cleanup --force` run could delete a worktree a live agent/monitor is +/// phase has a live agent (whatever its supervisor's liveness) or a +/// supervisor that answers ALIVE — closing the race where a real +/// `cleanup --force` run could delete a worktree a live agent/supervisor is /// still writing into (review: Codex HIGH, fail-closed on a live agent). +/// A STALE supervisor (or a failed probe) keeps the worktree unless +/// `--force` (50-12); GONE and NotLaunched follow the rules above. pub(crate) fn cleanup(project_root: &Path, force: bool) -> Result<(), CliError> { // Project-resolved (45-01): `cleanup_merged` computes "merged" relative // to the trunk and treats it as protected — both must be the configured @@ -903,19 +906,18 @@ pub(crate) fn cleanup(project_root: &Path, force: bool) -> Result<(), CliError> let agent_alive = phase .and_then(|p| agent_pid_from_file(project_root, p)) .is_some_and(agent::agent_running); - let monitor_pid = matched_state.and_then(|s| s.monitor_pid); - let monitor_alive = monitor_pid.is_some_and(agent::agent_running); - let phase_liveness = liveness(monitor_pid, monitor_alive, agent_alive); - - // A phase halted via `devflow start --until ` (20c) clears - // `monitor_pid` and its agent has already exited by design — that - // reads as `Liveness::Unknown` with `agent_alive == false`, which - // would otherwise sail straight through the live-agent refusal + // 50-12: the supervisor's liveness comes from its recorded socket, + // never a pid. + let phase_liveness = matched_state.map_or(PhaseLiveness::NotLaunched, probe_phase); + + // A phase halted via `devflow start --until ` (20c) has no + // running supervisor and its agent has already exited by design — + // which would otherwise sail straight through the live-agent refusal // below. Treat it the same way `doctor`'s `check_dead_agent`/ - // `check_dead_monitor` were taught about `facts.stopped` in this - // same phase: an intentionally-parked worktree is never implicitly - // safe to remove — require `--force`, mirroring the `reference` - // worktree's own precedent above. + // `check_supervisor` were taught about `facts.stopped`: an + // intentionally-parked worktree is never implicitly safe to remove — + // require `--force`, mirroring the `reference` worktree's own + // precedent above. let stopped = matched_state.is_some_and(|s| s.stopped); if stopped && !force { let phase_label = phase @@ -928,11 +930,10 @@ pub(crate) fn cleanup(project_root: &Path, force: bool) -> Result<(), CliError> continue; } - // Fail-closed on a live agent: refuse whenever the agent is alive - // (regardless of monitor liveness — Unknown/Stuck included) OR the - // monitor is actively running the stage (Healthy/BetweenStages). - // Only Stuck/Unknown WITHOUT a live agent proceeds. - if agent_alive || matches!(phase_liveness, Liveness::Healthy | Liveness::BetweenStages) { + // Fail-closed on a live run: refuse whenever the agent is alive + // (whatever the supervisor's liveness) OR the supervisor answers + // ALIVE, `--force` or not. + if agent_alive || matches!(phase_liveness, PhaseLiveness::Alive { .. }) { let phase_label = phase .map(|p| p.to_string()) .unwrap_or_else(|| "?".to_string()); @@ -943,6 +944,26 @@ pub(crate) fn cleanup(project_root: &Path, force: bool) -> Result<(), CliError> ))); } + // 50-12: a STALE supervisor may have left its agent group running, + // and a failed probe proves nothing either way: keep the worktree + // unless `--force`, and name the verb that ends the orphaned run. + if !force + && matches!( + phase_liveness, + PhaseLiveness::Stale | PhaseLiveness::ProbeFailed(_) + ) + { + let phase_label = phase + .map(|p| p.to_string()) + .unwrap_or_else(|| "?".to_string()); + println!( + "keeping worktree {} for phase {phase_label} — supervisor {}; run `devflow stop --phase {phase_label}` first, or pass --force to discard it", + wt.path.display(), + phase_liveness.describe(), + ); + continue; + } + match remove_worktree_with_retry(project_root, &wt.path, force) { Ok(()) => { print!("removed worktree {}", wt.path.display()); @@ -981,103 +1002,152 @@ pub(crate) fn cleanup(project_root: &Path, force: bool) -> Result<(), CliError> Ok(()) } -/// A phase's monitor/agent liveness, distinguishing a dead monitor (nothing -/// will call `devflow advance` when the agent exits) from a normal -/// between-stages moment (18b — "who watches the watcher"). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum Liveness { - /// Monitor and agent are both alive — the stage is actively running. - Healthy, - /// Monitor is alive, agent has exited — normal between-stages moment; - /// the monitor will advance the phase shortly. - BetweenStages, - /// The recorded monitor is dead. Whether or not the agent is also dead, - /// nothing will call `devflow advance` for this phase — it needs a - /// manual `devflow resume`. - Stuck, - /// No monitor PID has been recorded for this state — either none has - /// been spawned yet, or the state was written by a binary predating - /// this field. Never reported as a problem. - Unknown, -} - -impl Liveness { - pub(crate) fn describe(self) -> &'static str { - match self { - Liveness::Healthy => "healthy", - Liveness::BetweenStages => "between stages", - Liveness::Stuck => "stuck — needs devflow resume", - Liveness::Unknown => "unknown (no monitor recorded)", - } - } -} - -/// Pure liveness predicate — no I/O. `monitor_pid` is matched `None` first -/// so a state written by a pre-18b binary (carrying no `monitor_pid`) can -/// never be misclassified as `Stuck` (T-18-11). -fn liveness(monitor_pid: Option, monitor_alive: bool, agent_alive: bool) -> Liveness { - match monitor_pid { - None => Liveness::Unknown, - Some(_) => match (monitor_alive, agent_alive) { - (true, true) => Liveness::Healthy, - (true, false) => Liveness::BetweenStages, - (false, _) => Liveness::Stuck, - }, - } -} - /// A phase's supervisor liveness as `status`, `cleanup` and `doctor` report /// it (50-12): answered from the socket recorded in `state.supervisor` -/// through [`supervisor::probe`], never from a pid. +/// through [`supervisor::probe`], never from a pid. Three parts, like the +/// 18b presentation it replaces: this enum, [`PhaseLiveness::describe`], and +/// the pure mapping [`phase_liveness`], which `doctor`'s `check_supervisor` +/// reuses so the two can never drift. +/// +/// A supervisor runs its own advance in-process, so a moment between stages +/// is ALIVE (its owner reads `advancing`), not a separate state. #[derive(Debug, Clone, PartialEq, Eq)] -#[allow(dead_code)] // RED stub (50-12 Task 1) pub(crate) enum PhaseLiveness { /// The recorded socket answered a ping. Alive { /// Who owns the run right now, from the ping reply. owner: String, - /// The launch shape, from the ping reply. + /// The launch shape (`capture` or `stream`), from the ping reply. launch: String, }, - /// The socket file remains but nothing listens on it. + /// The socket file remains but nothing listens on it: the supervisor + /// died without cleaning up, and its agent group may still run. Stale, - /// No socket at the recorded path, and the phase is stopped. + /// No socket at the recorded path, and the phase is stopped: a normal + /// end. Gone, - /// No socket at the recorded path, but the phase is not stopped. + /// No socket at the recorded path while the phase is not stopped: + /// nothing will advance it. A finding, not a normal state. GoneWhileActive, - /// The state records no supervisor. + /// The state records no supervisor: nothing has been launched yet, or the + /// state predates the record. Never a finding. NotLaunched, - /// The probe could not answer; a path-free summary of why. + /// The probe could not answer; a path-free summary of why (see + /// [`probe_error_summary`]). Neither ALIVE nor GONE is guessed. ProbeFailed(String), } -/// RED stub (50-12 Task 1): answers `NotLaunched` for every input. -#[allow(dead_code)] +impl PhaseLiveness { + /// The operator-facing text. The three states the operator acts on lead + /// with `ALIVE`, `STALE` and `GONE`. + pub(crate) fn describe(&self) -> String { + match self { + PhaseLiveness::Alive { owner, launch } => { + format!("ALIVE (owner: {owner}, launch: {launch})") + } + PhaseLiveness::Stale => { + "STALE — the socket remains but no supervisor answers".to_string() + } + PhaseLiveness::Gone => { + "GONE — the supervisor has exited and the phase is stopped".to_string() + } + PhaseLiveness::GoneWhileActive => { + "GONE — no supervisor runs this phase, but it is not stopped".to_string() + } + PhaseLiveness::NotLaunched => "not launched (no supervisor recorded)".to_string(), + PhaseLiveness::ProbeFailed(why) => format!("UNKNOWN — the probe failed: {why}"), + } + } +} + +/// Pure mapping from a phase's supervisor record and its probe result to a +/// [`PhaseLiveness`]. No I/O. No record (or no probe) is `NotLaunched`; GONE +/// splits on `stopped`; a probe error is `ProbeFailed`, never a guess. fn phase_liveness( - _record: Option<&devflow_core::state::SupervisorRecord>, - _probe: Option>, - _stopped: bool, + record: Option<&SupervisorRecord>, + probe: Option>, + stopped: bool, ) -> PhaseLiveness { - PhaseLiveness::NotLaunched + let (Some(_), Some(probe)) = (record, probe) else { + return PhaseLiveness::NotLaunched; + }; + match probe { + Ok(supervisor::Liveness::Alive(reply)) => PhaseLiveness::Alive { + owner: reply.owner, + launch: reply.launch, + }, + Ok(supervisor::Liveness::Stale) => PhaseLiveness::Stale, + Ok(supervisor::Liveness::Gone) if stopped => PhaseLiveness::Gone, + Ok(supervisor::Liveness::Gone) => PhaseLiveness::GoneWhileActive, + Err(error) => PhaseLiveness::ProbeFailed(probe_error_summary(&error)), + } +} + +/// A [`supervisor::ProbeError`] as a path-free kind and cause (T-18-01): +/// `NotASocket`'s own text names the path, and a `NoReply` text may quote +/// whatever the peer sent, so neither is passed through. +fn probe_error_summary(error: &supervisor::ProbeError) -> String { + match error { + supervisor::ProbeError::Stat(cause) => { + format!("could not stat the socket ({})", cause.kind()) + } + supervisor::ProbeError::NotASocket(_) => "the recorded path is not a socket".to_string(), + supervisor::ProbeError::ForeignOwner { uid } => { + format!("the socket is owned by uid {uid}, not this user") + } + supervisor::ProbeError::Connect(cause) => { + format!("could not connect to the socket ({})", cause.kind()) + } + supervisor::ProbeError::NoReply(_) => "no valid ping reply".to_string(), + _ => "the probe failed".to_string(), + } } -/// Recovery verbs discoverable from a phase's liveness (21a, D-03) — the -/// pure, testable counterpart to `status`'s old inline Stuck `println!`. -/// Always includes `devflow resume` for `Stuck`; additionally includes -/// `devflow advance` when the phase is gate-pending (the operator answers -/// the gate then advances — the primary footgun this closes; widening the -/// predicate further risks suggesting `advance` where nothing proves it is -/// right, per 21-CONTEXT.md's Review Incorporation). Empty for any other -/// liveness, so a healthy/between-stages/unknown phase prints nothing new. -fn recovery_hints(state: &State, liveness: Liveness) -> Vec { - if liveness != Liveness::Stuck { - return Vec::new(); - } - let mut hints = vec![format!("devflow resume --phase {}", state.phase)]; - if state.gate_pending { - hints.push(format!("devflow advance --phase {}", state.phase)); - } - hints +/// Probe the socket `state.supervisor` records, once, and map the answer. +/// The only I/O half of [`phase_liveness`]; reads no pid. +fn probe_phase(state: &State) -> PhaseLiveness { + let record = state.supervisor.as_ref(); + let probe = record.map(|record| supervisor::probe(&record.socket_path)); + phase_liveness(record, probe, state.stopped) +} + +/// Recovery verbs discoverable from a phase's liveness (21a, D-03; 50-12) — +/// the pure, testable counterpart to `status`'s hints. STALE names +/// `devflow stop` (it ends the orphaned agent group and removes the socket) +/// and then `devflow resume`; GONE while active names `devflow resume`, plus +/// `devflow advance` when the phase is gate-pending (answer the gate, then +/// advance — widening that predicate risks suggesting `advance` where +/// nothing proves it is right, per 21-CONTEXT.md's Review Incorporation); a +/// failed probe names the socket to inspect. Empty for ALIVE, a stopped +/// GONE and NotLaunched. +fn recovery_hints(state: &State, liveness: &PhaseLiveness) -> Vec { + let phase = state.phase; + match liveness { + PhaseLiveness::Stale => vec![ + format!("devflow stop --phase {phase}"), + format!("devflow resume --phase {phase}"), + ], + PhaseLiveness::GoneWhileActive => { + let mut hints = vec![format!("devflow resume --phase {phase}")]; + if state.gate_pending { + hints.push(format!("devflow advance --phase {phase}")); + } + hints + } + PhaseLiveness::ProbeFailed(why) => { + let socket = state + .supervisor + .as_ref() + .map(|record| record.socket_path.display().to_string()) + .unwrap_or_default(); + vec![format!( + "the probe failed ({why}); inspect the socket at {socket}" + )] + } + PhaseLiveness::Alive { .. } | PhaseLiveness::Gone | PhaseLiveness::NotLaunched => { + Vec::new() + } + } } /// `status`'s in-stage progress line: real elapsed time since the phase's @@ -1149,19 +1219,15 @@ pub(crate) fn status(project_root: &Path) -> Result<(), CliError> { } None => println!(" agent_pid: none"), } - match state.monitor_pid { - Some(pid) => { - println!( - " monitor_pid: {pid} (running: {})", - agent::agent_running(pid) - ); - } - None => println!(" monitor_pid: none"), - } - let agent_alive = agent_pid.is_some_and(agent::agent_running); - let monitor_alive = state.monitor_pid.is_some_and(agent::agent_running); - let phase_liveness = liveness(state.monitor_pid, monitor_alive, agent_alive); - println!(" liveness: {}", phase_liveness.describe()); + // 50-12: liveness comes from the recorded socket, never a pid. + // `status` speaks only about the operator's own project, so it + // names the socket path (doctor does not, T-18-01). + let phase_liveness = probe_phase(state); + let socket = state.supervisor.as_ref().map_or_else( + || "none".to_string(), + |record| record.socket_path.display().to_string(), + ); + println!(" supervisor: {socket} — {}", phase_liveness.describe()); let phase_summary = last_events.remove(&state.phase); println!( "{}", @@ -1170,7 +1236,7 @@ pub(crate) fn status(project_root: &Path) -> Result<(), CliError> { phase_summary.as_ref().and_then(|s| s.stage_launched_ts) ) ); - for hint in recovery_hints(state, phase_liveness) { + for hint in recovery_hints(state, &phase_liveness) { println!(" → {hint}"); } if let Some(summary) = phase_summary { @@ -3848,17 +3914,12 @@ fn check_dead_agent(facts: &PhaseFacts) -> Option { /// The recorded monitor pid is dead — nothing will call `devflow advance` /// for this phase, whether or not the agent is also dead (an agent that /// outlived its monitor is orphaned too, since nothing will advance it when -/// it exits either). Reuses `liveness` rather than re-deriving the matrix, -/// so the two copies can never drift (18b, T-18-11's `Unknown` guard applies -/// here transitively — an unrecorded monitor is silently `Unknown`, never a -/// finding). +/// it exits either). An unrecorded monitor is never a finding (T-18-11). fn check_dead_monitor(facts: &PhaseFacts) -> Option { - if facts.stopped - || liveness(facts.monitor_pid, facts.monitor_alive, facts.agent_alive) != Liveness::Stuck - { + let pid = facts.monitor_pid?; + if facts.stopped || facts.monitor_alive { return None; } - let pid = facts.monitor_pid?; Some(PhaseFinding { phase: facts.phase, severity: Severity::Problem, @@ -5487,32 +5548,6 @@ mod tests { assert!(default_logs_phase(dir.path()).is_err()); } - /// 18b: a state with no recorded monitor is never reported as stuck, - /// regardless of the (unreliable, since no monitor was ever recorded) - /// liveness bits passed alongside it. - #[test] - fn liveness_unknown_when_no_monitor_recorded() { - assert_eq!(liveness(None, false, false), Liveness::Unknown); - assert_eq!(liveness(None, false, true), Liveness::Unknown); - assert_eq!(liveness(None, true, false), Liveness::Unknown); - assert_eq!(liveness(None, true, true), Liveness::Unknown); - } - - /// 18b: the full four-row matrix for a recorded monitor pid. A dead - /// agent with a dead monitor OR a live monitor with a dead agent are - /// different states — only the former is `Stuck` (nothing will call - /// `devflow advance`); the latter is a normal between-stages moment. An - /// agent that outlived its monitor is also `Stuck` — orphaned, since - /// nothing will advance it when it exits either. - #[test] - fn liveness_matrix_covers_all_four_rows() { - let pid = Some(4242); - assert_eq!(liveness(pid, true, true), Liveness::Healthy); - assert_eq!(liveness(pid, true, false), Liveness::BetweenStages); - assert_eq!(liveness(pid, false, false), Liveness::Stuck); - assert_eq!(liveness(pid, false, true), Liveness::Stuck); - } - /// A ping reply as a supervisor sends it; built through serde because /// `PingReply` is `#[non_exhaustive]`. fn ping_reply(owner: &str, launch: &str) -> devflow_core::supervisor::protocol::PingReply { @@ -5586,7 +5621,10 @@ mod tests { } for (at, row) in rows.iter().enumerate() { for other in &rows[at + 1..] { - assert_ne!(row, other, "stopped={stopped}: two outcomes share one answer"); + assert_ne!( + row, other, + "stopped={stopped}: two outcomes share one answer" + ); } } } @@ -6953,11 +6991,13 @@ mod tests { assert!(row.contains('?'), "row must render the unknown age: {row}"); } - /// 21a: `recovery_hints` returns a `resume` hint for a stuck phase, - /// additionally an `advance` hint when the phase is gate-pending - /// (answer the gate, then advance), and nothing for a non-stuck phase. + /// 21a / 50-12: `recovery_hints` returns a `resume` hint for a phase + /// whose supervisor is GONE while it is not stopped, additionally an + /// `advance` hint when the phase is gate-pending (answer the gate, then + /// advance), `stop` then `resume` for a STALE socket, and nothing for an + /// ALIVE, stopped or unlaunched phase. #[test] - fn recovery_hints_includes_resume_for_stuck() { + fn recovery_hints_includes_resume_for_gone_while_active() { let dir = tempfile::tempdir().unwrap(); let state = State::new( PhaseId::new(7), @@ -6966,13 +7006,13 @@ mod tests { dir.path().to_path_buf(), ); - let hints = recovery_hints(&state, Liveness::Stuck); + let hints = recovery_hints(&state, &PhaseLiveness::GoneWhileActive); assert_eq!(hints, vec!["devflow resume --phase 7".to_string()]); } #[test] - fn recovery_hints_includes_advance_when_stuck_and_gate_pending() { + fn recovery_hints_includes_advance_when_gone_while_active_and_gate_pending() { let dir = tempfile::tempdir().unwrap(); let mut state = State::new( PhaseId::new(7), @@ -6982,7 +7022,7 @@ mod tests { ); state.gate_pending = true; - let hints = recovery_hints(&state, Liveness::Stuck); + let hints = recovery_hints(&state, &PhaseLiveness::GoneWhileActive); assert_eq!( hints, @@ -6994,7 +7034,7 @@ mod tests { } #[test] - fn recovery_hints_empty_for_healthy() { + fn recovery_hints_names_stop_then_resume_for_stale() { let dir = tempfile::tempdir().unwrap(); let state = State::new( PhaseId::new(7), @@ -7003,7 +7043,35 @@ mod tests { dir.path().to_path_buf(), ); - assert!(recovery_hints(&state, Liveness::Healthy).is_empty()); + assert_eq!( + recovery_hints(&state, &PhaseLiveness::Stale), + vec![ + "devflow stop --phase 7".to_string(), + "devflow resume --phase 7".to_string(), + ] + ); + } + + #[test] + fn recovery_hints_empty_for_alive_stopped_and_unlaunched() { + let dir = tempfile::tempdir().unwrap(); + let state = State::new( + PhaseId::new(7), + AgentKind::Claude, + Mode::Auto, + dir.path().to_path_buf(), + ); + let alive = PhaseLiveness::Alive { + owner: "agent_running".to_string(), + launch: "stream".to_string(), + }; + + for liveness in [alive, PhaseLiveness::Gone, PhaseLiveness::NotLaunched] { + assert!( + recovery_hints(&state, &liveness).is_empty(), + "{liveness:?} needs no repair" + ); + } } /// 21a / 999.30 IN-01: the shared one-pass event summary's diff --git a/crates/devflow-cli/tests/supervisor_e2e.rs b/crates/devflow-cli/tests/supervisor_e2e.rs index 982936cd..0f2d7997 100644 --- a/crates/devflow-cli/tests/supervisor_e2e.rs +++ b/crates/devflow-cli/tests/supervisor_e2e.rs @@ -596,7 +596,10 @@ fn status_reports_each_liveness_state_with_its_repair_verb() { .find(&resume_verb) .unwrap_or_else(|| panic!("STALE names `{resume_verb}` after the stop:\n{stale}")); assert!(stop_at < resume_at, "stop comes before resume:\n{stale}"); - assert!(!stale.contains("ALIVE"), "a STALE run is not ALIVE:\n{stale}"); + assert!( + !stale.contains("ALIVE"), + "a STALE run is not ALIVE:\n{stale}" + ); fs::remove_file(&socket).expect("remove the stale socket"); let gone = run_status(&home, &root); From 6a0f0a6277d9daacabd92b76fbaf41a33b129d30 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 16:47:47 -0400 Subject: [PATCH 054/112] test(50-12): add failing tests for doctor's supervisor check - check_supervisor over STALE, GONE while active and a failed probe - the silent control over ALIVE, GONE while stopped and not launched - doctor --json supervisor facts carry no path Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 173 +++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 4af322b9..2a407ebe 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -3808,6 +3808,10 @@ pub(crate) struct PhaseFacts { /// written by a binary predating the field — never treated as a problem. pub(crate) monitor_pid: Option, pub(crate) monitor_alive: bool, + /// RED stub (50-12 Task 2): the supervisor's liveness. + pub(crate) supervisor: PhaseLiveness, + /// RED stub (50-12 Task 2): the recorded launch id. + pub(crate) supervisor_launch_id: Option, /// The most recent event's `event` field value, for display context. pub(crate) last_event: Option, /// The `stage` field of the most recent `stage_launched` event; `None` @@ -3931,6 +3935,12 @@ fn check_dead_monitor(facts: &PhaseFacts) -> Option { }) } +/// RED stub (50-12 Task 2): reports nothing. +#[allow(dead_code)] +fn check_supervisor(_facts: &PhaseFacts) -> Option { + None +} + /// The last `stage_launched` event named a different stage than /// `state.stage`. A `Warn`, not a `Problem` — a healthy pipeline legitimately /// has one stage in flight between the launch event and the next @@ -4053,6 +4063,8 @@ fn build_phase_facts( agent_alive, monitor_pid, monitor_alive, + supervisor: PhaseLiveness::NotLaunched, + supervisor_launch_id: None, last_event: last_event_name, last_launched_stage, open_gate_stages, @@ -7166,6 +7178,162 @@ mod tests { mod doctor_reconciliation { use super::*; + /// The launch id every fixture's supervisor record carries. + const SUPERVISOR_LAUNCH_ID: &str = "0123456789abcdef"; + + /// Every string anywhere inside `value`. + fn strings_in(value: &serde_json::Value, out: &mut Vec) { + match value { + serde_json::Value::String(text) => out.push(text.clone()), + serde_json::Value::Array(items) => { + items.iter().for_each(|item| strings_in(item, out)); + } + serde_json::Value::Object(map) => { + map.values().for_each(|item| strings_in(item, out)); + } + _ => {} + } + } + + /// 50-12: a STALE supervisor, one GONE while its phase is not + /// stopped, and a failed probe are each a `Problem` naming the + /// launch, never its socket path; STALE's repair stops then resumes, + /// GONE's resumes, and a failed probe points at `devflow status` + /// (which prints the operator's own socket path). + #[test] + fn check_supervisor_reports_stale_and_gone_while_active() { + let stale = check_supervisor(&PhaseFacts { + supervisor: PhaseLiveness::Stale, + ..agreeing_facts(PhaseId::new(21)) + }) + .expect("a STALE supervisor is a finding"); + assert_eq!(stale.severity, Severity::Problem); + assert_eq!( + stale.repair.as_deref(), + Some("devflow stop --phase 21 && devflow resume --phase 21") + ); + assert!( + stale.detail.contains("STALE") && stale.detail.contains(SUPERVISOR_LAUNCH_ID), + "{}", + stale.detail + ); + + let gone = check_supervisor(&PhaseFacts { + supervisor: PhaseLiveness::GoneWhileActive, + ..agreeing_facts(PhaseId::new(22)) + }) + .expect("GONE while active is a finding"); + assert_eq!(gone.severity, Severity::Problem); + assert_eq!(gone.repair.as_deref(), Some("devflow resume --phase 22")); + assert!(gone.detail.contains("GONE"), "{}", gone.detail); + + let record = devflow_core::state::SupervisorRecord::new( + PathBuf::from("/home/someone/.cache/devflow/sock/x.sock"), + SUPERVISOR_LAUNCH_ID.to_string(), + 10, + 20, + "boot".to_string(), + ); + let failed = phase_liveness( + Some(&record), + Some(Err(devflow_core::supervisor::ProbeError::NotASocket( + record.socket_path.clone(), + ))), + false, + ); + let failed = check_supervisor(&PhaseFacts { + supervisor: failed, + ..agreeing_facts(PhaseId::new(23)) + }) + .expect("a failed probe is a finding"); + assert_eq!(failed.severity, Severity::Problem); + assert!( + failed.detail.contains("not a socket") && !failed.detail.contains('/'), + "a failed probe names its kind and no path: {}", + failed.detail + ); + assert_eq!(failed.repair.as_deref(), Some("devflow status")); + + let reconciled = reconcile_phase(&PhaseFacts { + supervisor: PhaseLiveness::Stale, + ..agreeing_facts(PhaseId::new(21)) + }); + assert!( + reconciled.iter().any(|f| f.detail == stale.detail), + "reconcile_phase runs check_supervisor" + ); + } + + /// 50-12 control: ALIVE, GONE while stopped, and NotLaunched are not + /// findings, so the checks above can only fire for the three states + /// that need a repair. + #[test] + fn check_supervisor_is_silent_when_alive_or_stopped() { + let alive = agreeing_facts(PhaseId::new(24)); + assert!(check_supervisor(&alive).is_none()); + assert!(reconcile_phase(&alive).is_empty()); + let stopped = PhaseFacts { + supervisor: PhaseLiveness::Gone, + stopped: true, + ..agreeing_facts(PhaseId::new(25)) + }; + assert!(check_supervisor(&stopped).is_none()); + let unlaunched = PhaseFacts { + supervisor: PhaseLiveness::NotLaunched, + supervisor_launch_id: None, + ..agreeing_facts(PhaseId::new(26)) + }; + assert!(check_supervisor(&unlaunched).is_none()); + } + + /// 50-12 (T-50-32): `doctor --json` phase facts carry the + /// supervisor's liveness, owner, launch and launch id, and no string + /// in the reconciliation section holds a path. + #[test] + fn doctor_json_reports_supervisor_liveness_without_paths() { + let record = devflow_core::state::SupervisorRecord::new( + PathBuf::from("/home/someone/.cache/devflow/sock/x.sock"), + SUPERVISOR_LAUNCH_ID.to_string(), + 10, + 20, + "boot".to_string(), + ); + let failed = phase_liveness( + Some(&record), + Some(Err(devflow_core::supervisor::ProbeError::NotASocket( + record.socket_path.clone(), + ))), + false, + ); + let facts = vec![ + agreeing_facts(PhaseId::new(27)), + PhaseFacts { + supervisor: failed, + ..agreeing_facts(PhaseId::new(28)) + }, + ]; + let body = render_reconciliation_json(&facts); + let entries = body.as_array().expect("an array"); + let alive = &entries[0]["supervisor"]; + assert_eq!(alive["liveness"], "alive", "{body}"); + assert_eq!(alive["owner"], "agent_running", "{body}"); + assert_eq!(alive["launch"], "stream", "{body}"); + assert_eq!(alive["launch_id"], SUPERVISOR_LAUNCH_ID, "{body}"); + let failed = &entries + .iter() + .find(|entry| entry["phase"] == 28) + .expect("phase 28's finding")["supervisor"]; + assert_eq!(failed["liveness"], "probe_failed", "{body}"); + assert!(failed["owner"].is_null(), "{body}"); + let mut strings = Vec::new(); + strings_in(&body, &mut strings); + assert!(!strings.is_empty(), "the scan must see strings: {body}"); + assert!( + strings.iter().all(|text| !text.contains('/')), + "no string in the reconciliation section may hold a path: {strings:?}" + ); + } + /// A fully-agreeing baseline: `reconcile_phase` over this returns /// zero findings. Each test overrides only the field(s) needed to /// trigger the one check it's proving. @@ -7179,6 +7347,11 @@ mod tests { agent_alive: true, monitor_pid: Some(4343), monitor_alive: true, + supervisor: PhaseLiveness::Alive { + owner: "agent_running".to_string(), + launch: "stream".to_string(), + }, + supervisor_launch_id: Some(SUPERVISOR_LAUNCH_ID.to_string()), last_event: Some("stage_launched".into()), last_launched_stage: Some(Stage::Code), open_gate_stages: Vec::new(), From b96771cae04682a26673a2b7a1766a3f02d6aa5a Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 16:50:15 -0400 Subject: [PATCH 055/112] feat(50-12): report supervisor liveness in doctor and doctor --json - PhaseFacts carries supervisor (one probe) and its launch id in place of monitor_pid/monitor_alive - check_supervisor replaces check_dead_monitor: STALE, GONE while active and a failed probe are problems with their repair; path-free details - doctor --json phase facts carry supervisor {liveness, owner, launch, launch_id} Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 195 +++++++++++++++++------------ 1 file changed, 115 insertions(+), 80 deletions(-) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 2a407ebe..4ab0896b 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -1058,6 +1058,18 @@ impl PhaseLiveness { PhaseLiveness::ProbeFailed(why) => format!("UNKNOWN — the probe failed: {why}"), } } + + /// A stable machine label for `doctor --json`. + pub(crate) fn label(&self) -> &'static str { + match self { + PhaseLiveness::Alive { .. } => "alive", + PhaseLiveness::Stale => "stale", + PhaseLiveness::Gone => "gone", + PhaseLiveness::GoneWhileActive => "gone_while_active", + PhaseLiveness::NotLaunched => "not_launched", + PhaseLiveness::ProbeFailed(_) => "probe_failed", + } + } } /// Pure mapping from a phase's supervisor record and its probe result to a @@ -3803,14 +3815,12 @@ pub(crate) struct PhaseFacts { pub(crate) waiter: lock::HolderStatus, pub(crate) agent_pid: Option, pub(crate) agent_alive: bool, - /// The monitor pid recorded in `State.monitor_pid` (18b). `None` means - /// no monitor has been spawned for this state yet, or the state was - /// written by a binary predating the field — never treated as a problem. - pub(crate) monitor_pid: Option, - pub(crate) monitor_alive: bool, - /// RED stub (50-12 Task 2): the supervisor's liveness. + /// The supervisor's liveness, from one probe of the socket + /// `state.supervisor` records (50-12) — never a pid. pub(crate) supervisor: PhaseLiveness, - /// RED stub (50-12 Task 2): the recorded launch id. + /// The launch id `state.supervisor` records; `None` when no supervisor + /// is recorded. It identifies the supervisor in findings in place of its + /// socket path (T-18-01). pub(crate) supervisor_launch_id: Option, /// The most recent event's `event` field value, for display context. pub(crate) last_event: Option, @@ -3821,8 +3831,8 @@ pub(crate) struct PhaseFacts { pub(crate) feature_branch_exists: bool, /// Whether this phase was intentionally halted by `devflow start --until /// ` (20c) — `State.stopped`. A stopped phase's dead agent - /// pid/stale monitor pid are expected, not a crash; both - /// `check_dead_agent` and `check_dead_monitor` must recognize this + /// pid/exited supervisor are expected, not a crash; both + /// `check_dead_agent` and `check_supervisor` must recognize this /// marker instead of reporting a `Problem` (review: Codex HIGH — the /// doctor gap is bigger than `check_dead_agent` alone). pub(crate) stopped: bool, @@ -3915,32 +3925,41 @@ fn check_dead_agent(facts: &PhaseFacts) -> Option { }) } -/// The recorded monitor pid is dead — nothing will call `devflow advance` -/// for this phase, whether or not the agent is also dead (an agent that -/// outlived its monitor is orphaned too, since nothing will advance it when -/// it exits either). An unrecorded monitor is never a finding (T-18-11). -fn check_dead_monitor(facts: &PhaseFacts) -> Option { - let pid = facts.monitor_pid?; - if facts.stopped || facts.monitor_alive { - return None; - } +/// The supervisor's liveness needs a repair (50-12): STALE (its socket +/// remains, nothing answers, its agent group may still run), GONE while the +/// phase is not stopped (nothing will advance it), or a probe that could not +/// answer. Reuses [`phase_liveness`]'s result from the facts rather than +/// re-deriving it, so `status` and `doctor` cannot drift. ALIVE, a stopped +/// GONE and NotLaunched are never findings (T-18-11: an unrecorded +/// supervisor is never a problem). Identifies the supervisor by phase and +/// launch id, never by its socket path (T-18-01). +fn check_supervisor(facts: &PhaseFacts) -> Option { + let phase = facts.phase; + let repair = match &facts.supervisor { + PhaseLiveness::Stale => { + format!("devflow stop --phase {phase} && devflow resume --phase {phase}") + } + PhaseLiveness::GoneWhileActive => format!("devflow resume --phase {phase}"), + // `status` prints the operator's own socket path, which doctor + // must not. + PhaseLiveness::ProbeFailed(_) => "devflow status".to_string(), + PhaseLiveness::Alive { .. } | PhaseLiveness::Gone | PhaseLiveness::NotLaunched => { + return None; + } + }; + let launch = facts.supervisor_launch_id.as_deref().unwrap_or("?"); Some(PhaseFinding { - phase: facts.phase, + phase, severity: Severity::Problem, detail: format!( - "phase {}: monitor pid {pid} recorded but not running at stage {}", - facts.phase, facts.stage + "phase {phase}: supervisor launch {launch} at stage {} is {}", + facts.stage, + facts.supervisor.describe() ), - repair: Some(format!("devflow resume --phase {}", facts.phase)), + repair: Some(repair), }) } -/// RED stub (50-12 Task 2): reports nothing. -#[allow(dead_code)] -fn check_supervisor(_facts: &PhaseFacts) -> Option { - None -} - /// The last `stage_launched` event named a different stage than /// `state.stage`. A `Warn`, not a `Problem` — a healthy pipeline legitimately /// has one stage in flight between the launch event and the next @@ -3992,7 +4011,7 @@ fn reconcile_phase(facts: &PhaseFacts) -> Vec { check_open_gate_without_waiter(facts), check_orphan_gate(facts), check_dead_agent(facts), - check_dead_monitor(facts), + check_supervisor(facts), check_stage_event_drift(facts), check_missing_branch(facts), ] @@ -4036,8 +4055,11 @@ fn build_phase_facts( let waiter = lock::holder_status(project_root, phase); let agent_pid = agent_pid_from_file(project_root, phase); let agent_alive = agent_pid.is_some_and(agent::agent_running); - let monitor_pid = state.monitor_pid; - let monitor_alive = monitor_pid.is_some_and(agent::agent_running); + let supervisor = probe_phase(&state); + let supervisor_launch_id = state + .supervisor + .as_ref() + .map(|record| record.launch_id.clone()); let last_event = last_events.remove(&phase).map(|summary| summary.event); let last_launched_stage = last_event.as_ref().and_then(last_launched_stage_from_event); let last_event_name = last_event @@ -4061,10 +4083,8 @@ fn build_phase_facts( waiter, agent_pid, agent_alive, - monitor_pid, - monitor_alive, - supervisor: PhaseLiveness::NotLaunched, - supervisor_launch_id: None, + supervisor, + supervisor_launch_id, last_event: last_event_name, last_launched_stage, open_gate_stages, @@ -4147,12 +4167,29 @@ fn render_reconciliation_json(facts: &[PhaseFacts]) -> serde_json::Value { "detail": finding.detail, "repair": finding.repair, "last_event": phase_facts.last_event, + "supervisor": supervisor_json(phase_facts), }) }) .collect(), ) } +/// A phase's supervisor facts for `doctor --json` (50-12): its liveness +/// label, the owner and launch shape an ALIVE ping reported, and the +/// recorded launch id. Never the socket path (T-18-01, T-50-32). +fn supervisor_json(facts: &PhaseFacts) -> serde_json::Value { + let (owner, launch) = match &facts.supervisor { + PhaseLiveness::Alive { owner, launch } => (Some(owner), Some(launch)), + _ => (None, None), + }; + serde_json::json!({ + "liveness": facts.supervisor.label(), + "owner": owner, + "launch": launch, + "launch_id": facts.supervisor_launch_id, + }) +} + /// Build `doctor --json`'s `"environment"` array from the pre-existing /// tool/env checks (WR-01, 18-fix). Extracted so it can be composed with /// `render_reconciliation_json`'s array into ONE JSON document instead of @@ -7345,8 +7382,6 @@ mod tests { waiter: lock::HolderStatus::NoHolder, agent_pid: Some(4242), agent_alive: true, - monitor_pid: Some(4343), - monitor_alive: true, supervisor: PhaseLiveness::Alive { owner: "agent_running".to_string(), launch: "stream".to_string(), @@ -7527,27 +7562,27 @@ mod tests { assert!(findings[0].repair.is_none()); } - /// 18b: a dead monitor with a dead agent is `Stuck` — nothing will - /// call `devflow advance` for this phase — and reports a `Problem` - /// finding with a `devflow resume --phase N` repair. + /// 18b / 50-12: a supervisor GONE while its phase is not stopped, + /// with a dead agent — nothing will call `devflow advance` for this + /// phase — reports a `Problem` finding with a `devflow resume --phase + /// N` repair. #[test] - fn reconcile_reports_stuck_when_monitor_and_agent_are_both_dead() { + fn reconcile_reports_gone_supervisor_and_dead_agent() { let facts = PhaseFacts { - monitor_pid: Some(5150), - monitor_alive: false, + supervisor: PhaseLiveness::GoneWhileActive, agent_pid: Some(4242), agent_alive: false, ..agreeing_facts(PhaseId::new(8)) }; let findings = reconcile_phase(&facts); - let monitor_finding = findings + let supervisor_finding = findings .iter() - .find(|f| f.detail.contains("monitor pid")) - .expect("expected a monitor finding when monitor and agent are both dead"); - assert_eq!(monitor_finding.severity, Severity::Problem); - assert!(monitor_finding.detail.contains("monitor pid 5150")); + .find(|f| f.detail.contains("supervisor launch")) + .expect("expected a supervisor finding when the supervisor is gone"); + assert_eq!(supervisor_finding.severity, Severity::Problem); + assert!(supervisor_finding.detail.contains(SUPERVISOR_LAUNCH_ID)); assert_eq!( - monitor_finding.repair.as_deref(), + supervisor_finding.repair.as_deref(), Some("devflow resume --phase 8") ); } @@ -7576,17 +7611,14 @@ mod tests { } /// 20c (D-09 + review: Codex HIGH — the doctor gap is bigger than - /// `check_dead_agent`): the same stopped phase may also carry a - /// stale `monitor_pid` (though the stop path clears it — this - /// proves the guard holds even if that clear were ever bypassed). - /// `check_dead_monitor` must also recognize `facts.stopped` and - /// report ZERO findings. + /// `check_dead_agent`): the same stopped phase's supervisor has + /// exited. `check_supervisor` must also recognize `facts.stopped` + /// and report ZERO findings. #[test] - fn reconcile_phase_ignores_dead_monitor_when_stopped() { + fn reconcile_phase_ignores_gone_supervisor_when_stopped() { let facts = PhaseFacts { stage: Stage::Plan, - monitor_pid: Some(5150), - monitor_alive: false, + supervisor: PhaseLiveness::Gone, agent_pid: Some(4242), agent_alive: false, stopped: true, @@ -7595,45 +7627,48 @@ mod tests { let findings = reconcile_phase(&facts); assert!( findings.iter().all(|f| f.severity != Severity::Problem), - "a --until-stopped phase must yield zero Problem findings even with a \ - stale monitor_pid, got: {:?}", + "a --until-stopped phase must yield zero Problem findings even with an \ + exited supervisor, got: {:?}", findings.iter().map(|f| &f.detail).collect::>() ); } - /// 18b (T-18-11): an unrecorded monitor is unknown, not a problem — - /// a state file written by a pre-18b binary must never render as - /// stuck. + /// 18b (T-18-11): an unrecorded supervisor is not launched, not a + /// problem — a state file written before the record existed must + /// never render as a finding. #[test] - fn reconcile_is_silent_when_monitor_pid_is_unrecorded() { + fn reconcile_is_silent_when_no_supervisor_is_recorded() { let facts = PhaseFacts { - monitor_pid: None, - monitor_alive: false, + supervisor: PhaseLiveness::NotLaunched, + supervisor_launch_id: None, ..agreeing_facts(PhaseId::new(9)) }; assert!( reconcile_phase(&facts).is_empty(), - "an unrecorded monitor must never produce a finding" + "an unrecorded supervisor must never produce a finding" ); } - /// 18b: a live monitor with a dead agent is a normal between-stages - /// moment (the monitor hasn't advanced the phase yet), not a monitor - /// finding. `check_dead_agent`'s own pre-existing finding for the - /// dead agent pid is unrelated to this check and out of this plan's - /// scope. + /// 18b / 50-12: an ALIVE supervisor with a dead agent is a normal + /// between-stages moment (the supervisor runs the advance itself), + /// not a supervisor finding. `check_dead_agent`'s own pre-existing + /// finding for the dead agent pid is unrelated to this check. #[test] - fn reconcile_is_silent_when_monitor_alive_and_agent_dead() { + fn reconcile_is_silent_when_supervisor_alive_and_agent_dead() { let facts = PhaseFacts { - monitor_pid: Some(5150), - monitor_alive: true, + supervisor: PhaseLiveness::Alive { + owner: "advancing".to_string(), + launch: "stream".to_string(), + }, agent_alive: false, ..agreeing_facts(PhaseId::new(10)) }; let findings = reconcile_phase(&facts); assert!( - findings.iter().all(|f| !f.detail.contains("monitor pid")), - "a live monitor with a dead agent must not produce a monitor finding" + findings + .iter() + .all(|f| !f.detail.contains("supervisor launch")), + "a live supervisor with a dead agent must not produce a supervisor finding" ); } @@ -7646,8 +7681,7 @@ mod tests { gate_pending: true, agent_pid: Some(999_999), agent_alive: false, - monitor_pid: Some(999_998), - monitor_alive: false, + supervisor: PhaseLiveness::GoneWhileActive, last_launched_stage: Some(Stage::Validate), open_gate_stages: Vec::new(), feature_branch_exists: false, @@ -7660,14 +7694,15 @@ mod tests { vec![ Severity::Problem, // check_gate_pending_without_gate Severity::Problem, // check_dead_agent - Severity::Problem, // check_dead_monitor + Severity::Problem, // check_supervisor Severity::Warn, // check_stage_event_drift Severity::Warn, // check_missing_branch ] ); assert!(findings[0].detail.contains("gate_pending is true")); assert!(findings[1].detail.contains("agent pid 999999")); - assert!(findings[2].detail.contains("monitor pid 999998")); + assert!(findings[2].detail.contains("supervisor launch")); + assert!(findings[2].detail.contains("GONE")); assert!( findings[3] .detail From 7ccc11d7e341a4c49a52bba23080bf225202a2ae Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 16:53:40 -0400 Subject: [PATCH 056/112] test(50-12): add failing tests for the orphan sweep and 2.x census - doctor_reports_an_orphan_socket_without_removing_it over fixtures (a) to (i), with a snapshot of sock/ and its removal control - source scans: the orphan sweep only reads and calls sweep_for_in; the census docs name 2.x - the census matches the 2.x shapes and never a 3.x supervisor argv Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 399 +++++++++++++++++++++++++++++ crates/devflow-core/src/agent.rs | 148 +++++++++++ 2 files changed, 547 insertions(+) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 4ab0896b..ed61013c 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -4681,6 +4681,28 @@ fn render_stray_process_text(findings: &[StrayProcessFinding]) -> String { out } +/// One supervisor socket `doctor`'s read-only sweep reports (50-12, D-06). +pub(crate) struct OrphanSupervisorFinding { + /// The path-free finding: detail and `devflow stop --phase N --launch + /// ` repair. + pub(crate) finding: PhaseFinding, + /// The launch id that addresses exactly this socket. + pub(crate) launch_id: String, + /// `alive` or `stale`. + pub(crate) liveness: &'static str, + /// Whether the project root the socket names still exists. + pub(crate) root_exists: bool, +} + +/// RED stub (50-12 Task 3): reports nothing. +#[allow(dead_code)] +fn collect_orphan_supervisors( + _project_root: &Path, + _cache_dir: &Path, +) -> Vec { + Vec::new() +} + #[cfg(test)] mod tests { use super::*; @@ -7371,6 +7393,383 @@ mod tests { ); } + /// A short cache dir directly under `/tmp`, so every socket path + /// below it fits `sun_path`. + fn short_cache() -> tempfile::TempDir { + tempfile::Builder::new() + .prefix("dfs") + .tempdir_in("/tmp") + .expect("create a short cache dir under /tmp") + } + + /// Publish the socket for `(root, phase, launch_id)` under `cache`. + /// Dropping the returned listener leaves the socket STALE. + fn publish_launch( + cache: &Path, + root: &Path, + phase: u32, + launch_id: &str, + ) -> (supervisor::SocketPath, std::os::unix::net::UnixListener) { + let socket = + supervisor::SocketPath::for_launch(cache, root, PhaseId::new(phase), launch_id) + .expect("socket path fits sun_path"); + let listener = supervisor::publish(&socket).expect("publish the socket"); + (socket, listener) + } + + /// A complete sidecar for `(root, phase, launch_id)`, as a supervisor + /// writes it. + fn write_sidecar_for(socket: &supervisor::SocketPath, root: &Path, phase: u32, id: &str) { + let sidecar = serde_json::json!({ + "launch_id": id, + "project_root": root, + "phase": PhaseId::new(phase), + "stage": "code", + "agent": "claude", + "launch": "stream", + "agent_pid": 1, + "agent_pgid": 1, + "agent_start_time": 0, + "boot_id": "boot", + "supervisor_pid": 1, + "supervisor_start_time": 0, + "created_at": "2026-10-17T00:00:00Z", + }); + std::fs::write(socket.sidecar_path(), sidecar.to_string()).unwrap(); + supervisor::read_sidecar(&socket.sidecar_path()).expect("the planted sidecar parses"); + } + + /// A loadable state for `phase` whose supervisor record carries + /// `launch_id`. + fn save_state_with_launch(root: &Path, phase: u32, launch_id: &str) { + let mut state = State::new( + PhaseId::new(phase), + AgentKind::Claude, + Mode::Auto, + root.to_path_buf(), + ); + state.supervisor = Some(devflow_core::state::SupervisorRecord::new( + PathBuf::from("/elsewhere.sock"), + launch_id.to_string(), + 1, + 0, + "boot".to_string(), + )); + workflow::save_state(&state).unwrap(); + } + + /// Answers every ping on `listener` as a supervisor would, until + /// [`PingResponder::stop`]. + struct PingResponder { + done: std::sync::Arc, + socket: PathBuf, + thread: Option>, + } + + impl PingResponder { + fn start( + socket: &supervisor::SocketPath, + listener: std::os::unix::net::UnixListener, + launch_id: &str, + phase: u32, + ) -> Self { + use std::io::{BufRead as _, Write as _}; + let done = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let reply = serde_json::json!({ + "ok": true, + "launch_id": launch_id, + "phase": phase.to_string(), + "stage": "code", + "launch": "stream", + "owner": "agent_running", + "agent_pid": 1, + "agent_pgid": 1, + "agent_start_time": 0, + "boot_id": null, + "supervisor_pid": 1, + }); + let stop = std::sync::Arc::clone(&done); + let thread = std::thread::spawn(move || { + for stream in listener.incoming() { + if stop.load(std::sync::atomic::Ordering::SeqCst) { + break; + } + let Ok(stream) = stream else { continue }; + let mut line = String::new(); + if std::io::BufReader::new(&stream) + .read_line(&mut line) + .is_ok() + { + let _ = (&stream).write_all(format!("{reply}\n").as_bytes()); + } + } + }); + Self { + done, + socket: socket.as_path().to_path_buf(), + thread: Some(thread), + } + } + + fn stop(&mut self) { + self.done.store(true, std::sync::atomic::Ordering::SeqCst); + // Wake the blocked accept so the thread sees the flag. + let _ = std::os::unix::net::UnixStream::connect(&self.socket); + if let Some(thread) = self.thread.take() { + thread.join().expect("the ping responder thread"); + } + } + } + + impl Drop for PingResponder { + fn drop(&mut self) { + self.stop(); + } + } + + /// Every file under `dir` as (name, length, mtime), sorted. + fn snapshot(dir: &Path) -> Vec<(String, u64, std::time::SystemTime)> { + let mut files: Vec<_> = std::fs::read_dir(dir) + .expect("list the socket dir") + .map(|entry| { + let entry = entry.unwrap(); + let meta = std::fs::symlink_metadata(entry.path()).unwrap(); + ( + entry.file_name().to_string_lossy().into_owned(), + meta.len(), + meta.modified().unwrap(), + ) + }) + .collect(); + files.sort(); + files + } + + /// 50-12 (D-06, R5-I3, T-50-31, T-50-51, T-50-84): doctor's sweep + /// lists the ALIVE and STALE sockets that no loadable state of this + /// root claims — no state, an unparsable state, a superseded launch, + /// a missing or broken sidecar attributed by file name — and, + /// machine-wide, those whose project root is gone. Each names the + /// `--launch` teardown, never the bare stop. Another root's sockets + /// and the ones a state claims are not listed, and nothing under + /// `sock/` changes. + #[test] + fn doctor_reports_an_orphan_socket_without_removing_it() { + let cache = short_cache(); + let cache = cache.path(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path(); + let other = tempfile::tempdir().unwrap(); + let other_root = other.path(); + let deleted = tempfile::tempdir().unwrap(); + let deleted_root = deleted.path().to_path_buf(); + + // (a) STALE, sidecar for phase 7, whose state file is corrupt. + let (a, _) = publish_launch(cache, root, 7, "aaaa0000aaaa0007"); + write_sidecar_for(&a, root, 7, "aaaa0000aaaa0007"); + let state7 = workflow::state_path(root, PhaseId::new(7)); + std::fs::create_dir_all(state7.parent().unwrap()).unwrap(); + std::fs::write(&state7, "{").unwrap(); + // (b) STALE, superseded: phase 8's state names its successor. + let (b, _) = publish_launch(cache, root, 8, "aaaa0000aaaa0008"); + write_sidecar_for(&b, root, 8, "aaaa0000aaaa0008"); + save_state_with_launch(root, 8, "ffff0000ffff0008"); + // (c) STALE, another root that exists: not ours. + let (c, _) = publish_launch(cache, other_root, 7, "aaaa0000aaaa000c"); + write_sidecar_for(&c, other_root, 7, "aaaa0000aaaa000c"); + // (d) STALE, a project root deleted after the launch. + let (d, _) = publish_launch(cache, &deleted_root, 5, "aaaa0000aaaa000d"); + write_sidecar_for(&d, &deleted_root, 5, "aaaa0000aaaa000d"); + drop(deleted); + assert!(!deleted_root.exists(), "the root of (d) is deleted"); + // (e) ALIVE, this root's phase 9, no state, sidecar `{`. + let (e, listener) = publish_launch(cache, root, 9, "bbbb0000bbbb0000"); + std::fs::write(e.sidecar_path(), "{").unwrap(); + let mut responder = PingResponder::start(&e, listener, "bbbb0000bbbb0000", 9); + // (f) STALE, this root's phase 10, no sidecar, no state. + let (_f, _) = publish_launch(cache, root, 10, "cccc0000cccc0000"); + // (g) control: STALE, the other root's phase 9, no sidecar. + let (_g, _) = publish_launch(cache, other_root, 9, "dddd0000dddd0000"); + // (h) control: STALE, no sidecar, whose state claims it. + let (_h, _) = publish_launch(cache, root, 11, "eeee0000eeee0000"); + save_state_with_launch(root, 11, "eeee0000eeee0000"); + // (i) control: STALE with a sidecar, whose state claims it. + let (i, _) = publish_launch(cache, root, 12, "aaaa0000aaaa0012"); + write_sidecar_for(&i, root, 12, "aaaa0000aaaa0012"); + save_state_with_launch(root, 12, "aaaa0000aaaa0012"); + + let sock_dir = registry::socket_dir_in(cache); + let before = snapshot(&sock_dir); + let findings = collect_orphan_supervisors(root, cache); + assert_eq!( + snapshot(&sock_dir), + before, + "the sweep must change nothing under sock/" + ); + responder.stop(); + + let mut listed: Vec<(PhaseId, &str, &str, bool)> = findings + .iter() + .map(|orphan| { + ( + orphan.finding.phase, + orphan.launch_id.as_str(), + orphan.liveness, + orphan.root_exists, + ) + }) + .collect(); + listed.sort(); + assert_eq!( + listed, + vec![ + (PhaseId::new(5), "aaaa0000aaaa000d", "stale", false), + (PhaseId::new(7), "aaaa0000aaaa0007", "stale", true), + (PhaseId::new(8), "aaaa0000aaaa0008", "stale", true), + (PhaseId::new(9), "bbbb0000bbbb0000", "alive", true), + (PhaseId::new(10), "cccc0000cccc0000", "stale", true), + ], + "exactly (a), (b), (d), (e) and (f) are listed" + ); + for orphan in &findings { + let finding = &orphan.finding; + let repair = finding + .repair + .as_deref() + .expect("every orphan names a repair"); + assert_eq!( + repair, + format!( + "devflow stop --phase {} --launch {}", + finding.phase, orphan.launch_id + ), + "the repair addresses exactly this socket" + ); + assert!( + repair.contains("--launch") + && repair != format!("devflow stop --phase {}", finding.phase), + "never the bare stop, which would stop the state's own record: {repair}" + ); + assert!( + !finding.detail.contains('/'), + "a path-free detail: {}", + finding.detail + ); + assert_eq!(finding.severity, Severity::Problem); + } + let detail_of = |id: &str| { + findings + .iter() + .find(|orphan| orphan.launch_id == id) + .map(|orphan| orphan.finding.detail.clone()) + .unwrap() + }; + assert!(detail_of("aaaa0000aaaa000d").contains("no longer exists")); + assert!(detail_of("bbbb0000bbbb0000").contains("does not parse")); + assert!(detail_of("cccc0000cccc0000").contains("missing")); + assert!(detail_of("aaaa0000aaaa0008").contains("another launch")); + + // Control: removing (a)'s files changes the snapshot, so the + // comparison above can fail. + supervisor::remove_launch_files(&a).unwrap(); + assert_ne!( + snapshot(&sock_dir), + before, + "the snapshot must see a removal" + ); + } + + /// The comment-stripped body of `fn collect_orphan_supervisors(` in + /// `source` (cut at its first `mod tests {` line), from that line to + /// the next line that is exactly `}`; `None` when absent. + fn orphan_sweep_body(source: &str) -> Option> { + let lines: Vec<&str> = source + .lines() + .take_while(|line| !line.contains("mod tests {")) + .collect(); + let start = lines + .iter() + .position(|line| line.contains("fn collect_orphan_supervisors("))?; + let end = start + lines[start..].iter().position(|line| *line == "}")?; + Some( + lines[start..=end] + .iter() + .copied() + .filter(|line| !line.trim_start().starts_with("//")) + .collect(), + ) + } + + /// Why `source`'s `collect_orphan_supervisors` breaks R6-G4: absent + /// or under 5 lines, an acting call, or no `sweep_for_in(` call. + fn orphan_sweep_violations(source: &str) -> Vec { + const ACTING: [&str; 5] = [ + "remove_launch_files", + "remove_file", + "teardown_stale", + "request_shutdown", + "kill(", + ]; + let Some(body) = orphan_sweep_body(source) else { + return vec!["fn collect_orphan_supervisors( not found".to_string()]; + }; + let mut violations = Vec::new(); + if body.len() < 5 { + violations.push(format!("body has {} lines, under 5", body.len())); + } + for line in &body { + for call in ACTING { + if line.contains(call) { + violations.push(format!("acting call {call}: {line}")); + } + } + } + if !body.iter().any(|line| line.contains("sweep_for_in(")) { + violations.push("no sweep_for_in( call".to_string()); + } + violations + } + + /// 50-12 R6-G4 (T-50-31, T-50-84): doctor's orphan sweep only reads, + /// and attributes a sidecar-less socket through `sweep_for_in`, so + /// 50-06's digest rule stays in one place. Planted controls run the + /// same check. + #[test] + fn the_orphan_sweep_only_reads_and_attributes_through_sweep_for_in() { + let reads = "fn collect_orphan_supervisors(a: &Path, b: &Path) -> Vec {\n let mut out = Vec::new();\n for entry in supervisor::sweep_for_in(b, a, p) {\n out.push(entry);\n }\n out\n}\n"; + assert!( + orphan_sweep_violations(reads).is_empty(), + "a read-only fn passes" + ); + let acts = "fn collect_orphan_supervisors(a: &Path, b: &Path) -> Vec {\n let mut out = Vec::new();\n for entry in supervisor::sweep_for_in(b, a, p) {\n supervisor::remove_launch_files(&entry);\n out.push(entry);\n }\n out\n}\n"; + let found = orphan_sweep_violations(acts); + assert!( + found.len() == 1 && found[0].contains("remove_launch_files"), + "an acting call fails: {found:?}" + ); + let reimplements = "fn collect_orphan_supervisors(a: &Path, b: &Path) -> Vec {\n let mut out = Vec::new();\n // sweep_for_in(b, a, p) re-derived below\n for entry in supervisor::sweep_in(b) {\n out.push(entry);\n }\n out\n}\n"; + let found = orphan_sweep_violations(reimplements); + assert!( + found.len() == 1 && found[0].contains("no sweep_for_in"), + "a commented-out sweep_for_in fails: {found:?}" + ); + let absent = "fn other() {\n}\n"; + assert_eq!( + orphan_sweep_violations(absent), + vec!["fn collect_orphan_supervisors( not found".to_string()] + ); + + let source = std::fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")).join("src/commands.rs"), + ) + .expect("read src/commands.rs"); + assert_eq!( + orphan_sweep_violations(&source), + Vec::::new(), + "collect_orphan_supervisors must only read and must call sweep_for_in" + ); + } + /// A fully-agreeing baseline: `reconcile_phase` over this returns /// zero findings. Each test overrides only the field(s) needed to /// trigger the one check it's proving. diff --git a/crates/devflow-core/src/agent.rs b/crates/devflow-core/src/agent.rs index e7eef06f..c4a889be 100644 --- a/crates/devflow-core/src/agent.rs +++ b/crates/devflow-core/src/agent.rs @@ -586,6 +586,154 @@ pub fn looks_like_devflow_process(pid: u32) -> bool { mod tests { use super::*; + fn argv(args: &[&str]) -> Vec { + args.iter().map(ToString::to_string).collect() + } + + /// 50-12 (T-50-52): the `/proc` stray census matches only the two 2.x + /// shapes 50-11 deleted. Its two matchers still fire (the control), and + /// the 3.x supervisor's argv — both launch shapes, laid out as + /// `monitor::monitor_args` builds it — matches neither. + #[test] + fn the_stray_census_matches_only_the_2x_process_shapes() { + let wrapper_script = format!("cleanup() {{ :; }}\n{MONITOR_WRAPPER_MARKER}\nclaude -p"); + assert_eq!( + classify_stray_layer(&argv(&["sh", "-c", &wrapper_script])), + Some(StrayLayer::MonitorWrapper) + ); + assert_eq!( + classify_stray_layer(&argv(&["/x/devflow", "advance", "--phase", "7"])), + Some(StrayLayer::AdvanceChild) + ); + + let plan_shape = argv(&[ + "/x/devflow", + "__monitor", + "--project", + "/p", + "--phase", + "7", + "--launch", + "stream", + "--launch-id", + "abc", + ]); + let stream = argv(&[ + "/x/devflow", + "__monitor", + "--project", + "/p", + "--phase", + "7", + "--stage", + "code", + "--workdir", + "/w", + "--agent", + "claude", + "--launch", + "stream", + "--socket", + "/c/sock/x.sock", + "--launch-id", + "0123456789abcdef", + "--keep-env", + "GIT_CONFIG_COUNT", + "--prompt-file", + "/w/prompt", + "--idle-timeout-secs", + "60", + "--", + "claude", + "-p", + ]); + let capture = argv(&[ + "/x/devflow", + "__monitor", + "--project", + "/p", + "--phase", + "7", + "--stage", + "code", + "--workdir", + "/w", + "--agent", + "claude", + "--launch", + "capture", + "--socket", + "/c/sock/x.sock", + "--launch-id", + "0123456789abcdef", + "--", + "claude", + "-p", + ]); + for supervisor in [plan_shape, stream, capture] { + assert_eq!( + classify_stray_layer(&supervisor), + None, + "a 3.x supervisor is never a stray: {supervisor:?}" + ); + } + } + + /// The doc block directly above the line in `source` (cut at its first + /// `mod tests {` line) that starts with `item`: the contiguous `///` and + /// `#[` lines above it. `None` when the item is absent. + fn doc_block_above(source: &str, item: &str) -> Option { + let lines: Vec<&str> = source + .lines() + .take_while(|line| !line.contains("mod tests {")) + .collect(); + let at = lines.iter().position(|line| line.starts_with(item))?; + let block: Vec<&str> = lines[..at] + .iter() + .rev() + .take_while(|line| { + let line = line.trim_start(); + line.starts_with("///") || line.starts_with("#[") + }) + .copied() + .collect(); + Some(block.into_iter().rev().collect::>().join("\n")) + } + + /// Whether the doc block above `item` in `source` says the census is + /// 2.x-only; the reason when it does not. + fn census_doc_names_2x(source: &str, item: &str) -> Result<(), String> { + let doc = doc_block_above(source, item).ok_or_else(|| format!("{item} not found"))?; + if doc.contains("2.x") { + Ok(()) + } else { + Err(format!("the doc above {item} does not name 2.x:\n{doc}")) + } + } + + /// 50-12 R6-G4 (T-50-52): while the 2.x census exists, its docs must say + /// it matches only 2.x processes, so nobody reads it as covering 3.x. + /// The planted controls run the same extractor. + #[test] + fn the_stray_census_docs_say_it_is_2x_only() { + let planted = "/// Lists strays.\n#[derive(Debug)]\npub enum StrayLayer {"; + assert!(census_doc_names_2x(planted, "pub enum StrayLayer").is_err()); + let named = "/// Lists 2.x strays.\n#[derive(Debug)]\npub enum StrayLayer {"; + assert_eq!(census_doc_names_2x(named, "pub enum StrayLayer"), Ok(())); + assert!(census_doc_names_2x("fn other() {}", "pub enum StrayLayer").is_err()); + + let source = std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/agent.rs"), + ) + .expect("read src/agent.rs"); + for item in [ + "pub enum StrayLayer", + "pub fn discover_stray_devflow_processes", + ] { + assert_eq!(census_doc_names_2x(&source, item), Ok(())); + } + } + #[test] fn agent_running_detects_self() { // The current process is, by definition, running. From d9248b5d56ebf3204d2d38afc0c925c3a9dd0a26 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 17:00:53 -0400 Subject: [PATCH 057/112] feat(50-12): add doctor's read-only orphan supervisor sweep - collect_orphan_supervisors lists ALIVE/STALE sockets no state claims, deleted-root sockets machine-wide, and sidecar-less sockets attributed through sweep_for_in; each repair is stop --phase N --launch - doctor_in sweeps the given cache dir; text and --json carry the findings, and an unlistable socket dir is reported, not shown as none - the /proc stray census is documented as 2.x-only (StrayLayer, discover_stray_devflow_processes, MONITOR_WRAPPER_MARKER, doctor's stray finding, gate sweep --reap-strays); the status e2e pins that it never lists a live 3.x supervisor, with a planted 2.x control Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 339 +++++++++++++++++++-- crates/devflow-cli/src/main.rs | 25 +- crates/devflow-cli/tests/supervisor_e2e.rs | 53 ++++ crates/devflow-core/src/agent.rs | 39 ++- 4 files changed, 414 insertions(+), 42 deletions(-) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index ed61013c..35302af6 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -3206,6 +3206,14 @@ fn doctor_checks() -> Vec { /// Audit the environment and report what's installed, missing, or broken. pub(crate) fn doctor(project_root: &Path, json: bool) -> Result<(), CliError> { + doctor_in(project_root, json, registry::cache_dir().as_deref()) +} + +/// [`doctor`] against an explicit cache dir, whose `sock/` directory the +/// read-only supervisor sweep lists (50-12); `None` (no cache dir can be +/// resolved) skips the sweep without an error. Split out so tests sweep a +/// tempdir, never the operator's real cache. +fn doctor_in(project_root: &Path, json: bool, cache_dir: Option<&Path>) -> Result<(), CliError> { let checks = doctor_checks(); let facts = collect_phase_facts(project_root); @@ -3215,6 +3223,12 @@ pub(crate) fn doctor(project_root: &Path, json: bool) -> Result<(), CliError> { // scan, which never signals anything (T-25-62, doctor's read-only // contract). let stray_findings = collect_stray_process_findings(project_root); + // 50-12 (D-06): the read-only socket sweep — orphaned, superseded and + // deleted-root supervisors. It removes nothing and signals nothing. + let orphans = cache_dir + .map(|cache| collect_orphan_supervisors(project_root, cache)) + .unwrap_or_default(); + let sweep_unlistable = cache_dir.and_then(socket_dir_unlistable); if json { // WR-01 (18-fix): a single top-level JSON document — @@ -3228,7 +3242,13 @@ pub(crate) fn doctor(project_root: &Path, json: bool) -> Result<(), CliError> { // planning-doc check (D-05) and this plan's stray-process finding // (999.44) each extend this SAME object with one more key rather // than forking a second array. - let body = doctor_json_body(&checks, &facts, &doc_findings, &stray_findings); + let body = doctor_json_body( + &checks, + &facts, + &doc_findings, + &stray_findings, + (&orphans, sweep_unlistable.as_deref()), + ); println!( "{}", serde_json::to_string_pretty(&body).expect("doctor --json body must serialize") @@ -3254,6 +3274,10 @@ pub(crate) fn doctor(project_root: &Path, json: bool) -> Result<(), CliError> { print!("{}", render_reconciliation_text(&facts)); print!("{}", render_planning_doc_text(&doc_findings)); print!("{}", render_stray_process_text(&stray_findings)); + print!( + "{}", + render_orphan_supervisors_text(&orphans, sweep_unlistable.as_deref()) + ); } Ok(()) @@ -4220,17 +4244,24 @@ fn checks_json_value(checks: &[Check]) -> serde_json::Value { /// [...], "planning_doc_staleness": [...]}` — 21b's addition (D-05) extends /// this SAME object with a third key rather than forking a second reporter /// or printing a second top-level array. +/// 50-12 adds `"orphan_supervisors"` (the socket sweep's findings) and +/// `"orphan_supervisor_sweep_error"` (`null`, or why the socket directory +/// could not be listed, so an empty array is never mistaken for a clean +/// sweep). fn doctor_json_body( checks: &[Check], facts: &[PhaseFacts], doc_findings: &[PlanningDocFinding], stray_findings: &[StrayProcessFinding], + (orphans, sweep_unlistable): (&[OrphanSupervisorFinding], Option<&str>), ) -> serde_json::Value { serde_json::json!({ "environment": checks_json_value(checks), "reconciliation": render_reconciliation_json(facts), "planning_doc_staleness": render_planning_doc_findings_json(doc_findings), "stray_processes": render_stray_process_findings_json(stray_findings), + "orphan_supervisors": render_orphan_supervisors_json(orphans), + "orphan_supervisor_sweep_error": sweep_unlistable, }) } @@ -4636,6 +4667,13 @@ pub(crate) fn build_stray_process_findings( /// into the reachable set via [`unreachable_stray_candidates`] so a project /// the machine registry has not recorded still has its own live monitors /// protected from being reported as orphans. +/// +/// The census matches only the two 2.x process shapes (the `sh -c` monitor +/// wrapper and its `devflow advance` child), which a 3.x binary never +/// starts, so this finding covers 2.x leftovers only (50-12). A 3.x +/// supervisor is found through its socket by [`check_supervisor`] and +/// [`collect_orphan_supervisors`] (which also takes over the deleted-root +/// class) and ended with `devflow stop --phase N --launch `. fn collect_stray_process_findings(project_root: &Path) -> Vec { build_stray_process_findings(&unreachable_stray_candidates(&[project_root.to_path_buf()])) } @@ -4690,17 +4728,234 @@ pub(crate) struct OrphanSupervisorFinding { pub(crate) launch_id: String, /// `alive` or `stale`. pub(crate) liveness: &'static str, - /// Whether the project root the socket names still exists. + /// Whether the project root the socket names still exists. `true` for a + /// sidecar-less socket, which is attributed to this (existing) root. pub(crate) root_exists: bool, } -/// RED stub (50-12 Task 3): reports nothing. -#[allow(dead_code)] +/// Every ALIVE or STALE supervisor socket under `cache_dir` that nothing +/// claims (50-12, D-06): (1) this root's sockets whose phase has no loadable +/// state, or whose state records no supervisor or another launch +/// (superseded); (2) machine-wide, sockets whose sidecar names a project +/// root that no longer exists (the 999.44 deleted-root class the 2.x `/proc` +/// census used to cover); (3) (R5-I3) sockets of this root whose sidecar is +/// missing or does not parse, attributed by file name through +/// [`supervisor::sweep_for_in`] so 50-06's digest rule stays in one place. +/// +/// Not listed, by design: a socket named outside the 50-05 scheme (it +/// carries no launch id, so no verb addresses it), and a sidecar-less socket +/// of a deleted root (only the digest survives, so nothing attributes it). +/// +/// Read-only (D-06, T-50-31): it sweeps, probes and loads state, and removes +/// nothing and signals nothing. Every finding's repair is `devflow stop +/// --phase N --launch ` (50-14), never the bare stop, which addresses +/// the state's own record and would stop a live successor (T-50-51). fn collect_orphan_supervisors( - _project_root: &Path, - _cache_dir: &Path, + project_root: &Path, + cache_dir: &Path, ) -> Vec { - Vec::new() + let mut findings = Vec::new(); + for entry in supervisor::sweep_in(cache_dir) { + let liveness = match &entry.liveness { + Ok(supervisor::Liveness::Alive(_)) => "alive", + Ok(supervisor::Liveness::Stale) => "stale", + _ => continue, + }; + if let Some(sidecar) = &entry.sidecar { + // A root that cannot be examined counts as present: only a + // proven-missing root is the deleted-root class. + let root_exists = sidecar.project_root.try_exists().unwrap_or(true); + let reason = if !root_exists { + Some(OrphanReason::RootDeleted) + } else if sidecar.project_root == project_root { + unclaimed_by_state(project_root, sidecar.phase, &sidecar.launch_id) + } else { + None + }; + if let Some(reason) = reason { + let launch = (sidecar.phase, sidecar.launch_id.as_str()); + findings.push(orphan_finding(launch, liveness, root_exists, &reason)); + } + continue; + } + // R5-I3: no usable sidecar, so the file name is the only address. + let (Some(launch_id), Some(phase)) = ( + entry.launch_id_in_name.as_deref(), + phase_in_socket_name(&entry.socket_path), + ) else { + continue; + }; + let ours = supervisor::sweep_for_in(cache_dir, project_root, phase) + .iter() + .any(|candidate| candidate.socket_path == entry.socket_path); + let Some(unclaimed) = ours + .then(|| unclaimed_by_state(project_root, phase, launch_id)) + .flatten() + else { + continue; + }; + let reason = OrphanReason::NoUsableSidecar { + broken: entry.sidecar_error.is_some(), + unclaimed: Box::new(unclaimed), + }; + findings.push(orphan_finding((phase, launch_id), liveness, true, &reason)); + } + findings +} + +/// Why [`collect_orphan_supervisors`] lists a socket. +enum OrphanReason { + /// This root's phase has no state file. + NoState, + /// This root's phase has a state file that does not load. + StateUnloadable, + /// The phase's state records no supervisor. + NoRecord, + /// The phase's state records another launch (the R-10 aftermath). + Superseded, + /// The sidecar names a project root that no longer exists. + RootDeleted, + /// The sidecar is missing (`broken: false`) or does not parse, and the + /// state does not claim the launch for `unclaimed`'s reason. + NoUsableSidecar { + broken: bool, + unclaimed: Box, + }, +} + +impl OrphanReason { + /// Path-free text for the finding's detail (T-18-01). + fn describe(&self) -> String { + match self { + OrphanReason::NoState => "its phase has no state".to_string(), + OrphanReason::StateUnloadable => "its phase's state does not load".to_string(), + OrphanReason::NoRecord => "its phase's state records no supervisor".to_string(), + OrphanReason::Superseded => { + "its phase's state records another launch (superseded)".to_string() + } + OrphanReason::RootDeleted => "its project root no longer exists".to_string(), + OrphanReason::NoUsableSidecar { broken, unclaimed } => format!( + "its sidecar {}, and {}", + if *broken { + "does not parse" + } else { + "is missing" + }, + unclaimed.describe() + ), + } + } +} + +/// Whether this root's state for `phase` claims `launch_id`; the reason +/// when it does not. Reads the state file only. +fn unclaimed_by_state( + project_root: &Path, + phase: PhaseId, + launch_id: &str, +) -> Option { + match workflow::load_state(project_root, phase) { + Err(workflow::WorkflowError::MissingState(_)) => Some(OrphanReason::NoState), + Err(_) => Some(OrphanReason::StateUnloadable), + Ok(state) => match state.supervisor { + None => Some(OrphanReason::NoRecord), + Some(record) if record.launch_id != launch_id => Some(OrphanReason::Superseded), + Some(_) => None, + }, + } +} + +/// The phase segment of a socket file name in the 50-05 scheme +/// (`<016x digest>--.sock`): the text between the +/// first and second `-`. `None` when it does not parse as a phase. +fn phase_in_socket_name(socket: &Path) -> Option { + let stem = socket.file_stem()?.to_str()?; + stem.split('-').nth(1)?.parse().ok() +} + +/// One orphan finding for `(phase, launch_id)`, path-free, with the +/// `--launch` repair that tears down exactly that socket. +fn orphan_finding( + (phase, launch_id): (PhaseId, &str), + liveness: &'static str, + root_exists: bool, + reason: &OrphanReason, +) -> OrphanSupervisorFinding { + OrphanSupervisorFinding { + finding: PhaseFinding { + phase, + severity: Severity::Problem, + detail: format!( + "phase {phase}: supervisor launch {launch_id} is {} but {}", + liveness.to_uppercase(), + reason.describe() + ), + repair: Some(format!("devflow stop --phase {phase} --launch {launch_id}")), + }, + launch_id: launch_id.to_string(), + liveness, + root_exists, + } +} + +/// Whether doctor's socket sweep could look (50-12): `None` when the socket +/// directory is absent or listable, else why it cannot be listed. +/// [`supervisor::sweep_in`] answers an unlistable directory with no entries, +/// so without this check doctor would report "nothing" when it saw nothing. +fn socket_dir_unlistable(cache_dir: &Path) -> Option { + match std::fs::read_dir(registry::socket_dir_in(cache_dir)) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + Some(error.kind().to_string()) + } + _ => None, + } +} + +/// Build `doctor --json`'s `"orphan_supervisors"` array (50-12). +fn render_orphan_supervisors_json(findings: &[OrphanSupervisorFinding]) -> serde_json::Value { + serde_json::Value::Array( + findings + .iter() + .map(|orphan| { + serde_json::json!({ + "phase": orphan.finding.phase, + "launch_id": orphan.launch_id, + "liveness": orphan.liveness, + "root_exists": orphan.root_exists, + "severity": orphan.finding.severity.label(), + "detail": orphan.finding.detail, + "repair": orphan.finding.repair, + }) + }) + .collect(), + ) +} + +/// Build `doctor`'s text section for the socket sweep (50-12). Like +/// [`render_stray_process_text`] it prints nothing when there is nothing to +/// report, but a socket directory it could not list is reported, never +/// shown as "nothing". +fn render_orphan_supervisors_text( + findings: &[OrphanSupervisorFinding], + unlistable: Option<&str>, +) -> String { + if findings.is_empty() && unlistable.is_none() { + return String::new(); + } + let mut out = String::from("\nsupervisor sockets no state claims:\n"); + if let Some(why) = unlistable { + out.push_str(&format!( + " warn: the socket directory could not be listed ({why}); its sockets are not \ + reported\n" + )); + } + for orphan in findings { + out.push_str(&format!(" {}\n", orphan.finding.detail)); + if let Some(repair) = &orphan.finding.repair { + out.push_str(&format!(" repair: {repair}\n")); + } + } + out } #[cfg(test)] @@ -5652,7 +5907,7 @@ mod tests { "boot".to_string(), ); for stopped in [false, true] { - let rows = vec![ + let rows = [ phase_liveness(None, None, stopped), phase_liveness(Some(&record), Some(Ok(Probe::Gone)), stopped), phase_liveness(Some(&record), Some(Ok(Probe::Stale)), stopped), @@ -7679,6 +7934,34 @@ mod tests { ); } + /// 50-12: `supervisor::sweep_in` answers a socket directory it + /// cannot list with no entries, so doctor checks that case itself and + /// reports it rather than printing nothing. Controls: an absent and a + /// listable directory report nothing. + #[test] + fn doctor_reports_an_unlistable_socket_dir_instead_of_nothing() { + use std::os::unix::fs::PermissionsExt as _; + let cache = tempfile::tempdir().unwrap(); + assert_eq!(socket_dir_unlistable(cache.path()), None, "absent"); + assert_eq!(render_orphan_supervisors_text(&[], None), ""); + let sock = registry::socket_dir_in(cache.path()); + std::fs::create_dir_all(&sock).unwrap(); + assert_eq!(socket_dir_unlistable(cache.path()), None, "listable"); + + std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o000)).unwrap(); + let unlistable = socket_dir_unlistable(cache.path()); + std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o700)).unwrap(); + let why = unlistable.expect("a directory without read permission is unlistable"); + assert!(!why.contains('/'), "path-free: {why}"); + let text = render_orphan_supervisors_text(&[], Some(&why)); + assert!( + text.contains("could not be listed") && text.contains(&why), + "{text}" + ); + let body = doctor_json_body(&[], &[], &[], &[], (&[], Some(&why))); + assert_eq!(body["orphan_supervisor_sweep_error"], why.as_str()); + } + /// The comment-stripped body of `fn collect_orphan_supervisors(` in /// `source` (cut at its first `mod tests {` line), from that line to /// the next line that is exactly `}`; `None` when absent. @@ -8205,7 +8488,7 @@ mod tests { workflow::save_state(&state).unwrap(); let facts = collect_phase_facts(root); - let body = doctor_json_body(&checks, &facts, &[], &[]); + let body = doctor_json_body(&checks, &facts, &[], &[], (&[], None)); let serialized = serde_json::to_string(&body).unwrap(); let reparsed: serde_json::Value = serde_json::from_str(&serialized) .expect("doctor --json must be single-document JSON, not two concatenated arrays"); @@ -8228,10 +8511,14 @@ mod tests { "999.44: must carry the stray-process findings under a FOURTH key, \ never a second concatenated array: {reparsed}" ); + assert!( + reparsed.get("orphan_supervisors").is_some(), + "50-12: must carry the socket sweep's findings under their own key: {reparsed}" + ); assert_eq!( reparsed.as_object().unwrap().len(), - 4, - "doctor --json must have exactly four top-level keys: {reparsed}" + 6, + "doctor --json must have exactly six top-level keys: {reparsed}" ); assert!(reparsed["environment"].is_array()); assert!(reparsed["reconciliation"].is_array()); @@ -8279,8 +8566,18 @@ mod tests { .lines() .count(); - doctor(root, false).unwrap(); - doctor(root, false).unwrap(); + // 50-12: a hermetic cache whose STALE socket no state claims, so + // the run also exercises the socket sweep, which must leave it. + let cache = short_cache(); + let (orphan, _) = publish_launch(cache.path(), root, 91, "aaaa0000aaaa0091"); + write_sidecar_for(&orphan, root, 91, "aaaa0000aaaa0091"); + + doctor_in(root, false, Some(cache.path())).unwrap(); + doctor_in(root, false, Some(cache.path())).unwrap(); + assert!( + orphan.as_path().exists() && orphan.sidecar_path().exists(), + "doctor must not remove a socket or sidecar it reports" + ); let after_len = std::fs::metadata(&state_path).unwrap().len(); let after_modified = std::fs::metadata(&state_path).unwrap().modified().unwrap(); @@ -8396,9 +8693,9 @@ mod tests { detail: "detail".to_string(), repair: None, }]; - let body = doctor_json_body(&checks, &facts, &[], &stray_findings); + let body = doctor_json_body(&checks, &facts, &[], &stray_findings, (&[], None)); let obj = body.as_object().unwrap(); - assert_eq!(obj.len(), 4, "must be exactly four top-level keys: {body}"); + assert_eq!(obj.len(), 6, "must be exactly six top-level keys: {body}"); assert!(obj.contains_key("stray_processes")); let strays = obj["stray_processes"].as_array().unwrap(); assert_eq!(strays.len(), 1); @@ -8472,9 +8769,11 @@ mod tests { // Also exercise doctor() itself, end to end, twice — not just // the pure finding collector — proving the full read-only path. - doctor(dir.path(), false).unwrap(); + // A hermetic, empty cache: never sweep the operator's real one. + let cache = tempfile::tempdir().unwrap(); + doctor_in(dir.path(), false, Some(cache.path())).unwrap(); assert!(agent::agent_running(pid), "doctor() itself must not signal"); - doctor(dir.path(), false).unwrap(); + doctor_in(dir.path(), false, Some(cache.path())).unwrap(); assert!( agent::agent_running(pid), "doctor() must remain read-only across repeated runs" @@ -9164,13 +9463,13 @@ mod tests { detail: "detail".to_string(), repair: None, }]; - let body = doctor_json_body(&checks, &facts, &doc_findings, &[]); + let body = doctor_json_body(&checks, &facts, &doc_findings, &[], (&[], None)); let obj = body.as_object().unwrap(); assert_eq!( obj.len(), - 4, + 6, "must be exactly {{environment, reconciliation, planning_doc_staleness, \ - stray_processes}}: {body}" + stray_processes, orphan_supervisors, orphan_supervisor_sweep_error}}: {body}" ); assert!(obj.contains_key("environment")); assert!(obj.contains_key("reconciliation")); diff --git a/crates/devflow-cli/src/main.rs b/crates/devflow-cli/src/main.rs index 82795e5d..58023019 100644 --- a/crates/devflow-cli/src/main.rs +++ b/crates/devflow-cli/src/main.rs @@ -486,18 +486,21 @@ enum GateCmd { /// machine-wide regardless of this flag (CR-01, 25-15). #[arg(long)] root: Option, - /// Also discover and clear devflow processes shaped like a monitor - /// wrapper or an `advance` child (999.44), by scanning the OS + /// Also discover and clear devflow 2.x processes shaped like a + /// monitor wrapper or an `advance` child (999.44), by scanning the OS /// process table directly rather than trusting a lock file the - /// process itself wrote. Matches only two structural argv shapes, - /// owned by the calling user, older than the minimum age, and NOT - /// named by any registered root's state file or lock file (CR-01, - /// 25-15) — a pid a live registry entry still reaches is never - /// touched. Discovery itself stays registry-independent, so this - /// still catches a process whose project root no longer exists on - /// disk (`devflow stop`/the default sweep above cannot see - /// either). Off by default: preview with `--dry-run` before - /// authorising it for real. + /// process itself wrote. Both shapes are 2.x only: a 3.x binary + /// starts neither, so this never matches a 3.x supervisor, which + /// `devflow doctor` reports through its socket and + /// `devflow stop --phase N --launch ` ends. Matches only two + /// structural argv shapes, owned by the calling user, older than the + /// minimum age, and NOT named by any registered root's state file or + /// lock file (CR-01, 25-15) — a pid a live registry entry still + /// reaches is never touched. Discovery itself stays + /// registry-independent, so this still catches a 2.x process whose + /// project root no longer exists on disk (`devflow stop`/the default + /// sweep above cannot see either). Off by default: preview with + /// `--dry-run` before authorising it for real. #[arg(long)] reap_strays: bool, }, diff --git a/crates/devflow-cli/tests/supervisor_e2e.rs b/crates/devflow-cli/tests/supervisor_e2e.rs index 0f2d7997..50d818e9 100644 --- a/crates/devflow-cli/tests/supervisor_e2e.rs +++ b/crates/devflow-cli/tests/supervisor_e2e.rs @@ -565,6 +565,8 @@ fn status_reports_each_liveness_state_with_its_repair_verb() { let sidecar = supervisor::read_sidecar(&socket.with_extension("json")).expect("sidecar"); let pid = sidecar.supervisor_pid; + census_never_sees_a_live_supervisor(pid); + assert!( devflow_core::agent::is_same_process(pid, sidecar.supervisor_start_time), "the sidecar's supervisor {pid} is the recorded process" @@ -613,3 +615,54 @@ fn status_reports_each_liveness_state_with_its_repair_verb() { "GONE names no stop, there is nothing left to stop:\n{gone}" ); } + +/// A 2.x-shaped monitor wrapper (`sh -c` with the trap line the 2.x script +/// carried), in its own process group, which drop SIGKILLs and reaps. +struct PlantedWrapper(std::process::Child); + +impl PlantedWrapper { + fn spawn() -> Self { + use std::os::unix::process::CommandExt as _; + // The trailing `:` keeps `sh` from exec-ing `sleep` in its place, so + // the wrapper's own argv stays visible in `/proc`. + let child = std::process::Command::new("sh") + .arg("-c") + .arg("cleanup() { :; }\ntrap cleanup TERM INT\nsleep 30\n:") + .process_group(0) + .spawn() + .expect("spawn the planted 2.x wrapper"); + Self(child) + } +} + +impl Drop for PlantedWrapper { + fn drop(&mut self) { + common::signal_group(self.0.id(), libc::SIGKILL); + let _ = self.0.wait(); + } +} + +/// 50-12 (T-50-52): the `/proc` stray census, run against a real 3.x +/// supervisor, never lists it. The control is a planted 2.x-shaped wrapper +/// that the same census run must list, so an empty census cannot pass. +fn census_never_sees_a_live_supervisor(supervisor_pid: u32) { + let wrapper = PlantedWrapper::spawn(); + let wrapper_pid = wrapper.0.id(); + let mut census = Vec::new(); + assert!( + holds_within(RUN_WAIT, || { + census = devflow_core::agent::discover_stray_devflow_processes(); + census.iter().any(|stray| stray.pid == wrapper_pid) + }), + "control: the census must list the planted 2.x wrapper {wrapper_pid}" + ); + assert!( + devflow_core::agent::agent_running(supervisor_pid), + "the supervisor {supervisor_pid} is alive while the census runs" + ); + assert!( + census.iter().all(|stray| stray.pid != supervisor_pid), + "the 2.x census must never list the 3.x supervisor {supervisor_pid}: {census:?}" + ); + drop(wrapper); +} diff --git a/crates/devflow-core/src/agent.rs b/crates/devflow-core/src/agent.rs index c4a889be..eb60007e 100644 --- a/crates/devflow-core/src/agent.rs +++ b/crates/devflow-core/src/agent.rs @@ -339,8 +339,15 @@ pub fn process_age(pid: u32) -> Option { /// cleanup — cleared on the next invocation — never a missed one. pub const STRAY_MIN_AGE: std::time::Duration = std::time::Duration::from_secs(2); -/// Which structural layer of a DevFlow-spawned process tree +/// Which structural layer of a DevFlow 2.x process tree /// [`discover_stray_devflow_processes`] matched a candidate against. +/// +/// Both layers are 2.x process shapes, and a 3.x binary starts neither +/// (50-11 deleted the `sh` monitor wrapper and its `devflow advance` child): +/// the census is kept for 2.x leftovers only. A 3.x supervisor is a +/// `devflow __monitor` process that this census never matches; `doctor` +/// finds it through its socket (its supervisor findings), and `devflow stop +/// --phase N --launch ` ends it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum StrayLayer { /// The monitor wrapper shell spawned by `monitor::spawn_monitor` — the @@ -370,12 +377,11 @@ pub struct StrayProcess { pub layer: StrayLayer, } -/// The monitor wrapper's trap-installation line, copied byte-for-byte from -/// `monitor::spawn_monitor_inner`'s literal script text (see -/// `crates/devflow-core/src/monitor.rs`) — not paraphrased or reduced to a -/// single word, so a reader can grep both files for this exact string to -/// confirm they still agree. If the wrapper script's text ever changes, this -/// constant must change with it in the same commit. +/// The 2.x monitor wrapper's trap-installation line, as its `sh -c` script +/// carried it. That script no longer exists in this source tree: 50-11 +/// replaced it with the Rust `__monitor` supervisor, so nothing a 3.x binary +/// starts carries this marker. It is kept, unchanged, so the census can +/// still recognise a 2.x wrapper left running from before the upgrade. const MONITOR_WRAPPER_MARKER: &str = "trap cleanup TERM INT"; /// The devflow CLI binary's name — `crates/devflow-cli/Cargo.toml`'s @@ -387,10 +393,21 @@ const DEVFLOW_BINARY_NAME: &str = "devflow"; /// `devflow-cli/src/main.rs`), matched against argv[1] for Layer 2. const ADVANCE_SUBCOMMAND: &str = "advance"; -/// Census both of DevFlow's orphan-prone process layers directly from the OS -/// process table — the only remaining discovery surface once a project root -/// has been deleted off disk, taking every registry entry, lock file and -/// state file with it (999.44). +/// Census both of DevFlow 2.x's orphan-prone process layers directly from +/// the OS process table — for 2.x, the only remaining discovery surface once +/// a project root has been deleted off disk, taking every registry entry, +/// lock file and state file with it (999.44). +/// +/// **2.x-only.** Both matchers name 2.x process shapes (the `sh -c` monitor +/// wrapper and its trailing `devflow advance` child) that 50-11 deleted, so +/// this census never matches a process a 3.x binary starts. It is kept for +/// 2.x leftovers (D-20 keeps none live at install). A 3.x supervisor is +/// found through its socket instead — `doctor`'s supervisor findings, +/// including the deleted-root class this census used to cover — and ended +/// with `devflow stop --phase N --launch `. Do not add a `__monitor` +/// matcher: a live supervisor's pid is in no reachability safety set once +/// `monitor_pid` leaves it (50-22), so `--reap-strays` would treat every +/// live run as a stray. /// /// This is a pure, read-only survey: it never signals a process. Deciding /// whether to act on a result, and re-confirming identity immediately From 3e7a3afd28a52b0ef4c4fe290bcbea6735f52867 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 17:16:29 -0400 Subject: [PATCH 058/112] test(50-13): add failing tests for the publish-before-unlink handoff - V-13: an advancing stage N parks at the after_publish hold with N+1 recorded and alive while its own socket still answers 'advancing' - observer control: the same 5 ms observer sees a simulated pre-clear gap - RV-04 source scan with planted controls for the handed-off marking Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_launch.rs | 579 ++++++++++++++++++++++ 1 file changed, 579 insertions(+) diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index 1902525d..b24bf378 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -7111,4 +7111,583 @@ mod tests { the guard for ANY pipe-owning launch (Claude + Antigravity today)" ); } + + // ---- 50-13: the stage handoff, advance safe points and the gate wait ---- + + /// The `:` hold variable `test_hold` reads (50-13). + const HOLD_ENV: &str = "DEVFLOW_TEST_HOLD"; + + /// The marker a handoff fixture's first agent prints when it completes. + const SUCCESS_MARKER: &str = r#"DEVFLOW_RESULT: {"status":"success"}"#; + + /// How long a handoff child waits for any one step of the chain. + const HANDOFF_STEP_WAIT: Duration = Duration::from_secs(45); + + /// A PATH dir whose fake `claude` answers `--version`. Its first run + /// (stage N) waits for `go` in its working directory (the project root) + /// and then prints `first_marker` and exits 0. Every later run (stage + /// N+1) waits for `go2` and then prints a success marker. Both waits are + /// bounded (about 60 s), so no fake agent outlives a failed test by long, + /// and the fixture's `stop_until` ends the chain after N+1. + fn handoff_claude_dir(first_marker: &str) -> tempfile::TempDir { + let path_dir = agent_free_dir_with_agent_stub("claude"); + let script = format!( + "#!/bin/sh\ncase \"$1\" in --version) exit 0 ;; esac\n\ + if [ ! -e first-done ]; then\n : > first-done\n gate=go\n marker='{first_marker}'\n\ + else\n gate=go2\n marker='{SUCCESS_MARKER}'\nfi\n\ + i=0\nwhile [ ! -e \"$gate\" ] && [ \"$i\" -lt 1200 ]; do sleep 0.05; i=$((i+1)); done\n\ + printf '%s\\n' \"$marker\"\nexit 0\n" + ); + std::fs::write(path_dir.path().join("claude"), script).unwrap(); + let real_sleep = ["/usr/bin/sleep", "/bin/sleep"] + .into_iter() + .map(Path::new) + .find(|path| path.is_file()) + .expect("sleep must exist to run this test"); + std::os::unix::fs::symlink(real_sleep, path_dir.path().join("sleep")).unwrap(); + path_dir + } + + /// The parent half of a handoff test: run `name` in a hermetic child on + /// the advance-running monitor entry, with `DEVFLOW_TEST_HOLD` naming + /// `hold` (when given) and a release path in a parent-owned dir. + fn run_handoff_child(name: &str, hold: Option<&str>, first_marker: &str) { + let path_dir = handoff_claude_dir(first_marker); + let hold_dir = tempfile::Builder::new() + .prefix("dfh") + .tempdir_in("/tmp") + .unwrap(); + let hold_value = hold.map(|point| { + std::ffi::OsString::from(format!( + "{point}:{}", + hold_dir.path().join("release").display() + )) + }); + let mut env: Vec<(&str, &std::ffi::OsStr)> = vec![ + (TEST_MONITOR_FULL_ENV, std::ffi::OsStr::new("1")), + ("DEVFLOW_GATE_TIMEOUT_SECS", std::ffi::OsStr::new("120")), + ]; + if let Some(value) = &hold_value { + env.push((HOLD_ENV, value.as_os_str())); + } + let output = run_hermetic_test_in_child(name, path_dir.path(), &env); + devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, name); + } + + /// One launched stage N in a handoff child: the repo, its phase, N's + /// record and the hermetic cache dir. Dropping it releases any hold and + /// reaps every supervisor whose socket is still in the cache dir, so a + /// failing assertion leaves no `__monitor` or fake agent behind. + struct HandoffRun { + _dir: tempfile::TempDir, + root: PathBuf, + phase: PhaseId, + n: devflow_core::state::SupervisorRecord, + cache: PathBuf, + } + + impl HandoffRun { + /// Launch stage Define for `phase` (capture shape, Supervise mode, + /// `--until plan`) and return once N is recorded. + fn launch(phase: PhaseId) -> Self { + let cache = PathBuf::from( + std::env::var_os("DEVFLOW_CACHE_DIR").expect("the parent passes DEVFLOW_CACHE_DIR"), + ); + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_path_buf(); + init_repo(&root); + std::fs::create_dir_all(root.join(".planning")).unwrap(); + std::fs::write( + root.join(".planning/ROADMAP.md"), + format!("# Roadmap\n\n### Phase {phase}: handoff fixture\n"), + ) + .unwrap(); + let mut state = State::new(phase, AgentKind::Claude, Mode::Supervise, root.clone()); + state.legacy_claude_launch = true; + state.stop_until = Some(Stage::Plan); + workflow::save_state(&state).unwrap(); + let launched = launch_stage(&mut state, None, None); + let n = state.supervisor.clone(); + let run = Self { + _dir: dir, + root, + phase, + n: n.clone().unwrap_or_else(|| { + panic!("the launch must record stage N's supervisor: {launched:?}") + }), + cache, + }; + launched.unwrap(); + run + } + + /// The release path `test_hold` waits on, from `DEVFLOW_TEST_HOLD`. + fn release_path() -> Option { + let hold = std::env::var(HOLD_ENV).ok()?; + let (_, path) = hold.split_once(':')?; + Some(PathBuf::from(path)) + } + + /// Let stage N's agent finish. + fn go(&self) { + std::fs::write(self.root.join("go"), "").unwrap(); + } + + /// Block until a supervisor reports it reached the configured hold. + fn wait_held(&self) { + let release = Self::release_path().expect("this child runs with a hold"); + let held = PathBuf::from(format!("{}.held", release.display())); + assert!( + poll_until(HANDOFF_STEP_WAIT, || held.exists()), + "the {} hold was never reached; events:\n{}", + std::env::var(HOLD_ENV).unwrap_or_default(), + self.events() + ); + } + + /// The run's whole event log, for failure messages. + fn events(&self) -> String { + std::fs::read_to_string(devflow_core::events::events_path(&self.root)) + .unwrap_or_default() + } + + /// Release the configured hold. + fn release(&self) { + if let Some(release) = Self::release_path() { + std::fs::write(release, "").unwrap(); + } + } + + fn state(&self) -> State { + workflow::load_state(&self.root, self.phase).expect("the run's state loads") + } + } + + impl Drop for HandoffRun { + fn drop(&mut self) { + self.release(); + // Two passes: a stage N still advancing when the first pass + // stops it may have launched N+1 by the second. + for _ in 0..2 { + let sockets: Vec = std::fs::read_dir(self.cache.join("sock")) + .map(|entries| { + entries + .filter_map(|entry| Some(entry.ok()?.path())) + .filter(|path| path.extension().is_some_and(|ext| ext == "sock")) + .collect() + }) + .unwrap_or_default(); + for socket in sockets { + let Ok(sidecar) = + devflow_core::supervisor::read_sidecar(&socket.with_extension("json")) + else { + continue; + }; + let mut holder = self.state_holder(); + holder.supervisor = Some(devflow_core::state::SupervisorRecord::new( + socket, + sidecar.launch_id, + sidecar.agent_pgid, + sidecar.agent_start_time, + sidecar.boot_id, + )); + drop(ReapMonitorOnDrop::after_launch(&holder)); + } + } + } + } + + impl HandoffRun { + /// A throwaway `State` to carry one record into `ReapMonitorOnDrop`. + fn state_holder(&self) -> State { + State::new( + self.phase, + AgentKind::Claude, + Mode::Supervise, + self.root.clone(), + ) + } + } + + /// What a handoff observer saw in one poll: the state's recorded + /// supervisor and how its socket probed. + #[derive(Debug, Clone, PartialEq, Eq)] + enum Seen { + Alive(String), + Stale, + Gone, + NoRecord, + Unloadable, + ProbeFailed, + } + + fn observe(root: &Path, phase: PhaseId) -> Seen { + let Ok(state) = workflow::load_state(root, phase) else { + return Seen::Unloadable; + }; + let Some(record) = state.supervisor else { + return Seen::NoRecord; + }; + match devflow_core::supervisor::probe(&record.socket_path) { + Ok(devflow_core::supervisor::Liveness::Alive(reply)) => Seen::Alive(reply.launch_id), + Ok(devflow_core::supervisor::Liveness::Stale) => Seen::Stale, + Ok(devflow_core::supervisor::Liveness::Gone) => Seen::Gone, + Err(_) => Seen::ProbeFailed, + } + } + + /// A thread that polls [`observe`] every 5 ms (and whether `watched` + /// exists) until stopped. + struct Observer { + stop: std::sync::Arc, + handle: std::thread::JoinHandle>, + } + + impl Observer { + fn start(root: &Path, phase: PhaseId, watched: &Path) -> Self { + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (root, watched, flag) = ( + root.to_path_buf(), + watched.to_path_buf(), + std::sync::Arc::clone(&stop), + ); + let handle = std::thread::spawn(move || { + let mut seen = Vec::new(); + while !flag.load(std::sync::atomic::Ordering::Acquire) { + seen.push((observe(&root, phase), watched.exists())); + std::thread::sleep(Duration::from_millis(5)); + } + seen + }); + Self { stop, handle } + } + + fn finish(self) -> Vec<(Seen, bool)> { + self.stop.store(true, std::sync::atomic::Ordering::Release); + self.handle.join().expect("the observer thread") + } + } + + /// Every observation between the first `Alive(from)` and the first + /// `Alive(to)` that reads `Gone`, `Stale` or no record: the handoff's + /// gaps. An error when either endpoint was never observed. + fn handoff_gaps(seen: &[(Seen, bool)], from: &str, to: &str) -> Result, String> { + let first = |id: &str| { + seen.iter() + .position(|(observed, _)| *observed == Seen::Alive(id.to_string())) + }; + let start = first(from).ok_or_else(|| format!("never saw {from} alive"))?; + let end = first(to).ok_or_else(|| format!("never saw {to} alive"))?; + if end < start { + return Err(format!("saw {to} alive before {from}")); + } + Ok(seen[start..end] + .iter() + .map(|(observed, _)| observed.clone()) + .filter(|observed| matches!(observed, Seen::Gone | Seen::Stale | Seen::NoRecord)) + .collect()) + } + + /// V-13 (R-1, T-50-34): a stage handoff publishes N+1 before N unlinks. + /// Stage N's agent finishes, N's in-process advance launches N+1 and + /// records it, and N is parked at the `after_publish` hold (lock still + /// held). At the hold the state names N+1, N+1 probes Alive, and N's + /// socket still exists and answers with owner `advancing` (RV-04: N is + /// not marked handed off inside the advance). Released, N unlinks and + /// N+1 stays Alive. A 5 ms observer must see no Gone, Stale or missing + /// record between N's first Alive and N+1's first Alive; it can miss a + /// gap shorter than 5 ms, so that check is supplementary and + /// `the_handoff_observer_detects_a_pre_clear_gap` shows it sees one. + #[test] + fn stage_handoff_publishes_the_next_socket_before_the_old_one_is_unlinked() { + const NAME: &str = "pipeline_launch::tests::stage_handoff_publishes_the_next_socket_before_the_old_one_is_unlinked"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, Some("after_publish"), SUCCESS_MARKER); + return; + } + let _guard = env_lock(); + let run = HandoffRun::launch(PhaseId::new(1301)); + let observer = Observer::start(&run.root, run.phase, &run.n.socket_path); + run.go(); + run.wait_held(); + + let next = run + .state() + .supervisor + .expect("at the after_publish hold the state records a supervisor"); + assert_ne!( + next.launch_id, run.n.launch_id, + "at the after_publish hold the state names N+1, not N" + ); + match devflow_core::supervisor::probe(&next.socket_path) { + Ok(devflow_core::supervisor::Liveness::Alive(reply)) => { + assert_eq!(reply.launch_id, next.launch_id); + } + other => panic!("N+1 must probe Alive at the hold, got {other:?}"), + } + assert!( + run.n.socket_path.exists(), + "N's socket file must still exist at the after_publish hold" + ); + match devflow_core::supervisor::probe(&run.n.socket_path) { + Ok(devflow_core::supervisor::Liveness::Alive(reply)) => assert_eq!( + reply.owner, "advancing", + "RV-04: N stays Advancing until its advance returns" + ), + other => panic!("N must still answer at the hold, got {other:?}"), + } + + run.release(); + assert!( + poll_until(HANDOFF_STEP_WAIT, || { + devflow_core::supervisor::probe(&run.n.socket_path) + .is_ok_and(|liveness| liveness == devflow_core::supervisor::Liveness::Gone) + }), + "N unlinks its socket once its advance has returned" + ); + match devflow_core::supervisor::probe(&next.socket_path) { + Ok(devflow_core::supervisor::Liveness::Alive(reply)) => { + assert_eq!(reply.launch_id, next.launch_id); + } + other => panic!("N+1 must stay Alive after N exits, got {other:?}"), + } + let seen = observer.finish(); + let gaps = handoff_gaps(&seen, &run.n.launch_id, &next.launch_id) + .unwrap_or_else(|why| panic!("observer: {why}: {seen:?}")); + assert!( + gaps.is_empty(), + "state read GONE or no record across the handoff: {gaps:?}" + ); + } + + /// A live in-process supervisor serving `launch_id` at a short socket + /// path in `dir`, for the observer control. + fn fake_supervisor( + dir: &Path, + root: &Path, + phase: PhaseId, + launch_id: &str, + ) -> ( + devflow_core::supervisor::runtime::Supervisor, + devflow_core::state::SupervisorRecord, + ) { + let path = dir.join(format!("{launch_id}.sock")); + let socket = devflow_core::supervisor::SocketPath::new(path.clone()).unwrap(); + let sidecar: devflow_core::supervisor::Sidecar = + serde_json::from_value(serde_json::json!({ + "launch_id": launch_id, + "project_root": root, + "phase": phase, + "stage": null, + "agent": "claude", + "launch": "capture", + "agent_pid": 0, + "agent_pgid": 0, + "agent_start_time": 0, + "boot_id": "", + "supervisor_pid": std::process::id(), + "supervisor_start_time": 0, + "created_at": "2026-10-17T00:00:00Z", + })) + .unwrap(); + let supervisor = devflow_core::supervisor::runtime::Supervisor::serve(socket, sidecar) + .expect("serve the fake supervisor"); + let record = devflow_core::state::SupervisorRecord::new( + path, + launch_id.to_string(), + 0, + 0, + String::new(), + ); + (supervisor, record) + } + + /// The observer control for V-13: the same observer and gap rule applied + /// to a simulated pre-clear handoff (a save with no supervisor between + /// N's record and N+1's, the WR-04 pitfall) reports the gap; the same + /// sequence without the pre-clear reports none. Both endpoints are real + /// in-process supervisors answering pings. + #[test] + fn the_handoff_observer_detects_a_pre_clear_gap() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let sock_dir = tempfile::Builder::new() + .prefix("dfs") + .tempdir_in("/tmp") + .unwrap(); + let phase = PhaseId::new(1302); + let (sup_n, rec_n) = fake_supervisor(sock_dir.path(), root, phase, "obsn"); + let (sup_next, rec_next) = fake_supervisor(sock_dir.path(), root, phase, "obsnext"); + let save = |record: Option<&devflow_core::state::SupervisorRecord>| { + let mut state = State::new(phase, AgentKind::Claude, Mode::Supervise, root.into()); + state.supervisor = record.cloned(); + workflow::save_state(&state).unwrap(); + }; + let hold = Duration::from_millis(80); + + for pre_clear in [true, false] { + save(Some(&rec_n)); + let observer = Observer::start(root, phase, &rec_n.socket_path); + std::thread::sleep(hold); + if pre_clear { + save(None); + std::thread::sleep(hold); + } + save(Some(&rec_next)); + std::thread::sleep(hold); + let seen = observer.finish(); + let gaps = handoff_gaps(&seen, "obsn", "obsnext") + .unwrap_or_else(|why| panic!("observer: {why}: {seen:?}")); + if pre_clear { + assert!( + gaps.contains(&Seen::NoRecord), + "the observer must see the pre-clear gap: {seen:?}" + ); + } else { + assert!(gaps.is_empty(), "control: no pre-clear, no gap: {gaps:?}"); + } + } + sup_n.finish(); + sup_next.finish(); + } + + /// The RV-04 order rule, as a pure scan over `src`: `spawn_agent_and_record` + /// records N+1 as the successor and never marks N handed off, and + /// `run_monitor` takes the successor and marks the hand-off. Each function + /// is its comment-stripped lines from the column-0 line starting with its + /// signature through the next line that is exactly `}`. + fn handoff_order_violations(src: &str) -> Vec { + fn body<'a>(src: &'a str, signature: &str, floor: usize) -> Option> { + let mut lines = src.lines(); + let first = lines.by_ref().find(|line| line.starts_with(signature))?; + let mut body = vec![first]; + for line in lines { + if line.trim_start().starts_with("//") { + continue; + } + body.push(line); + if line == "}" { + break; + } + } + (body.len() >= floor).then_some(body) + } + let has = |lines: &[&str], needle: &str| lines.iter().any(|line| line.contains(needle)); + let mut violations = Vec::new(); + match body(src, "fn spawn_agent_and_record(", 40) { + None => violations.push(SCAN_SPAWN_NOT_FOUND.to_string()), + Some(lines) => { + if has(&lines, "OwnerPhase::HandedOff") { + violations.push(SCAN_SPAWN_HANDS_OFF.to_string()); + } + if !has(&lines, "record_successor(") { + violations.push(SCAN_SPAWN_NO_SUCCESSOR.to_string()); + } + } + } + match body(src, "pub(crate) fn run_monitor(", 10) { + None => violations.push(SCAN_RUN_NOT_FOUND.to_string()), + Some(lines) => { + if !has(&lines, "OwnerPhase::HandedOff") { + violations.push(SCAN_RUN_NO_HAND_OFF.to_string()); + } + if !has(&lines, "take_successor(") { + violations.push(SCAN_RUN_NO_TAKE.to_string()); + } + } + } + violations + } + + const SCAN_SPAWN_NOT_FOUND: &str = + "spawn_agent_and_record not found (or under 40 comment-stripped lines)"; + const SCAN_SPAWN_HANDS_OFF: &str = "spawn_agent_and_record enters the handed-off phase"; + const SCAN_SPAWN_NO_SUCCESSOR: &str = "spawn_agent_and_record never calls record_successor"; + const SCAN_RUN_NOT_FOUND: &str = "run_monitor not found (or under 10 comment-stripped lines)"; + const SCAN_RUN_NO_HAND_OFF: &str = "run_monitor never enters the handed-off phase"; + const SCAN_RUN_NO_TAKE: &str = "run_monitor never calls take_successor"; + + /// RV-04 (permanent source scan): N is marked handed off only in + /// `run_monitor`, after its advance returned and only when the advance + /// recorded a successor; `spawn_agent_and_record` records the successor + /// and leaves N `Advancing`. Planted controls prove the scan can flag + /// each violation and passes a correct layout. + #[test] + fn the_handed_off_marking_stays_in_run_monitor_after_the_advance() { + let real = handoff_order_violations(include_str!("pipeline_launch.rs")); + assert!(real.is_empty(), "pipeline_launch.rs: {real:?}"); + + let pad = |n: usize| " let _filler = 0;\n".repeat(n); + let spawn = |extra: &str| format!("fn spawn_agent_and_record(\n{}{extra}}}\n", pad(40)); + let run = |extra: &str| format!("pub(crate) fn run_monitor(\n{}{extra}}}\n", pad(10)); + let good_spawn = spawn(" record_successor(record);\n"); + let good_run = run( + " if let Some(next) = take_successor() {\n token.enter(OwnerPhase::HandedOff);\n }\n", + ); + let only = |violation: &str| vec![violation.to_string()]; + + assert_eq!( + handoff_order_violations(&format!("{good_spawn}{good_run}")), + Vec::::new(), + "a correct pair" + ); + assert_eq!( + handoff_order_violations(&format!( + "{}{good_run}", + spawn(" record_successor(record);\n token.enter(OwnerPhase::HandedOff);\n") + )), + only(SCAN_SPAWN_HANDS_OFF) + ); + assert_eq!( + handoff_order_violations(&format!( + "{}{good_run}", + spawn( + " record_successor(record);\n // token.enter(OwnerPhase::HandedOff);\n" + ) + )), + Vec::::new(), + "a comment line is not code" + ); + assert_eq!( + handoff_order_violations(&format!("{}{good_run}", spawn(""))), + only(SCAN_SPAWN_NO_SUCCESSOR) + ); + assert_eq!( + handoff_order_violations(&format!( + "{good_spawn}{}", + run(" let _next = take_successor();\n") + )), + only(SCAN_RUN_NO_HAND_OFF) + ); + assert_eq!( + handoff_order_violations(&format!( + "{good_spawn}{}", + run(" token.enter(OwnerPhase::HandedOff);\n") + )), + only(SCAN_RUN_NO_TAKE) + ); + assert_eq!( + handoff_order_violations(&good_run), + only(SCAN_SPAWN_NOT_FOUND) + ); + assert_eq!( + handoff_order_violations(&good_spawn), + only(SCAN_RUN_NOT_FOUND) + ); + assert_eq!( + handoff_order_violations(&format!( + "fn spawn_agent_and_record(\n{} record_successor(record);\n}}\n{good_run}", + pad(5) + )), + only(SCAN_SPAWN_NOT_FOUND), + "a body under its floor" + ); + assert_eq!( + handoff_order_violations(&format!( + "{good_spawn}pub(crate) fn run_monitor(\n take_successor();\n token.enter(OwnerPhase::HandedOff);\n}}\n" + )), + only(SCAN_RUN_NOT_FOUND), + "a run_monitor under its floor" + ); + } } From 8a97aebd2988d0a2129fb71d19966ea1eac1dfd0 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 17:18:50 -0400 Subject: [PATCH 059/112] feat(50-13): publish the next supervisor before the old one unlinks - StopTokenGuard: a thread-local slot for the supervisor's stop token and the successor its advance published (record_successor/take_successor) - run_monitor marks itself handed off only after the advance returned and only when a successor was recorded, then finishes (RV-04) - cfg(test) test_hold points before_spawn, before_publish, before_record_save and after_publish, active only inside an advance - run_monitor takes the supervise-only flag directly (supervise_monitor folded in); main and the harness entry pass it Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/main.rs | 31 ++-- crates/devflow-cli/src/pipeline_launch.rs | 168 ++++++++++++++++++++-- crates/devflow-cli/src/test_support.rs | 2 +- 3 files changed, 171 insertions(+), 30 deletions(-) diff --git a/crates/devflow-cli/src/main.rs b/crates/devflow-cli/src/main.rs index 58023019..0e04d60b 100644 --- a/crates/devflow-cli/src/main.rs +++ b/crates/devflow-cli/src/main.rs @@ -613,20 +613,23 @@ fn run() -> Result<(), CliError> { prompt_file, idle_timeout_secs, argv, - } => run_monitor(&pipeline_launch::MonitorInvocation { - project_root: project_root(project)?, - phase, - stage, - workdir, - agent, - launch, - socket, - launch_id, - keep_env, - prompt_file, - idle_timeout_secs, - argv, - }), + } => run_monitor( + &pipeline_launch::MonitorInvocation { + project_root: project_root(project)?, + phase, + stage, + workdir, + agent, + launch, + socket, + launch_id, + keep_env, + prompt_file, + idle_timeout_secs, + argv, + }, + true, + ), Command::Resume { phase, agent, diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index b24bf378..e7fd41e7 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -40,8 +40,9 @@ use devflow_core::outcome_policy::{self, Action}; use devflow_core::phase_id::PhaseId; use devflow_core::prompt; use devflow_core::stage::Stage; +use devflow_core::state::SupervisorRecord; use devflow_core::state::{AgentKind, ApprovedCheckpoint, CheckpointApproval, State}; -use devflow_core::supervisor::runtime::AgentOutcome; +use devflow_core::supervisor::runtime::{AgentOutcome, OwnerPhase, StopToken}; use devflow_core::verify::CheckpointDeclaration; use devflow_core::{ agent_result, agents, canary, events, gsd_config, lock, mode, monitor, verify, workflow, @@ -952,25 +953,139 @@ pub(crate) const SUPERVISOR_STOP_REASON: &str = /// records the stop. const STOP_RECORD_LOCK_WAIT: Duration = Duration::from_secs(30); +/// What a supervisor's advance reads on this thread (50-13): its stop token, +/// for the advance's safe points, and the successor the advance recorded. +struct EngagedAdvance { + #[cfg_attr( + not(test), + expect(dead_code, reason = "read by the advance safe points (50-13 Task 2)") + )] + token: StopToken, + successor: Option, +} + +thread_local! { + /// Set only while [`StopTokenGuard`] is alive. Thread-local, not a + /// process-global static, so parallel in-crate tests never share it; an + /// advance run by anything other than a supervisor (the hidden + /// `devflow advance`, a test calling `advance_with` directly) sees `None` + /// and every safe point proceeds. + static ENGAGED_ADVANCE: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +/// Makes a supervisor's stop token visible to the safe points of the +/// advance running on this thread, for as long as the guard lives (RAII, +/// the [`AutoChainGuard`] shape). Dropping it clears the token and the +/// successor slot. +struct StopTokenGuard { + /// Keeps the guard on the thread whose slot it set. + _not_send: std::marker::PhantomData<*const ()>, +} + +impl StopTokenGuard { + fn engage(token: StopToken) -> Self { + ENGAGED_ADVANCE.with(|slot| { + *slot.borrow_mut() = Some(EngagedAdvance { + token, + successor: None, + }); + }); + Self { + _not_send: std::marker::PhantomData, + } + } +} + +impl Drop for StopTokenGuard { + fn drop(&mut self) { + ENGAGED_ADVANCE.with(|slot| slot.borrow_mut().take()); + } +} + +/// Run `f` on the engaged stop token, or return `None` when no supervisor's +/// advance is running on this thread. +#[cfg_attr( + not(test), + expect(dead_code, reason = "called by the advance safe points (50-13 Task 2)") +)] +fn with_stop_token(f: impl FnOnce(&StopToken) -> R) -> Option { + let token = ENGAGED_ADVANCE.with(|slot| slot.borrow().as_ref().map(|e| e.token.clone())); + token.map(|token| f(&token)) +} + +/// Remember `record` as the successor this advance published, so +/// [`run_monitor`] marks this supervisor handed off once the advance has +/// returned. A no-op when no supervisor's advance is engaged. +fn record_successor(record: SupervisorRecord) { + ENGAGED_ADVANCE.with(|slot| { + if let Some(engaged) = slot.borrow_mut().as_mut() { + engaged.successor = Some(record); + } + }); +} + +/// Take the successor [`record_successor`] stored, if any. +fn take_successor() -> Option { + ENGAGED_ADVANCE.with(|slot| { + slot.borrow_mut() + .as_mut() + .and_then(|engaged| engaged.successor.take()) + }) +} + +/// How long a [`test_hold`] waits for its release file. +#[cfg(test)] +const TEST_HOLD_WAIT: Duration = Duration::from_secs(30); + +/// Test-only hold point (50-13): inside a supervisor's own advance (a stop +/// token engaged), when `DEVFLOW_TEST_HOLD` is `:` for this +/// `point`, write `.held` and wait (bounded by [`TEST_HOLD_WAIT`]) until +/// `` exists, so a test can land a stop, a signal or a kill in a chosen +/// window. A launch made outside an advance (the test's own first launch) +/// never holds. Compiled out of non-test builds, together with every call. +#[cfg(test)] +fn test_hold(point: &str) { + if with_stop_token(|_| ()).is_none() { + return; + } + let Ok(hold) = std::env::var("DEVFLOW_TEST_HOLD") else { + return; + }; + let Some((at, path)) = hold.split_once(':') else { + return; + }; + if at != point { + return; + } + let _ = std::fs::write(format!("{path}.held"), ""); + let release = Path::new(path); + let deadline = std::time::Instant::now() + TEST_HOLD_WAIT; + while !release.exists() && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } +} + /// The `__monitor` process body (50-11): supervise the stage's agent under /// an addressed supervisor (`monitor::run_supervised_stage`), then, only /// when the agent completed on its own, advance the stage machine /// in-process (D-02). A stopped run is recorded as stopped and never /// advances. /// +/// The handoff (50-13, R-1): the advance runs with this supervisor's stop +/// token engaged for its safe points and, when it launches the next stage, +/// records the next supervisor under the per-phase lock before it returns. +/// Only after it has returned (the lock released) is this supervisor marked +/// handed off, so every stop that reaches it during the advance is a +/// flagged `Advancing` stop (RV-04), and only then does `finish` unlink its +/// socket: the state never names a socket that is gone. +/// +/// `advance` is `true` on every production path; the test build's +/// supervise-only monitor harness entry (50-10) passes `false` so an +/// in-crate launch supervises its agent without moving the stage machine on. +/// /// Runs in the detached `__monitor` process, never in the operator's CLI. -pub(crate) fn run_monitor(invocation: &MonitorInvocation) -> Result<(), CliError> { - supervise_monitor(invocation, true) -} - -/// [`run_monitor`]'s body. `advance` is `true` on every production path; the -/// test build's supervise-only monitor harness entry (50-10) passes `false` -/// so an in-crate launch supervises its agent without moving the stage -/// machine on. -pub(crate) fn supervise_monitor( - invocation: &MonitorInvocation, - advance: bool, -) -> Result<(), CliError> { +pub(crate) fn run_monitor(invocation: &MonitorInvocation, advance: bool) -> Result<(), CliError> { let MonitorInvocation { project_root, phase, @@ -1029,7 +1144,15 @@ pub(crate) fn supervise_monitor( let result = match run.outcome { AgentOutcome::Completed(_) if advance => { - advance_for_stage(project_root, Some(phase), stage) + let stop_guard = StopTokenGuard::engage(run.supervisor.token()); + let advanced = advance_for_stage(project_root, Some(phase), stage); + // The advance has returned and released the per-phase lock. Only + // a run whose advance published a successor is handed off. + if take_successor().is_some() { + run.supervisor.token().enter(OwnerPhase::HandedOff); + } + drop(stop_guard); + advanced } AgentOutcome::Completed(_) => { wait_for_launch_record(project_root, phase, &invocation.launch_id); @@ -1292,8 +1415,12 @@ fn spawn_agent_and_record( monitor::MonitorLaunch::PipeOwning { .. } => "stream", monitor::MonitorLaunch::Legacy => "capture", }; + #[cfg(test)] + test_hold("before_spawn"); let published = monitor::spawn_monitor(state, program, args, extra_env, launch, &ctx) .map_err(|err| CliError::Message(format!("could not spawn monitor: {err}")))?; + #[cfg(test)] + test_hold("before_publish"); let pid = published.supervisor_pid; let socket_path = published.record.socket_path.display().to_string(); let launch_id = published.record.launch_id.clone(); @@ -1303,9 +1430,20 @@ fn spawn_agent_and_record( // in this same save; it is never cleared first (WR-04's pre-clear above // applies to `monitor_pid` only). `monitor_pid` names the supervisor // until 50-22 removes it. + // + // The handoff (50-13, R-1): this save is the publish. The previous + // supervisor, when this launch is its advance, stays alive and + // `Advancing` through it (RV-04: never handed off here) and only + // unlinks its own socket after the advance has returned, so a reader + // between the two always finds a live address. + #[cfg(test)] + test_hold("before_record_save"); state.monitor_pid = Some(pid); - state.supervisor = Some(published.record); + state.supervisor = Some(published.record.clone()); workflow::save_state(state)?; + record_successor(published.record); + #[cfg(test)] + test_hold("after_publish"); match devflow_core::ship::consume_cron_instructions(&state.project_root, state.phase) { Ok(Some(path_kind)) => { let path_kind = match path_kind { diff --git a/crates/devflow-cli/src/test_support.rs b/crates/devflow-cli/src/test_support.rs index 6667d0d6..05644d9a 100644 --- a/crates/devflow-cli/src/test_support.rs +++ b/crates/devflow-cli/src/test_support.rs @@ -744,7 +744,7 @@ fn run_monitor_harness_entry(advance: bool) { idle_timeout_secs, argv, }; - crate::pipeline_launch::supervise_monitor(&invocation, advance) + crate::pipeline_launch::run_monitor(&invocation, advance) .expect("the harness-entry monitor must supervise its agent"); } From f2f8a6d0282af1753154079fca34ff52ae96a38f Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 17:21:20 -0400 Subject: [PATCH 060/112] test(50-13): add failing tests for advance safe points and stopped state - a stop taken at the before_spawn hold must end the run at safe point 2, with a no-stop control that launches the next stage - a stop at before_publish must shut N+1 down without publishing it and answer under the lock-deadlock bound - advance_with must refuse a stopped state (control: unstopped advances) - a stopped supervisor whose state will not load still answers stopped Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_launch.rs | 355 ++++++++++++++++++++++ 1 file changed, 355 insertions(+) diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index e7fd41e7..20029313 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -949,6 +949,14 @@ pub(crate) struct MonitorInvocation { pub(crate) const SUPERVISOR_STOP_REASON: &str = "stopped through the supervisor (socket shutdown or SIGTERM/SIGINT)"; +/// The stop reason an advance records when it stops at a safe point +/// (50-13, R-2). +#[cfg_attr( + not(test), + expect(dead_code, reason = "RED scaffold: the safe points (50-13 Task 2) record it") +)] +pub(crate) const ADVANCE_STOP_REASON: &str = "stopped through the supervisor during advance"; + /// How long a stopped supervisor waits for the per-phase lock before it /// records the stop. const STOP_RECORD_LOCK_WAIT: Duration = Duration::from_secs(30); @@ -7399,6 +7407,19 @@ mod tests { fn state(&self) -> State { workflow::load_state(&self.root, self.phase).expect("the run's state loads") } + + /// Every `.sock` in the cache dir's `sock/` other than N's. + fn other_sockets(&self) -> Vec { + std::fs::read_dir(self.cache.join("sock")) + .map(|entries| { + entries + .filter_map(|entry| Some(entry.ok()?.path())) + .filter(|path| path.extension().is_some_and(|ext| ext == "sock")) + .filter(|path| *path != self.n.socket_path) + .collect() + }) + .unwrap_or_default() + } } impl Drop for HandoffRun { @@ -7828,4 +7849,338 @@ mod tests { "a run_monitor under its floor" ); } + + /// The run's events named `name`. + fn events_named(root: &Path, name: &str) -> Vec { + std::fs::read_to_string(devflow_core::events::events_path(root)) + .unwrap_or_default() + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|event| event["event"].as_str() == Some(name)) + .collect() + } + + /// Send `shutdown` to the supervisor at `socket` on the wire, wait for + /// its `accepted` line (returned first), run `then`, and return the final + /// line read within `final_wait`. Unlike `request_shutdown`, the caller + /// acts at a known point: after the supervisor has taken the stop. + fn shutdown_accepted_then( + socket: &Path, + then: impl FnOnce(), + final_wait: Duration, + ) -> (serde_json::Value, serde_json::Value) { + use std::io::{BufRead as _, Write as _}; + let stream = std::os::unix::net::UnixStream::connect(socket).expect("connect to N"); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + (&stream) + .write_all(b"{\"op\":\"shutdown\"}\n") + .expect("send shutdown"); + let mut reader = std::io::BufReader::new(&stream); + let mut line = String::new(); + reader.read_line(&mut line).expect("the accepted line"); + let accepted: serde_json::Value = serde_json::from_str(&line) + .unwrap_or_else(|err| panic!("accepted line {line:?}: {err}")); + then(); + stream.set_read_timeout(Some(final_wait)).unwrap(); + line.clear(); + reader.read_line(&mut line).expect("the final line"); + let last: serde_json::Value = + serde_json::from_str(&line).unwrap_or_else(|err| panic!("final line {line:?}: {err}")); + (accepted, last) + } + + /// Whether any process is left in group `pgid` (`kill(-pgid, 0)`). + fn group_alive(pgid: u32) -> bool { + let Ok(signed) = libc::pid_t::try_from(pgid) else { + return false; + }; + assert!(signed > 1, "refusing to probe group {signed}"); + // SAFETY: signal 0 only checks for existence; `-signed < -1` names + // one process group, never our own (0) or every process (-1). + unsafe { libc::kill(-signed, 0) == 0 } + } + + /// The body of the held-advance pair: N parks at `before_spawn` (safe + /// point 2 follows the hold), then a stop is (or is not) taken before the + /// release. + fn held_advance_before_spawn(phase: PhaseId, stop: bool) { + let _guard = env_lock(); + let run = HandoffRun::launch(phase); + run.go(); + run.wait_held(); + let release = || run.release(); + if stop { + let (accepted, last) = + shutdown_accepted_then(&run.n.socket_path, release, Duration::from_secs(30)); + assert_eq!(accepted["owner"], "advancing", "{accepted}"); + assert_eq!(last["stopped"], true, "N answers stopped: {last}"); + assert!( + poll_until(HANDOFF_STEP_WAIT, || { + devflow_core::supervisor::probe(&run.n.socket_path) + .is_ok_and(|liveness| liveness == devflow_core::supervisor::Liveness::Gone) + }), + "N's socket is gone after the stop" + ); + assert!( + poll_until(Duration::from_secs(5), || matches!( + lock::holder_status(&run.root, run.phase), + lock::HolderStatus::NoHolder + )), + "the per-phase lock is released" + ); + let state = run.state(); + assert!(state.stopped, "state.stopped after a stop at safe point 2"); + assert_eq!(state.stop_reason.as_deref(), Some(ADVANCE_STOP_REASON)); + assert_eq!( + stage_launched_count(&run.root, run.phase), + 1, + "no next stage is launched: {}", + run.events() + ); + let stopped = events_named(&run.root, "advance_stopped"); + assert_eq!(stopped.len(), 1, "{}", run.events()); + assert_eq!(stopped[0]["safe_point"], 2, "{}", stopped[0]); + assert!( + run.other_sockets().is_empty(), + "no next supervisor was spawned" + ); + } else { + release(); + assert!( + poll_until(HANDOFF_STEP_WAIT, || stage_launched_count( + &run.root, run.phase + ) == 2), + "control: without a stop the held advance launches the next stage: {}", + run.events() + ); + let next = run + .state() + .supervisor + .expect("control: the next stage is recorded"); + assert_ne!(next.launch_id, run.n.launch_id); + assert!( + matches!( + devflow_core::supervisor::probe(&next.socket_path), + Ok(devflow_core::supervisor::Liveness::Alive(_)) + ), + "control: the next supervisor is Alive" + ); + assert!(!run.state().stopped, "control: not stopped"); + } + } + + /// R-2, safe point 2: a stop taken while N is advancing is honoured right + /// before the next stage's spawn. Nothing is launched, the state is + /// stopped with the safe-point reason, the lock is released and N's + /// socket is gone. + #[test] + fn a_stop_during_advance_stops_at_the_next_safe_point_and_launches_nothing() { + const NAME: &str = "pipeline_launch::tests::a_stop_during_advance_stops_at_the_next_safe_point_and_launches_nothing"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, Some("before_spawn"), SUCCESS_MARKER); + return; + } + held_advance_before_spawn(PhaseId::new(1303), true); + } + + /// The control for the held-advance pair: the same hold and no stop, so + /// the next stage is launched and Alive. + #[test] + fn without_a_stop_the_held_advance_launches_the_next_stage() { + const NAME: &str = + "pipeline_launch::tests::without_a_stop_the_held_advance_launches_the_next_stage"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, Some("before_spawn"), SUCCESS_MARKER); + return; + } + held_advance_before_spawn(PhaseId::new(1304), false); + } + + /// R-2, safe point 3 (T-50-54): a stop taken while N holds the lock + /// between N+1's ping and its publish shuts N+1 down through its socket + /// and does not publish it. The state keeps N's record and is stopped with + /// N's safe-point reason; no agent of N+1 survives; and N answers + /// `stopped` in under `TERMINATE_VERIFY_WAIT + 3 s`, below both bounded + /// waits a lock deadlock would sit out (N's final timeout and N+1's lock + /// wait), so a stopped N+1 that waited on N's lock fails the timing. + #[test] + fn a_stop_between_the_next_spawn_and_its_publish_shuts_the_next_supervisor_down() { + const NAME: &str = "pipeline_launch::tests::a_stop_between_the_next_spawn_and_its_publish_shuts_the_next_supervisor_down"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, Some("before_publish"), SUCCESS_MARKER); + return; + } + let _guard = env_lock(); + let run = HandoffRun::launch(PhaseId::new(1305)); + run.go(); + run.wait_held(); + let next_socket = match run.other_sockets().as_slice() { + [one] => one.clone(), + other => panic!("exactly one next supervisor is spawned at the hold: {other:?}"), + }; + let next = devflow_core::supervisor::read_sidecar(&next_socket.with_extension("json")) + .expect("N+1's sidecar"); + assert!(group_alive(next.agent_pgid), "N+1's agent runs at the hold"); + + let mut released_at = None; + let (accepted, last) = shutdown_accepted_then( + &run.n.socket_path, + || { + released_at = Some(std::time::Instant::now()); + run.release(); + }, + Duration::from_secs(30), + ); + let took = released_at.expect("released").elapsed(); + assert_eq!(accepted["owner"], "advancing", "{accepted}"); + assert_eq!(last["stopped"], true, "N answers stopped: {last}"); + let bound = devflow_core::agent::TERMINATE_VERIFY_WAIT + Duration::from_secs(3); + assert!( + took < bound, + "N's stop took {took:?}, not under {bound:?}: a lock deadlock between N and N+1" + ); + assert!( + poll_until(HANDOFF_STEP_WAIT, || { + devflow_core::supervisor::probe(&next_socket) + .is_ok_and(|liveness| liveness == devflow_core::supervisor::Liveness::Gone) + }), + "N+1 was shut down through its socket" + ); + assert!( + poll_until(Duration::from_secs(5), || !group_alive(next.agent_pgid)), + "no agent of N+1 survives" + ); + let state = run.state(); + assert_eq!( + state.supervisor.as_ref().map(|record| &record.launch_id), + Some(&run.n.launch_id), + "N+1 was never published: the state keeps N's record" + ); + assert!(state.stopped, "state.stopped after a stop at safe point 3"); + assert_eq!(state.stop_reason.as_deref(), Some(ADVANCE_STOP_REASON)); + assert_eq!( + stage_launched_count(&run.root, run.phase), + 1, + "{}", + run.events() + ); + let stopped = events_named(&run.root, "advance_stopped"); + assert_eq!(stopped.len(), 1, "{}", run.events()); + assert_eq!(stopped[0]["safe_point"], 3, "{}", stopped[0]); + } + + /// Write a capture whose Layer 1 marker reads success, with exit code 0. + fn write_success_capture(root: &Path, phase: PhaseId) { + let stdout = devflow_core::agent_result::stdout_path(root, phase); + std::fs::create_dir_all(stdout.parent().unwrap()).unwrap(); + std::fs::write(&stdout, format!("work done\n{SUCCESS_MARKER}\n")).unwrap(); + std::fs::write( + devflow_core::agent_result::exit_code_path(root, phase), + "0\n", + ) + .unwrap(); + } + + /// 999.138 residual (A6): `advance_with` refuses a stopped state with an + /// `advance_failed {reason: "stopped"}` event and an error naming + /// `devflow resume`. Control: the same state with `stopped: false` + /// advances (here to its `--until define` stop, so nothing launches). + #[test] + fn advance_refuses_a_stopped_state() { + let _guard = env_lock(); + for stopped in [true, false] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let phase = PhaseId::new(1307); + let mut state = State::new( + phase, + AgentKind::Claude, + Mode::Supervise, + root.to_path_buf(), + ); + state.stop_until = Some(Stage::Define); + state.stopped = stopped; + workflow::save_state(&state).unwrap(); + write_success_capture(root, phase); + + let result = advance_with(root, phase, Some(Stage::Define), Duration::from_secs(5)); + let failed = events_named(root, "advance_failed"); + if stopped { + let err = result.expect_err("a stopped state must not advance"); + assert!( + err.to_string().contains("devflow resume --phase 1307"), + "the error names the repair: {err}" + ); + assert_eq!(failed.len(), 1, "{failed:?}"); + assert_eq!(failed[0]["reason"], "stopped", "{}", failed[0]); + assert!( + events_named(root, "advance_evaluated").is_empty(), + "a refused advance evaluates nothing" + ); + } else { + result.expect("control: an unstopped state advances"); + assert!(failed.is_empty(), "control: {failed:?}"); + assert_eq!(events_named(root, "advance_evaluated").len(), 1); + let finished = events_named(root, "workflow_finished"); + assert_eq!(finished.len(), 1, "control reaches its --until stop"); + } + } + } + + /// T-50-68: a supervisor stopped while its state file will not load still + /// answers `stopped` (not an error, a closed connection or the final + /// timeout), its socket goes, its agent group goes, and the state bytes + /// are unchanged: the no-write path. Control: the same stop on a run + /// whose state loads records `stopped`, so the unchanged bytes are the + /// no-write path and not a harness that cannot see a write. + #[test] + fn a_stopped_supervisor_whose_state_will_not_load_still_answers_stopped() { + const NAME: &str = "pipeline_launch::tests::a_stopped_supervisor_whose_state_will_not_load_still_answers_stopped"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, None, SUCCESS_MARKER); + return; + } + let _guard = env_lock(); + let wait = 2 * devflow_core::agent::TERMINATE_VERIFY_WAIT + Duration::from_secs(5); + for corrupt in [true, false] { + let run = HandoffRun::launch(PhaseId::new(if corrupt { 1308 } else { 1309 })); + let state_file = workflow::state_path(&run.root, run.phase); + if corrupt { + std::fs::write(&state_file, b"{ this is not json").unwrap(); + } + let before = std::fs::read(&state_file).unwrap(); + let outcome = devflow_core::supervisor::request_shutdown(&run.n.socket_path, wait); + assert!( + matches!( + outcome, + Ok(devflow_core::supervisor::ShutdownOutcome::Stopped { .. }) + ), + "corrupt={corrupt}: the stop reads Stopped, got {outcome:?}" + ); + assert!( + poll_until(HANDOFF_STEP_WAIT, || { + devflow_core::supervisor::probe(&run.n.socket_path) + .is_ok_and(|liveness| liveness == devflow_core::supervisor::Liveness::Gone) + }), + "corrupt={corrupt}: the socket is gone" + ); + assert!( + poll_until(Duration::from_secs(5), || !group_alive(run.n.agent_pgid)), + "corrupt={corrupt}: the agent group is gone" + ); + if corrupt { + assert_eq!( + std::fs::read(&state_file).unwrap(), + before, + "the state that will not load is left byte for byte" + ); + } else { + let state = run.state(); + assert!(state.stopped, "control: a loadable state records stopped"); + assert_eq!(state.stop_reason.as_deref(), Some(SUPERVISOR_STOP_REASON)); + } + } + } } From 08d130eaca51a902f4c5c2841e4e14637a004f54 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 17:25:05 -0400 Subject: [PATCH 061/112] feat(50-13): honour a stop at the advance safe points - safe point 1 after advance_with loads state, 2 before the next spawn, 3 after the next supervisor's ping: record stopped and return - safe point 3 shuts the unpublished next supervisor down by socket - advance_with refuses a stopped state (advance_failed reason stopped) - a stopped supervisor records its stop only when the state names its launch (lockless check, then under the lock), never fails before finish, and waits at most 2 x TERMINATE_VERIFY_WAIT + 5 s for the lock Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_launch.rs | 185 +++++++++++++++++++--- 1 file changed, 159 insertions(+), 26 deletions(-) diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index 20029313..50307a0e 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -42,7 +42,7 @@ use devflow_core::prompt; use devflow_core::stage::Stage; use devflow_core::state::SupervisorRecord; use devflow_core::state::{AgentKind, ApprovedCheckpoint, CheckpointApproval, State}; -use devflow_core::supervisor::runtime::{AgentOutcome, OwnerPhase, StopToken}; +use devflow_core::supervisor::runtime::{AgentOutcome, OwnerPhase, SafePoint, StopToken}; use devflow_core::verify::CheckpointDeclaration; use devflow_core::{ agent_result, agents, canary, events, gsd_config, lock, mode, monitor, verify, workflow, @@ -951,23 +951,19 @@ pub(crate) const SUPERVISOR_STOP_REASON: &str = /// The stop reason an advance records when it stops at a safe point /// (50-13, R-2). -#[cfg_attr( - not(test), - expect(dead_code, reason = "RED scaffold: the safe points (50-13 Task 2) record it") -)] pub(crate) const ADVANCE_STOP_REASON: &str = "stopped through the supervisor during advance"; -/// How long a stopped supervisor waits for the per-phase lock before it -/// records the stop. -const STOP_RECORD_LOCK_WAIT: Duration = Duration::from_secs(30); +/// The bound on the two waits a stop can put between two supervisors +/// (50-13, T-50-54): a stopped supervisor's wait for the per-phase lock +/// before it records its stop, and safe point 3's wait for an unpublished +/// next supervisor's final `stopped` line. `2 × TERMINATE_VERIFY_WAIT + 5 s`: +/// the next supervisor's kill grace plus margin. +const SUPERVISOR_STOP_WAIT: Duration = + Duration::from_secs(2 * devflow_core::agent::TERMINATE_VERIFY_WAIT.as_secs() + 5); /// What a supervisor's advance reads on this thread (50-13): its stop token, /// for the advance's safe points, and the successor the advance recorded. struct EngagedAdvance { - #[cfg_attr( - not(test), - expect(dead_code, reason = "read by the advance safe points (50-13 Task 2)") - )] token: StopToken, successor: Option, } @@ -1013,10 +1009,6 @@ impl Drop for StopTokenGuard { /// Run `f` on the engaged stop token, or return `None` when no supervisor's /// advance is running on this thread. -#[cfg_attr( - not(test), - expect(dead_code, reason = "called by the advance safe points (50-13 Task 2)") -)] fn with_stop_token(f: impl FnOnce(&StopToken) -> R) -> Option { let token = ENGAGED_ADVANCE.with(|slot| slot.borrow().as_ref().map(|e| e.token.clone())); token.map(|token| f(&token)) @@ -1042,6 +1034,63 @@ fn take_successor() -> Option { }) } +/// An advance safe point (50-13, R-2): when the engaged supervisor's stop +/// has been requested, record the stop in `state` and return `true`, so the +/// caller returns at once without launching anything (the per-phase lock +/// guard drops on the way out, and the supervisor's `finish` answers a +/// parked caller `stopped`). `false` with no stop requested, or outside a +/// supervisor's advance. +fn stopped_at_safe_point(state: &mut State, safe_point: u8) -> Result { + if with_stop_token(StopToken::safe_point) != Some(SafePoint::StopHere) { + return Ok(false); + } + record_advance_stop(state, safe_point)?; + Ok(true) +} + +/// Persist a stop an advance honoured at `safe_point`: `stopped` with +/// [`ADVANCE_STOP_REASON`], and an `advance_stopped` event naming the point. +fn record_advance_stop(state: &mut State, safe_point: u8) -> Result<(), CliError> { + state.stopped = true; + state.stop_reason = Some(ADVANCE_STOP_REASON.to_string()); + workflow::save_state(state)?; + events::emit( + &state.project_root, + state.phase, + "advance_stopped", + serde_json::json!({ + "stage": state.stage.to_string(), + "safe_point": safe_point, + }), + ); + info!( + "phase {}: {ADVANCE_STOP_REASON} (safe point {safe_point})", + state.phase + ); + Ok(()) +} + +/// Safe point 3's shutdown of a next supervisor that was spawned and +/// answered its ping but is not published: ask it to stop through its own +/// socket, bounded by [`SUPERVISOR_STOP_WAIT`]. It records nothing itself +/// (the state does not name it), so this launcher's stop is the record. +fn shut_down_unpublished(record: &SupervisorRecord) { + match devflow_core::supervisor::request_shutdown(&record.socket_path, SUPERVISOR_STOP_WAIT) { + Ok(devflow_core::supervisor::ShutdownOutcome::Stopped { .. }) => info!( + "stopped the unpublished next supervisor at {}", + record.socket_path.display() + ), + Ok(other) => warn!( + "the unpublished next supervisor at {} answered {other:?} to shutdown", + record.socket_path.display() + ), + Err(err) => warn!( + "could not stop the unpublished next supervisor at {}: {err}", + record.socket_path.display() + ), + } +} + /// How long a [`test_hold`] waits for its release file. #[cfg(test)] const TEST_HOLD_WAIT: Duration = Duration::from_secs(30); @@ -1166,7 +1215,10 @@ pub(crate) fn run_monitor(invocation: &MonitorInvocation, advance: bool) -> Resu wait_for_launch_record(project_root, phase, &invocation.launch_id); Ok(()) } - _ => record_supervisor_stop(project_root, phase), + _ => { + record_supervisor_stop(project_root, phase, &invocation.launch_id); + Ok(()) + } }; // Never a `stopped` line for a completed run: `finish` decides the final // line from the recorded outcome, so a shutdown that arrived while @@ -1244,18 +1296,63 @@ fn kept_env(names: &[String]) -> Vec<(String, String)> { .collect() } -/// Record a run its supervisor stopped: under the per-phase lock (bounded -/// wait), `state.stopped = true` with [`SUPERVISOR_STOP_REASON`]. Never -/// advances. -fn record_supervisor_stop(project_root: &Path, phase: PhaseId) -> Result<(), CliError> { - let _lock = lock::acquire_blocking(project_root, phase, STOP_RECORD_LOCK_WAIT) - .map_err(|err| CliError::Message(format!("could not record the stop: {err}")))?; - let mut state = workflow::load_state(project_root, phase)?; +/// Record a run its supervisor stopped: `state.stopped = true` with +/// [`SUPERVISOR_STOP_REASON`], under the per-phase lock. Never advances. +/// +/// Only when the state names THIS launch (50-13, T-50-54): checked without +/// the lock first, then again under it. A next supervisor that was never +/// published (safe point 3 stops it while its launcher holds the lock and +/// records the stop itself) and a superseded supervisor (the state names its +/// successor, whose run it must not mark stopped) both write nothing, so +/// neither waits on a lock its stopper holds. +/// +/// Never fails (T-50-68): a state that will not load, a lock wait that +/// times out or a save that fails is logged, naming the state path and why +/// `stopped` was not recorded, and the caller still reaches `finish`, which +/// answers a parked stop caller `stopped`. +fn record_supervisor_stop(project_root: &Path, phase: PhaseId, launch_id: &str) { + let path = workflow::state_path(project_root, phase); + let names_this_launch = |state: &State| { + state + .supervisor + .as_ref() + .is_some_and(|record| record.launch_id == launch_id) + }; + let not_recorded = |why: &str| { + warn!( + "phase {phase}: `stopped` was not recorded in {}: {why}", + path.display() + ); + }; + match workflow::load_state(project_root, phase) { + Err(err) => return not_recorded(&format!("the state does not load ({err})")), + Ok(state) if !names_this_launch(&state) => { + info!( + "phase {phase}: the state does not name launch {launch_id}; its stop is \ + not this supervisor's to record" + ); + return; + } + Ok(_) => {} + } + let _lock = match lock::acquire_blocking(project_root, phase, SUPERVISOR_STOP_WAIT) { + Ok(guard) => guard, + Err(err) => return not_recorded(&format!("the per-phase lock was not acquired ({err})")), + }; + let mut state = match workflow::load_state(project_root, phase) { + Ok(state) if names_this_launch(&state) => state, + Ok(_) => { + info!("phase {phase}: launch {launch_id} was superseded before its stop was recorded"); + return; + } + Err(err) => return not_recorded(&format!("the state does not load ({err})")), + }; state.stopped = true; state.stop_reason = Some(SUPERVISOR_STOP_REASON.to_string()); - workflow::save_state(&state)?; + if let Err(err) = workflow::save_state(&state) { + return not_recorded(&format!("the save failed ({err})")); + } info!("phase {phase}: {SUPERVISOR_STOP_REASON}"); - Ok(()) } /// How long the supervise-only entry waits for its launcher's record. @@ -1425,10 +1522,23 @@ fn spawn_agent_and_record( }; #[cfg(test)] test_hold("before_spawn"); + // Safe point 2 (50-13): a stop taken during the advance ends it before + // the next stage's supervisor is spawned. + if stopped_at_safe_point(state, 2)? { + return Ok(()); + } let published = monitor::spawn_monitor(state, program, args, extra_env, launch, &ctx) .map_err(|err| CliError::Message(format!("could not spawn monitor: {err}")))?; #[cfg(test)] test_hold("before_publish"); + // Safe point 3 (50-13): the next supervisor answered its ping but is not + // published. On a stop, shut it down through its socket and never save + // its record; the state keeps this run's record, marked stopped. + if with_stop_token(StopToken::safe_point) == Some(SafePoint::StopHere) { + shut_down_unpublished(&published.record); + record_advance_stop(state, 3)?; + return Ok(()); + } let pid = published.supervisor_pid; let socket_path = published.record.socket_path.display().to_string(); let launch_id = published.record.launch_id.clone(); @@ -2041,6 +2151,29 @@ pub(crate) fn advance_with( // different file. A duplicate advance of THIS phase queues behind the // lock, then stage binding below handles a launch that has moved on. let mut state = workflow::load_state(project_root, phase)?; + // Safe point 1 (50-13): a stop taken while this supervisor waited for + // the lock ends the advance before it evaluates anything. + if stopped_at_safe_point(&mut state, 1)? { + return Ok(()); + } + // 999.138 residual: a stopped run never advances. `resume` clears + // `stopped` before it launches, so no caller advances a stopped state + // legitimately. + if state.stopped { + events::emit( + project_root, + phase, + "advance_failed", + serde_json::json!({ + "reason": "stopped", + "stop_reason": state.stop_reason, + }), + ); + return Err(CliError::Message(format!( + "phase {phase} is stopped ({}); run `devflow resume --phase {phase}` to continue it", + state.stop_reason.as_deref().unwrap_or("no reason recorded") + ))); + } // CR-03 (49-REVIEW.md): a hand-run `advance` updates the run's notify // command; the detached chain carries no variable, so its `advance` // keeps the persisted one. From d19be85e5c8ed43976cd0d9252960ca8df0a98a6 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 17:33:04 -0400 Subject: [PATCH 062/112] test(50-13): add failing tests for the cancellable gate wait - gates: a cancel ends poll_response_cancellable within 1.5 s (control: the uncancelled wait returns the response or times out); the new fn is an additive RED scaffold that ignores cancel - V-15: SIGTERM to a gate-parked supervisor must release the lock, stop the state and remove the socket within 5 s; control holds until answered - R-10: a SIGKILL of N at after_publish leaves state on the live N+1 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_launch.rs | 184 ++++++++++++++++++++++ crates/devflow-core/src/gates.rs | 78 +++++++++ 2 files changed, 262 insertions(+) diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index 50307a0e..02f5133b 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -8316,4 +8316,188 @@ mod tests { } } } + + /// The marker a gate fixture's first agent prints: a failed Define, + /// which the never-silent path parks at a human gate. + const FAILED_MARKER: &str = r#"DEVFLOW_RESULT: {"status":"failed","reason":"gate fixture"}"#; + + /// Signal stage N's supervisor by the pid in its sidecar, after + /// confirming that pid is still the recorded process. + fn signal_supervisor(run: &HandoffRun, signal: libc::c_int) { + let sidecar = + devflow_core::supervisor::read_sidecar(&run.n.socket_path.with_extension("json")) + .expect("N's sidecar"); + let pid = sidecar.supervisor_pid; + assert!( + devflow_core::agent::is_same_process(pid, sidecar.supervisor_start_time), + "the sidecar's supervisor {pid} is the recorded process" + ); + let signed = libc::pid_t::try_from(pid).expect("pid fits pid_t"); + assert!(signed > 1, "refusing to signal pid {signed}"); + // SAFETY: `signed > 1` names the supervisor just identity-checked. + assert_eq!(unsafe { libc::kill(signed, signal) }, 0, "signal {pid}"); + } + + /// Launch a gate fixture and return once N is parked at its Define gate: + /// the gate file exists and N holds the lock (its owner phase is the + /// control's assertion, not a precondition here). + fn launch_parked_at_gate(phase: PhaseId) -> HandoffRun { + let run = HandoffRun::launch(phase); + run.go(); + let gate = Gates::gate_path(&run.root, phase, Stage::Define); + assert!( + poll_until(HANDOFF_STEP_WAIT, || gate.exists() + && events_named(&run.root, "gate_fired").len() == 1), + "N parks at the Define gate: {}", + run.events() + ); + assert!( + matches!( + lock::holder_status(&run.root, phase), + lock::HolderStatus::Live { .. } + ), + "N holds the per-phase lock across the gate wait" + ); + run + } + + /// V-15 (R-2, T-50-33): SIGTERM to a supervisor parked at a gate aborts + /// the wait (safe point 4): within 5 s the lock has no holder, the state + /// is stopped with the safe-point reason, the socket is gone and no next + /// stage was launched. + #[test] + fn a_signal_during_a_gate_wait_aborts_and_releases_the_lock() { + const NAME: &str = + "pipeline_launch::tests::a_signal_during_a_gate_wait_aborts_and_releases_the_lock"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, None, FAILED_MARKER); + return; + } + let _guard = env_lock(); + let run = launch_parked_at_gate(PhaseId::new(1310)); + signal_supervisor(&run, libc::SIGTERM); + let ended = poll_until(Duration::from_secs(5), || { + matches!( + lock::holder_status(&run.root, run.phase), + lock::HolderStatus::NoHolder + ) && devflow_core::supervisor::probe(&run.n.socket_path) + .is_ok_and(|liveness| liveness == devflow_core::supervisor::Liveness::Gone) + }); + assert!( + ended, + "within 5 s of SIGTERM the lock is released and the socket gone: {}", + run.events() + ); + let state = run.state(); + assert!(state.stopped, "state.stopped after a signal at the gate"); + assert_eq!(state.stop_reason.as_deref(), Some(ADVANCE_STOP_REASON)); + assert_eq!( + stage_launched_count(&run.root, run.phase), + 1, + "{}", + run.events() + ); + let stopped = events_named(&run.root, "advance_stopped"); + assert_eq!(stopped.len(), 1, "{}", run.events()); + assert_eq!(stopped[0]["safe_point"], 4, "{}", stopped[0]); + } + + /// The V-15 control: with no signal the gate wait holds (lock held, N in + /// `gate_wait`) after 3 s, and answering the gate lets it proceed: an + /// abort answer releases the lock and ends the supervisor. + #[test] + fn without_a_signal_the_gate_wait_holds_until_answered() { + const NAME: &str = + "pipeline_launch::tests::without_a_signal_the_gate_wait_holds_until_answered"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, None, FAILED_MARKER); + return; + } + let _guard = env_lock(); + let run = launch_parked_at_gate(PhaseId::new(1311)); + std::thread::sleep(Duration::from_secs(3)); + assert!( + matches!( + lock::holder_status(&run.root, run.phase), + lock::HolderStatus::Live { .. } + ), + "control: still parked with the lock held after 3 s" + ); + assert!( + matches!( + devflow_core::supervisor::probe(&run.n.socket_path), + Ok(devflow_core::supervisor::Liveness::Alive(ref reply)) if reply.owner == "gate_wait" + ), + "control: N still answers gate_wait" + ); + assert!(!run.state().stopped, "control: not stopped while waiting"); + Gates::respond( + &run.root, + run.phase, + Stage::Define, + &devflow_core::gates::GateResponse { + approved: false, + note: Some("abort: control answer".to_string()), + responded_by: Some("test".to_string()), + }, + ) + .unwrap(); + let ended = poll_until(HANDOFF_STEP_WAIT, || { + matches!( + lock::holder_status(&run.root, run.phase), + lock::HolderStatus::NoHolder + ) && devflow_core::supervisor::probe(&run.n.socket_path) + .is_ok_and(|liveness| liveness == devflow_core::supervisor::Liveness::Gone) + }); + assert!( + ended, + "control: the answered gate proceeds: {}", + run.events() + ); + assert_eq!(events_named(&run.root, "workflow_aborted").len(), 1); + assert!(events_named(&run.root, "advance_stopped").is_empty()); + } + + /// R-10: a SIGKILL of N between N+1's publish and N's unlink (N parked + /// at `after_publish`) leaves the state naming the live N+1, and N's + /// leftover socket probes Stale with a sidecar whose launch id differs + /// from the state's. + #[test] + fn a_kill_between_the_next_publish_and_the_old_unlink_leaves_state_on_the_live_next_supervisor() + { + const NAME: &str = "pipeline_launch::tests::a_kill_between_the_next_publish_and_the_old_unlink_leaves_state_on_the_live_next_supervisor"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, Some("after_publish"), SUCCESS_MARKER); + return; + } + let _guard = env_lock(); + let run = HandoffRun::launch(PhaseId::new(1312)); + run.go(); + run.wait_held(); + signal_supervisor(&run, libc::SIGKILL); + assert!( + poll_until(HANDOFF_STEP_WAIT, || matches!( + devflow_core::supervisor::probe(&run.n.socket_path), + Ok(devflow_core::supervisor::Liveness::Stale) + )), + "N's leftover socket probes Stale after the kill" + ); + let state = run.state(); + let next = state.supervisor.expect("the state names a supervisor"); + assert_ne!(next.launch_id, run.n.launch_id, "the state names N+1"); + match devflow_core::supervisor::probe(&next.socket_path) { + Ok(devflow_core::supervisor::Liveness::Alive(reply)) => { + assert_eq!(reply.launch_id, next.launch_id); + } + other => panic!("N+1 must be Alive after N's kill, got {other:?}"), + } + let leftover = + devflow_core::supervisor::read_sidecar(&run.n.socket_path.with_extension("json")) + .expect("N's sidecar is left behind"); + assert_ne!( + leftover.launch_id, next.launch_id, + "N's leftover sidecar names a launch other than the state's" + ); + assert_eq!(leftover.launch_id, run.n.launch_id); + } } diff --git a/crates/devflow-core/src/gates.rs b/crates/devflow-core/src/gates.rs index 49589650..111af9d4 100644 --- a/crates/devflow-core/src/gates.rs +++ b/crates/devflow-core/src/gates.rs @@ -309,6 +309,19 @@ impl Gates { } } + /// [`poll_response`](Self::poll_response) that a caller can cancel + /// (50-13, R-2): RED scaffold. + pub fn poll_response_cancellable( + project_root: &Path, + phase: PhaseId, + stage: Stage, + timeout_secs: u64, + cancel: impl Fn() -> bool, + ) -> Option { + let _ = &cancel; + Self::poll_response(project_root, phase, stage, timeout_secs) + } + /// Write an ack file signalling the response was read. pub fn ack(project_root: &Path, phase: PhaseId, stage: Stage) -> Result { let path = Self::ack_path(project_root, phase, stage); @@ -1066,4 +1079,69 @@ mod tests { fire_gate_notify_with_fallback(PhaseId::new(11), Stage::Ship, "ctx", false, Some(&cmd)); assert!(sentinel.exists()); } + + /// R-2 (50-13): a cancel set during the wait ends it within a second, + /// even once the backoff has grown past a second. The cancel is set + /// 3.2 s in, where `poll_response`'s own schedule (sleeps of 1, 2, then + /// 4 s) would sleep until 7 s; the 60 s timeout rules out a timeout. + #[test] + fn poll_response_cancellable_returns_abort_within_a_second_of_cancel() { + let dir = tempfile::tempdir().unwrap(); + let cancelled = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let setter = std::sync::Arc::clone(&cancelled); + let set_at = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(3200)); + setter.store(true, std::sync::atomic::Ordering::SeqCst); + std::time::Instant::now() + }); + let got = Gates::poll_response_cancellable( + dir.path(), + PhaseId::new(13), + Stage::Define, + 60, + || cancelled.load(std::sync::atomic::Ordering::SeqCst), + ); + let returned = std::time::Instant::now(); + let set_at = set_at.join().unwrap(); + assert!(got.is_none(), "a cancelled wait has no response: {got:?}"); + let after_cancel = returned.saturating_duration_since(set_at); + assert!( + after_cancel < std::time::Duration::from_millis(1500), + "returned {after_cancel:?} after the cancel, not within 1.5 s" + ); + } + + /// The control: never cancelled, the wait returns the response that + /// appears, and with no response it times out like `poll_response`. + #[test] + fn poll_response_cancellable_returns_the_response_when_not_cancelled() { + let dir = tempfile::tempdir().unwrap(); + let (phase, stage) = (PhaseId::new(13), Stage::Define); + Gates::write_gate(dir.path(), phase, stage, "ctx").unwrap(); + let response = GateResponse { + approved: true, + note: Some("go".into()), + responded_by: Some("human".into()), + }; + let root = dir.path().to_path_buf(); + let answer = response.clone(); + let writer = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(1300)); + Gates::respond(&root, phase, stage, &answer).unwrap(); + }); + let got = Gates::poll_response_cancellable(dir.path(), phase, stage, 30, || false); + writer.join().unwrap(); + assert_eq!(got, Some(response)); + + let empty = tempfile::tempdir().unwrap(); + let started = std::time::Instant::now(); + assert!( + Gates::poll_response_cancellable(empty.path(), phase, Stage::Ship, 1, || false) + .is_none() + ); + assert!( + started.elapsed() >= std::time::Duration::from_secs(1), + "an uncancelled wait runs to its timeout" + ); + } } From 978a63ebde2a6d3c2664ed96eb20d8e9ece3b3e8 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sat, 17 Oct 2026 17:35:12 -0400 Subject: [PATCH 063/112] feat(50-13): make the advance gate wait cancellable by a stop - Gates::poll_response_cancellable: poll_response's loop with sleeps of at most 1 s and a cancel check each iteration (additive; poll_response unchanged) - an advance's gate wait enters GateWait, polls with the stop token as cancel and, when stopped, clears the gate, records stopped (safe point 4) and unwinds through an error so the lock is released; not Abort, which would delete the state - the --until stop clears state.supervisor beside monitor_pid Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_gate.rs | 70 +++++++++++++++++++++-- crates/devflow-cli/src/pipeline_launch.rs | 41 ++++++++++++- crates/devflow-core/src/gates.rs | 35 +++++++++++- 3 files changed, 137 insertions(+), 9 deletions(-) diff --git a/crates/devflow-cli/src/pipeline_gate.rs b/crates/devflow-cli/src/pipeline_gate.rs index 3c38c6d5..736fc019 100644 --- a/crates/devflow-cli/src/pipeline_gate.rs +++ b/crates/devflow-cli/src/pipeline_gate.rs @@ -21,7 +21,7 @@ use crate::CliError; use crate::config_parse::{foreground_gate_timeout_secs, gate_timeout_secs}; -use crate::pipeline_launch::launch_stage; +use crate::pipeline_launch::{ADVANCE_STOP_REASON, GateWaitEnd, launch_stage}; use crate::pipeline_outcomes::{run_checkout_hooks, truncate_reason}; use devflow_core::gates::{self, GateAction, GateError, GateResponse, Gates}; use devflow_core::hooks; @@ -70,8 +70,12 @@ pub(crate) fn transition( state.stop_reason = Some(format!("stopped after {from} completed (--until {from})")); // T-20-03b / doctor gap (D-09): clear monitor_pid/gate_pending so // neither check_dead_agent nor check_dead_monitor misreports this - // intentional stop as a crashed agent or dead monitor. + // intentional stop as a crashed agent or dead monitor. The supervisor + // record goes with monitor_pid (50-13): the supervisor running this + // advance finishes once it returns, so the record would otherwise + // name a socket that is gone. state.monitor_pid = None; + state.supervisor = None; state.gate_pending = false; workflow::save_state(state)?; events::emit( @@ -426,8 +430,13 @@ pub(crate) fn run_gate_with_timeout( Err(err) => return Err(err.into()), } } - match Gates::poll_response(project_root, state.phase, stage, timeout_secs) { - Some(response) => { + match crate::pipeline_launch::wait_for_gate_response( + project_root, + state.phase, + stage, + timeout_secs, + ) { + GateWaitEnd::Answered(response) => { state.gate_pending = false; // WR-10 (49-REVIEW.md): the operator may have edited the run's // notify command on disk while this gate waited; keep that edit @@ -455,7 +464,25 @@ pub(crate) fn run_gate_with_timeout( ); Ok(action) } - None => { + GateWaitEnd::Stopped => { + // Safe point 4 (50-13, R-2): the supervisor running this advance + // was stopped during the wait. The gate has no waiter any more, so + // it is cleared (no stale request or response outlives the run), + // the stop is recorded, and an error unwinds every caller through + // `?`, which releases the per-phase lock. Deliberately not + // `GateAction::Abort`: `abort()` deletes the state file, and a + // stopped run keeps its state for `devflow resume`. + let _ = Gates::cleanup(project_root, state.phase, stage); + state.gate_pending = false; + if let Ok(on_disk) = workflow::load_state(project_root, state.phase) { + state.gate_notify_cmd = on_disk.gate_notify_cmd; + } + crate::pipeline_launch::record_advance_stop(state, 4)?; + Err(CliError::Message(format!( + "the gate wait for stage {stage} ended: {ADVANCE_STOP_REASON}" + ))) + } + GateWaitEnd::TimedOut => { events::emit( project_root, state.phase, @@ -888,6 +915,39 @@ mod tests { assert!(devflow_core::ship_evidence::is_stopped_at(&evidence)); } + /// 50-13: the `--until` stop clears the supervisor record together with + /// `monitor_pid`, because the supervisor running this advance finishes + /// once the advance returns. Both are set first, so a stop that clears + /// neither (or only the pid) fails here. + #[test] + fn until_stop_clears_the_supervisor_record_with_the_monitor_pid() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let phase = PhaseId::new(1313); + let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); + state.stage = Stage::Plan; + state.stop_until = Some(Stage::Plan); + state.monitor_pid = Some(4242); + state.supervisor = Some(devflow_core::state::SupervisorRecord::new( + root.join("until.sock"), + "until00000000000".to_string(), + 4242, + 1, + String::new(), + )); + workflow::save_state(&state).unwrap(); + + transition(root, &mut state, Stage::Code).unwrap(); + + let reloaded = workflow::load_state(root, phase).unwrap(); + assert!(reloaded.stopped, "the --until stop marks the run stopped"); + assert_eq!(reloaded.monitor_pid, None); + assert_eq!( + reloaded.supervisor, None, + "the supervisor record is cleared with monitor_pid" + ); + } + #[test] fn terminal_merge_failure_reopens_actionable_gate_and_never_reports_finished() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index 02f5133b..e332a9aa 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -1050,7 +1050,7 @@ fn stopped_at_safe_point(state: &mut State, safe_point: u8) -> Result Result<(), CliError> { +pub(crate) fn record_advance_stop(state: &mut State, safe_point: u8) -> Result<(), CliError> { state.stopped = true; state.stop_reason = Some(ADVANCE_STOP_REASON.to_string()); workflow::save_state(state)?; @@ -1091,6 +1091,45 @@ fn shut_down_unpublished(record: &SupervisorRecord) { } } +/// How an advance's gate wait ended (safe point 4). +#[derive(Debug)] +pub(crate) enum GateWaitEnd { + /// The gate was answered. + Answered(devflow_core::gates::GateResponse), + /// No answer arrived within the timeout. + TimedOut, + /// The supervisor running this advance was stopped during the wait. + Stopped, +} + +/// An advance's gate wait (50-13, R-2, safe point 4). Inside a supervisor's +/// advance the run is `GateWait` for the wait and `Advancing` again after +/// it, and the poll is cancelled by the supervisor's stop token, so a +/// SIGTERM/SIGINT or socket stop ends the wait within about a second as +/// [`GateWaitEnd::Stopped`]. Anywhere else (the hidden `devflow advance`, a +/// foreground gate) it is `Gates::poll_response`, unchanged. +pub(crate) fn wait_for_gate_response( + project_root: &Path, + phase: PhaseId, + stage: Stage, + timeout_secs: u64, +) -> GateWaitEnd { + let Some(token) = with_stop_token(StopToken::clone) else { + return Gates::poll_response(project_root, phase, stage, timeout_secs) + .map_or(GateWaitEnd::TimedOut, GateWaitEnd::Answered); + }; + token.enter(OwnerPhase::GateWait); + let polled = Gates::poll_response_cancellable(project_root, phase, stage, timeout_secs, || { + token.stop_requested() + }); + token.enter(OwnerPhase::Advancing); + match polled { + Some(response) => GateWaitEnd::Answered(response), + None if token.safe_point() == SafePoint::StopHere => GateWaitEnd::Stopped, + None => GateWaitEnd::TimedOut, + } +} + /// How long a [`test_hold`] waits for its release file. #[cfg(test)] const TEST_HOLD_WAIT: Duration = Duration::from_secs(30); diff --git a/crates/devflow-core/src/gates.rs b/crates/devflow-core/src/gates.rs index 111af9d4..8aff9782 100644 --- a/crates/devflow-core/src/gates.rs +++ b/crates/devflow-core/src/gates.rs @@ -310,7 +310,14 @@ impl Gates { } /// [`poll_response`](Self::poll_response) that a caller can cancel - /// (50-13, R-2): RED scaffold. + /// (50-13, R-2): the same loop, except that every sleep is at most one + /// second and `cancel` is checked on every iteration, so a cancel ends + /// the wait within about a second of being set. + /// + /// Returns the response when it appears, or `None` on a timeout or a + /// cancel. A caller tells the two apart by asking its own `cancel` + /// again: a `None` returned before the timeout means `cancel` answered + /// `true`. A response already on disk wins over a cancel. pub fn poll_response_cancellable( project_root: &Path, phase: PhaseId, @@ -318,8 +325,30 @@ impl Gates { timeout_secs: u64, cancel: impl Fn() -> bool, ) -> Option { - let _ = &cancel; - Self::poll_response(project_root, phase, stage, timeout_secs) + let path = Self::response_path(project_root, phase, stage); + let deadline = Duration::from_secs(timeout_secs); + let mut waited = Duration::ZERO; + let mut backoff = Duration::from_secs(1); + let cap = Duration::from_secs(60); + let slice = Duration::from_secs(1); + debug!( + "polling (cancellable) for gate response at {}", + path.display() + ); + loop { + if let Ok(contents) = std::fs::read_to_string(&path) + && let Ok(response) = serde_json::from_str::(&contents) + { + return Some(response); + } + if waited >= deadline || cancel() { + return None; + } + let sleep = backoff.min(deadline - waited).min(slice); + std::thread::sleep(sleep); + waited += sleep; + backoff = (backoff * 2).min(cap); + } } /// Write an ack file signalling the response was read. From 7641c39952b652f5d2ebc5665cbfa84fab59516e Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 08:57:26 -0400 Subject: [PATCH 064/112] test(50-12): add failing tests for a ping closed under the probe A listener released while a probe's connection waits in its backlog resets that connection (or breaks the pipe before the ping is sent). probe() reports that as NoReply, though a fresh connect would be refused and answer Stale. The first test releases the listener once the probe's connect is queued and expects Stale; the control keeps a live listener that closes every connection unread and expects NoReply after exactly one more connect. Both fail at this commit: NoReply("could not send ping: Broken pipe") for the first, one accepted connect instead of two for the control. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor.rs | 89 +++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs index a51cf169..dbb35a67 100644 --- a/crates/devflow-core/src/supervisor.rs +++ b/crates/devflow-core/src/supervisor.rs @@ -902,6 +902,95 @@ mod tests { ); } + /// Wait up to `within` for `fd` to be readable (a queued connection on a + /// listener, data on a stream); whether it became readable. + fn readable_within(fd: &impl std::os::fd::AsRawFd, within: std::time::Duration) -> bool { + let mut entry = libc::pollfd { + fd: fd.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + let millis = libc::c_int::try_from(within.as_millis()).unwrap_or(libc::c_int::MAX); + loop { + // SAFETY: one initialized pollfd naming a descriptor the caller + // holds open for the whole call. + let ready = unsafe { libc::poll(&raw mut entry, 1, millis) }; + if ready >= 0 || io::Error::last_os_error().kind() != io::ErrorKind::Interrupted { + return ready == 1; + } + } + } + + /// A listener released while the probe's connection waits in its backlog + /// resets that connection. That is what a probe meets when a supervisor's + /// last thread exits after its leader already reads as a zombie, or when + /// a child forked by another thread drops the last copy of the listener. + /// The reset is the moment nothing listens any more, so the probe + /// connects once more, is refused, and answers Stale, not NoReply. + #[test] + fn probe_answers_stale_for_a_listener_released_under_its_ping() { + let cache = short_cache(); + let (socket, _, listener) = published(cache.path()); + let releaser = thread::spawn(move || { + // Release the listener with the probe's connect queued in it and + // never accepted. + let queued = readable_within(&listener, std::time::Duration::from_secs(30)); + drop(listener); + queued + }); + let outcome = probe(socket.as_path()); + assert!( + releaser.join().expect("releaser thread"), + "the probe's connect never reached the listener" + ); + assert!( + matches!(outcome, Ok(Liveness::Stale)), + "a listener released under the ping must be Stale, got {outcome:?}" + ); + } + + /// Control for the re-probe: a listener that stays up but closes every + /// connection with the ping unread is never Stale. The probe connects + /// exactly twice (one more connect after the first close) and answers + /// NoReply. + #[test] + fn probe_never_reads_a_live_listener_that_closes_connections_as_stale() { + /// Stops an unbounded re-probe from hanging this test. + const ACCEPT_AT_MOST: usize = 3; + let cache = short_cache(); + let (socket, _, listener) = published(cache.path()); + let (stop, stopped) = mpsc::channel::<()>(); + let server = thread::spawn(move || { + let mut accepted = 0; + while accepted < ACCEPT_AT_MOST && stopped.try_recv().is_err() { + if !readable_within(&listener, std::time::Duration::from_millis(50)) { + continue; + } + let (stream, _) = listener.accept().expect("accept a probe"); + accepted += 1; + // Close with the ping unread, which resets the client. + assert!( + readable_within(&stream, std::time::Duration::from_secs(30)), + "the probe never sent its ping" + ); + drop(stream); + } + (listener, accepted) + }); + let outcome = probe(socket.as_path()); + stop.send(()).expect("stop the server"); + let (listener, accepted) = server.join().expect("server thread"); + drop(listener); + assert!( + matches!(outcome, Err(ProbeError::NoReply(_))), + "a live listener that closes every connection is NoReply, never Stale, got {outcome:?}" + ); + assert_eq!( + accepted, 2, + "the probe connects once more after a closed ping, and only once" + ); + } + #[test] fn probe_reports_gone_only_when_no_socket_file_exists() { let cache = short_cache(); From 37fa4e2150937d92647a7e05efa47c9091dee829 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 08:58:10 -0400 Subject: [PATCH 065/112] fix(50-12): re-probe a socket whose ping connection closes under it probe() answered NoReply when a supervisor's listener was being released at the moment it probed: the connect lands in the dying listener's backlog, and the ping then reads a reset or hits a broken pipe. Two ways this happens: - A SIGKILLed supervisor is multi-threaded. Its leader can read as a zombie (agent_running false) while another thread still holds the fd table, so the listener outlives the "gone" signal. The 50-12 e2e status_reports_each_liveness_state_with_its_repair_verb failed this way: every failure caught the leader in Z with sibling threads in R. - Under `cargo test`, a child forked by another test thread keeps a copy of a listener the test just dropped until it execs. The 50-09 backstop fixture behind pidfd_group_signal_is_used_when_the_kernel_supports_it failed this way. ping() now marks a reset or broken pipe as `closed`, and probe() then runs once more: a released listener refuses the fresh connect, which is Stale; a listener still up takes it, and its answer stands. A second close is NoReply, so Stale still comes only from a refused connect and the backstop arms on the same evidence as before. Under load the e2e failed 5/60 and the backstop test 6/60 before this change, and 0/60 and 0/120 after, with the re-probe observed taking the failing path 3 and 7 times. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/supervisor.rs | 37 ++++++--- .../devflow-core/src/supervisor/protocol.rs | 83 ++++++++++++++++--- 2 files changed, 97 insertions(+), 23 deletions(-) diff --git a/crates/devflow-core/src/supervisor.rs b/crates/devflow-core/src/supervisor.rs index dbb35a67..f4b372a4 100644 --- a/crates/devflow-core/src/supervisor.rs +++ b/crates/devflow-core/src/supervisor.rs @@ -299,20 +299,33 @@ pub enum ProbeError { /// the stat and the connect is reported, never guessed to be `Gone`. /// 4. A connect is not proof of life (a listen backlog accepts it with /// nobody serving): only a valid ping reply makes it `Alive`. +/// 5. A ping whose connection is closed under it (a reset or a broken pipe) +/// runs steps 1 to 4 once more. A listener released while the connection +/// waits in its backlog closes it like that (an owner whose last thread +/// is still exiting, or a forked child dropping the last copy of the +/// listener), and the fresh connect is then refused: `Stale`. A listener +/// that is still up takes the fresh connect, and its answer stands; a +/// second close is [`ProbeError::NoReply`]. `Stale` still comes only from +/// a refused connect. pub fn probe(path: &Path) -> Result { - if !own_socket_at(path)? { - return Ok(Liveness::Gone); - } - let stream = match UnixStream::connect(path) { - Err(error) if error.kind() == io::ErrorKind::ConnectionRefused => { - return Ok(Liveness::Stale); + let mut connected_again = false; + loop { + if !own_socket_at(path)? { + return Ok(Liveness::Gone); } - Err(error) => return Err(ProbeError::Connect(error)), - Ok(stream) => stream, - }; - protocol::ping(stream) - .map(Liveness::Alive) - .map_err(ProbeError::NoReply) + let stream = match UnixStream::connect(path) { + Err(error) if error.kind() == io::ErrorKind::ConnectionRefused => { + return Ok(Liveness::Stale); + } + Err(error) => return Err(ProbeError::Connect(error)), + Ok(stream) => stream, + }; + match protocol::ping(stream) { + Ok(reply) => return Ok(Liveness::Alive(reply)), + Err(error) if error.closed && !connected_again => connected_again = true, + Err(error) => return Err(ProbeError::NoReply(error.message)), + } + } } /// [`probe`]'s pre-connect checks, shared with [`request_shutdown`]: diff --git a/crates/devflow-core/src/supervisor/protocol.rs b/crates/devflow-core/src/supervisor/protocol.rs index 2af20252..5b454acd 100644 --- a/crates/devflow-core/src/supervisor/protocol.rs +++ b/crates/devflow-core/src/supervisor/protocol.rs @@ -237,26 +237,59 @@ pub(crate) fn peer_is_same_user(_stream: &UnixStream) -> bool { false } +/// Why [`ping`] got no valid reply. +#[derive(Debug)] +pub(crate) struct PingError { + /// What went wrong, as a message. + pub(crate) message: String, + /// The connection was closed under the ping: the send hit a broken pipe + /// or the read a reset. A listener released while the connection waits + /// in its backlog does this, so only a fresh connect can tell whether + /// anything still listens. + pub(crate) closed: bool, +} + +impl PingError { + fn other(message: String) -> Self { + Self { + message, + closed: false, + } + } + + fn io(context: &str, error: &io::Error) -> Self { + Self { + message: format!("{context}: {error}"), + closed: matches!( + error.kind(), + io::ErrorKind::BrokenPipe | io::ErrorKind::ConnectionReset + ), + } + } +} + /// Client side: send one ping on a connected stream and parse the reply. /// Every failure (timeout, closed connection, oversized or malformed line, -/// `"ok":false`) is returned as a message; a connect alone is never proof of -/// life. -pub(crate) fn ping(stream: UnixStream) -> Result { - set_timeouts(&stream).map_err(|error| format!("could not set socket timeouts: {error}"))?; +/// `"ok":false`) is returned as a [`PingError`]; a connect alone is never +/// proof of life. +pub(crate) fn ping(stream: UnixStream) -> Result { + set_timeouts(&stream) + .map_err(|error| PingError::io("could not set socket timeouts", &error))?; let request = serde_json::to_string(&Request::Ping) - .map_err(|error| format!("could not encode ping: {error}"))?; + .map_err(|error| PingError::other(format!("could not encode ping: {error}")))?; (&stream) .write_all(format!("{request}\n").as_bytes()) - .map_err(|error| format!("could not send ping: {error}"))?; + .map_err(|error| PingError::io("could not send ping", &error))?; let line = read_line(&mut BufReader::new(&stream)) - .map_err(|error| format!("no ping reply: {error}"))?; + .map_err(|error| PingError::io("no ping reply", &error))?; let value: serde_json::Value = serde_json::from_str(&line) - .map_err(|error| format!("ping reply is not JSON ({error}): {line}"))?; + .map_err(|error| PingError::other(format!("ping reply is not JSON ({error}): {line}")))?; if value.get("ok") != Some(&serde_json::Value::Bool(true)) { - return Err(format!("ping reply is not ok: {line}")); + return Err(PingError::other(format!("ping reply is not ok: {line}"))); } - serde_json::from_value(value) - .map_err(|error| format!("ping reply is missing fields ({error}): {line}")) + serde_json::from_value(value).map_err(|error| { + PingError::other(format!("ping reply is missing fields ({error}): {line}")) + }) } #[cfg(test)] @@ -325,4 +358,32 @@ mod tests { "x".repeat(MAX_LINE - 1) ); } + + /// `ping` marks a send into a closed peer (a broken pipe) as `closed`, + /// which is what lets `probe` connect once more. The control is a peer + /// that reads the ping and closes without a reply: a plain missing reply, + /// not `closed`. + #[test] + fn ping_marks_a_broken_pipe_closed_and_a_missing_reply_not() { + let (client, peer) = UnixStream::pair().expect("a socket pair"); + drop(peer); + let broken = ping(client).expect_err("a closed peer gives no reply"); + assert!( + broken.closed, + "a broken pipe is a closed connection: {broken:?}" + ); + + let (client, peer) = UnixStream::pair().expect("a socket pair"); + let reader = std::thread::spawn(move || { + let request = read_request(&peer).expect("the peer reads the ping"); + drop(peer); + request + }); + let unanswered = ping(client).expect_err("a peer that never replies gives no reply"); + assert_eq!(reader.join().expect("reader thread"), Request::Ping); + assert!( + !unanswered.closed, + "a close after the ping was read is a missing reply: {unanswered:?}" + ); + } } From f6bf3d321d77b1dcfcffd7cfcf24f7873bddc25f Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 09:20:53 -0400 Subject: [PATCH 066/112] test(50-14): add failing tests for stop through the supervisor socket - V-17: stop ends a lock-free live run through its socket (e2e), with its trigger-released control that advances without a stop - D-09 repair verb: stop ends a live run whose state will not load, through its one sidecar-found socket, without writing the state - C13: a handed_off reply is retried against the reloaded record, and stop gives up after 3 retries recording nothing - the 6d46008 race: a supervisor that dies under its shutdown request is re-probed and torn down; a live listener closing it is an error Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 424 +++++++++++++++++++++++++++ crates/devflow-cli/tests/stop_e2e.rs | 348 +++++++++++++++++++++- 2 files changed, 770 insertions(+), 2 deletions(-) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 35302af6..38b3b2e9 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -9635,4 +9635,428 @@ mod tests { ); } } + + // ----------------------------------------------------------------------- + // 50-14: `stop` through the run's supervisor socket (crash row C13, and + // a supervisor that dies under its shutdown request). + // ----------------------------------------------------------------------- + + /// A 0700 socket directory under a short tempdir in `/tmp`, so every + /// socket path below it fits `sun_path`. The tempdir removes it on drop. + fn stop_socket_dir() -> (tempfile::TempDir, PathBuf) { + use std::os::unix::fs::PermissionsExt as _; + let base = tempfile::Builder::new() + .prefix("dfs") + .tempdir_in("/tmp") + .expect("create a short socket base under /tmp"); + let dir = base.path().join("sock"); + std::fs::create_dir(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + (base, dir) + } + + /// Bind a listener at `path` with mode 0600, as a published supervisor + /// socket is. + fn bind_supervisor_socket(path: &Path) -> std::os::unix::net::UnixListener { + use std::os::unix::fs::PermissionsExt as _; + let listener = std::os::unix::net::UnixListener::bind(path).expect("bind the socket"); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).unwrap(); + listener + } + + /// Wait up to `within` for `fd` to be readable (a queued connection on a + /// listener, data on a stream); whether it became readable. + fn readable_within(fd: &impl std::os::fd::AsRawFd, within: std::time::Duration) -> bool { + let mut entry = libc::pollfd { + fd: fd.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + let millis = libc::c_int::try_from(within.as_millis()).unwrap_or(libc::c_int::MAX); + loop { + // SAFETY: one initialized pollfd naming a descriptor the caller + // holds open for the whole call. + let ready = unsafe { libc::poll(&raw mut entry, 1, millis) }; + if ready >= 0 + || std::io::Error::last_os_error().kind() != std::io::ErrorKind::Interrupted + { + return ready == 1; + } + } + } + + /// A well-formed ping reply line for a scripted supervisor. + fn scripted_ping_reply(launch_id: &str, phase: PhaseId, owner: &str) -> String { + serde_json::json!({ + "ok": true, + "launch_id": launch_id, + "phase": phase.to_string(), + "stage": "code", + "launch": "stream", + "owner": owner, + "agent_pid": 1, + "agent_pgid": 1, + "agent_start_time": 0, + "boot_id": null, + "supervisor_pid": 1, + }) + .to_string() + } + + /// Accept one connection on `listener` within `within`, or `None`. + fn accept_within( + listener: &std::os::unix::net::UnixListener, + within: std::time::Duration, + ) -> Option { + if !readable_within(listener, within) { + return None; + } + Some(listener.accept().expect("accept a queued connection").0) + } + + /// Read one request line from `stream`, or `None` when none arrives + /// within the read timeout. + fn read_request_line(stream: &std::os::unix::net::UnixStream) -> Option { + use std::io::BufRead as _; + stream + .set_read_timeout(Some(std::time::Duration::from_secs(5))) + .unwrap(); + let mut line = String::new(); + std::io::BufReader::new(stream).read_line(&mut line).ok()?; + (!line.is_empty()).then(|| line.trim_end().to_string()) + } + + /// The lines a scripted supervisor sends for one `shutdown` request, and + /// whether it then unlinks its socket and stops serving. + struct ShutdownScript { + lines: Vec, + unlink_after: bool, + } + + /// A scripted supervisor: serves `socket` on a thread, logs each + /// connection's one request line, answers `ping` with + /// [`scripted_ping_reply`] and the n-th `shutdown` (1-based) with + /// `on_shutdown(n)`. Dropping it stops the thread and removes the socket. + struct ScriptedSupervisor { + socket: PathBuf, + log: std::sync::Arc>>, + done: std::sync::Arc, + thread: Option>, + } + + impl ScriptedSupervisor { + fn serve( + socket: &Path, + launch_id: &str, + phase: PhaseId, + owner: &str, + mut on_shutdown: impl FnMut(usize) -> ShutdownScript + Send + 'static, + ) -> Self { + use std::io::Write as _; + use std::sync::atomic::Ordering; + let listener = bind_supervisor_socket(socket); + let log = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let done = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let ping = scripted_ping_reply(launch_id, phase, owner); + let (thread_log, thread_done, path) = (log.clone(), done.clone(), socket.to_path_buf()); + let thread = std::thread::spawn(move || { + let mut shutdowns = 0; + while !thread_done.load(Ordering::SeqCst) { + let Some(mut stream) = + accept_within(&listener, std::time::Duration::from_millis(50)) + else { + continue; + }; + let Some(request) = read_request_line(&stream) else { + continue; + }; + thread_log.lock().unwrap().push(request.clone()); + if request.contains("\"ping\"") { + let _ = writeln!(stream, "{ping}"); + continue; + } + shutdowns += 1; + let script = on_shutdown(shutdowns); + for line in &script.lines { + let _ = writeln!(stream, "{line}"); + } + if script.unlink_after { + let _ = std::fs::remove_file(&path); + return; + } + } + }); + Self { + socket: socket.to_path_buf(), + log, + done, + thread: Some(thread), + } + } + + /// How many `shutdown` requests this supervisor has received. + fn shutdowns(&self) -> usize { + self.log + .lock() + .unwrap() + .iter() + .filter(|line| line.contains("\"shutdown\"")) + .count() + } + } + + impl Drop for ScriptedSupervisor { + fn drop(&mut self) { + self.done.store(true, std::sync::atomic::Ordering::SeqCst); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + let _ = std::fs::remove_file(&self.socket); + } + } + + /// Save a state for `phase` whose supervisor record names `socket` and + /// `launch_id`. + fn save_state_naming(root: &Path, phase: PhaseId, socket: &Path, launch_id: &str) { + let mut state = match workflow::load_state(root, phase) { + Ok(state) => state, + Err(_) => State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()), + }; + state.supervisor = Some(SupervisorRecord::new( + socket.to_path_buf(), + launch_id.to_string(), + 1, + 0, + "boot".to_string(), + )); + workflow::save_state(&state).unwrap(); + } + + const HANDED_OFF_LINE: &str = r#"{"ok":false,"error":"handed_off"}"#; + const ACCEPTED_LINE: &str = r#"{"ok":true,"accepted":true,"owner":"agent_running"}"#; + const STOPPED_LINE: &str = + r#"{"ok":true,"stopped":true,"exit_code":143,"advance_suppressed":true}"#; + + /// Crash row C13 (V-57): the state names supervisor A; A answers the + /// shutdown by saving the state naming B (N+1's record is saved before N + /// is marked handed off) and replying `handed_off`. `stop` reloads the + /// state and stops B: A is asked once, B once, and `stopped` is recorded. + #[test] + fn stop_retries_a_handed_off_reply_against_the_reloaded_record() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_path_buf(); + let phase = PhaseId::new(1401); + let (_base, sock) = stop_socket_dir(); + let (a_path, b_path) = (sock.join("a.sock"), sock.join("b.sock")); + save_state_naming(&root, phase, &a_path, "launch-a"); + + let b = ScriptedSupervisor::serve(&b_path, "launch-b", phase, "agent_running", |_| { + ShutdownScript { + lines: vec![ACCEPTED_LINE.to_string(), STOPPED_LINE.to_string()], + unlink_after: false, + } + }); + let (a_root, a_b_path) = (root.clone(), b_path.clone()); + let a = ScriptedSupervisor::serve(&a_path, "launch-a", phase, "handed_off", move |_| { + save_state_naming(&a_root, phase, &a_b_path, "launch-b"); + ShutdownScript { + lines: vec![HANDED_OFF_LINE.to_string()], + unlink_after: false, + } + }); + + let result = stop(&root, phase); + assert!( + result.is_ok(), + "stop must reach B through the reloaded record: {result:?}" + ); + assert_eq!(a.shutdowns(), 1, "A is asked exactly once"); + assert_eq!(b.shutdowns(), 1, "B is asked exactly once"); + let state = workflow::load_state(&root, phase).unwrap(); + assert!(state.stopped, "the stop B performed is recorded"); + } + + /// C13 control (V-57): every reload names A again, so `stop` gives up + /// after the first request and 3 retries, with an error naming + /// `handed_off` and the attempt count, and records nothing. A unlinks its + /// socket after a fifth request, so a loop past the cap ends as Gone and + /// fails the count instead of hanging. + #[test] + fn stop_gives_up_after_three_handed_off_retries_and_records_nothing() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_path_buf(); + let phase = PhaseId::new(1402); + let (_base, sock) = stop_socket_dir(); + let a_path = sock.join("a.sock"); + save_state_naming(&root, phase, &a_path, "launch-a"); + let state_file = workflow::state_path(&root, phase); + let before = std::fs::read(&state_file).unwrap(); + + let a = ScriptedSupervisor::serve(&a_path, "launch-a", phase, "handed_off", |count| { + ShutdownScript { + lines: vec![HANDED_OFF_LINE.to_string()], + unlink_after: count >= 5, + } + }); + + let error = stop(&root, phase) + .expect_err("a supervisor that keeps answering handed_off is not a stop") + .to_string(); + assert!( + error.contains("handed_off") && error.contains('4'), + "the error names handed_off and the attempt count: {error}" + ); + assert_eq!(a.shutdowns(), 4, "the first request plus exactly 3 retries"); + assert_eq!( + std::fs::read(&state_file).unwrap(), + before, + "nothing is recorded" + ); + } + + /// The sidecar and state record of a launch recorded in an earlier boot: + /// the backstop proves its group gone without reading a process, so it + /// removes the files and signals nothing. + fn plant_previous_boot_launch(root: &Path, phase: PhaseId, socket: &Path) { + let sidecar = serde_json::json!({ + "launch_id": "launch-r", + "project_root": root, + "phase": phase, + "stage": "code", + "agent": "claude", + "launch": "stream", + "agent_pid": 99_999_999, + "agent_pgid": 99_999_999, + "agent_start_time": 1, + "boot_id": "an-earlier-boot", + "supervisor_pid": 99_999_998, + "supervisor_start_time": 1, + "created_at": "2026-10-17T00:00:00Z", + }); + std::fs::write(socket.with_extension("json"), sidecar.to_string()).unwrap(); + let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); + state.supervisor = Some(SupervisorRecord::new( + socket.to_path_buf(), + "launch-r".to_string(), + 99_999_999, + 1, + "an-earlier-boot".to_string(), + )); + workflow::save_state(&state).unwrap(); + } + + /// The race 6d46008 left to `stop`: the supervisor answers the probe's + /// ping, then its listener is released with the shutdown's connect still + /// queued, so `request_shutdown` reads a reset or a broken pipe. That is + /// a supervisor that died, not a failed stop: `stop` probes once more, + /// reads STALE, runs the backstop (which removes the files and signals + /// nothing for this earlier-boot launch) and records `stopped`. + #[test] + fn stop_acts_on_a_supervisor_that_dies_under_its_shutdown_request() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_path_buf(); + let phase = PhaseId::new(1403); + let (_base, sock) = stop_socket_dir(); + let socket = sock.join("r.sock"); + plant_previous_boot_launch(&root, phase, &socket); + let listener = bind_supervisor_socket(&socket); + let ping = scripted_ping_reply("launch-r", phase, "agent_running"); + let server = std::thread::spawn(move || { + use std::io::Write as _; + let wait = std::time::Duration::from_secs(30); + let Some(mut first) = accept_within(&listener, wait) else { + return "no probe connected"; + }; + if read_request_line(&first).is_none_or(|line| !line.contains("\"ping\"")) { + return "the first request was not a ping"; + } + let _ = writeln!(first, "{ping}"); + drop(first); + // Release the listener with the shutdown's connect queued in it + // and never accepted. + if !readable_within(&listener, wait) { + return "no shutdown connect was queued"; + } + drop(listener); + "released" + }); + + let result = stop(&root, phase); + assert_eq!(server.join().expect("server thread"), "released"); + assert!( + result.is_ok(), + "a supervisor that died under the shutdown is torn down, not an error: {result:?}" + ); + assert!( + !socket.exists() && !socket.with_extension("json").exists(), + "the backstop removed the stale socket and its sidecar" + ); + assert!( + workflow::load_state(&root, phase).unwrap().stopped, + "the confirmed-not-live stop is recorded" + ); + } + + /// Control for the race: a listener that stays up but closes the + /// shutdown (and every later connection) with its request unread is not + /// a dead supervisor. `stop` fails, removes nothing and records nothing. + #[test] + fn stop_reports_a_shutdown_closed_by_a_live_listener() { + /// Stops an unbounded re-probe from hanging this test. + const ACCEPT_AT_MOST: usize = 6; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_path_buf(); + let phase = PhaseId::new(1404); + let (_base, sock) = stop_socket_dir(); + let socket = sock.join("l.sock"); + plant_previous_boot_launch(&root, phase, &socket); + let state_file = workflow::state_path(&root, phase); + let before = std::fs::read(&state_file).unwrap(); + let listener = bind_supervisor_socket(&socket); + let ping = scripted_ping_reply("launch-r", phase, "agent_running"); + let (stop_server, stopped) = std::sync::mpsc::channel::<()>(); + let server = std::thread::spawn(move || { + use std::io::Write as _; + let mut accepted = 0; + while accepted < ACCEPT_AT_MOST && stopped.try_recv().is_err() { + let Some(mut stream) = + accept_within(&listener, std::time::Duration::from_millis(50)) + else { + continue; + }; + accepted += 1; + if accepted == 1 { + let _ = read_request_line(&stream); + let _ = writeln!(stream, "{ping}"); + continue; + } + // Close with the request unread, which resets the client. + let _ = readable_within(&stream, std::time::Duration::from_secs(5)); + drop(stream); + } + (listener, accepted) + }); + + let result = stop(&root, phase); + stop_server.send(()).expect("stop the server"); + let (listener, accepted) = server.join().expect("server thread"); + drop(listener); + assert!( + accepted >= 2, + "stop never sent its shutdown ({accepted} connects)" + ); + assert!( + result.is_err(), + "a live listener that closes the shutdown is not a stop: {result:?}" + ); + assert!( + socket.exists() && socket.with_extension("json").exists(), + "nothing is removed" + ); + assert_eq!( + std::fs::read(&state_file).unwrap(), + before, + "nothing is recorded" + ); + } } diff --git a/crates/devflow-cli/tests/stop_e2e.rs b/crates/devflow-cli/tests/stop_e2e.rs index 91874397..65f52abb 100644 --- a/crates/devflow-cli/tests/stop_e2e.rs +++ b/crates/devflow-cli/tests/stop_e2e.rs @@ -16,10 +16,11 @@ use devflow_core::gates::{GateResponse, Gates}; use devflow_core::mode::Mode; use devflow_core::phase_id::PhaseId; use devflow_core::stage::Stage; -use devflow_core::state::{AgentKind, State}; +use devflow_core::state::{AgentKind, State, SupervisorRecord}; +use devflow_core::supervisor::{self, Liveness}; #[cfg(unix)] use std::os::unix::ffi::OsStrExt; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Child, Command}; use std::time::{Duration, Instant}; @@ -1167,3 +1168,346 @@ fn stop_positional_root_survives_a_non_utf8_path() { "a non-UTF-8 root must survive clap and reach project_root:\n{stderr}" ); } + +// --------------------------------------------------------------------------- +// 50-14: `stop` addresses the run's supervisor socket (D-04, D-06). +// +// These tests launch real supervised runs with `devflow start --mode +// supervise` under a `HermeticHome`, so every socket and sidecar lives under +// the test's tempdir cache. A pid a test signals comes from a sidecar and is +// confirmed with `agent::is_same_process` against its recorded start time. +// --------------------------------------------------------------------------- + +/// How long a test waits for a supervised run to reach a state. +const RUN_WAIT: Duration = Duration::from_secs(60); + +/// A repository on `develop` with a roadmap entry for `phase`, the shape +/// `supervisor_e2e.rs` launches a real supervised run from. +fn init_supervised_repo(root: &Path, phase: PhaseId) { + git(root, &["init", "-q"]); + git(root, &["config", "user.email", "devflow@example.com"]); + git(root, &["config", "user.name", "DevFlow Tests"]); + git(root, &["config", "commit.gpgsign", "false"]); + git(root, &["config", "core.hooksPath", "/dev/null"]); + git(root, &["checkout", "-q", "-b", "develop"]); + std::fs::create_dir_all(root.join(".planning")).unwrap(); + std::fs::write(root.join("README.md"), "base\n").unwrap(); + std::fs::write( + root.join(".planning/ROADMAP.md"), + format!("# Roadmap\n\n### Phase {phase}: stop fixture\n"), + ) + .unwrap(); + git(root, &["add", "README.md", ".planning/ROADMAP.md"]); + git(root, &["commit", "-q", "-m", "base"]); +} + +/// The fake `claude`'s health check and delivery canary, as in +/// `supervisor_e2e.rs`; `body` runs for the real stage turn. +fn fake_claude(body: &str) -> String { + format!( + r#"if [ "$1" = "--version" ]; then echo "2.1.220 (Claude Code)"; exit 0; fi +read -r turn +case "$turn" in + *DEVFLOW_DELIVERY_CANARY_*) + token=$(printf '%s' "$turn" | grep -o 'DEVFLOW_DELIVERY_CANARY_[0-9a-f]*' | head -1) + printf '{{"type":"result","subtype":"success","is_error":false,"session_id":"s-fake","result":"%s"}}\n' "$token" + exit 0 + ;; +esac +{body}"# + ) +} + +/// A fake agent that keeps running until `trigger` exists, then commits one +/// empty commit, prints the success marker and exits 0. +fn triggered_agent(trigger: &Path) -> String { + fake_claude(&format!( + r#"while [ ! -e '{trigger}' ]; do sleep 0.05; done +git commit -q --allow-empty -m "fake stage work" +printf 'work done\nDEVFLOW_RESULT: {{"status":"success","commits":1}}\n' +exit 0"#, + trigger = trigger.display() + )) +} + +/// The `stage_launched` events recorded for `phase`. +fn stage_launched_count(root: &Path, phase: PhaseId) -> usize { + std::fs::read_to_string(devflow_core::events::events_path(root)) + .unwrap_or_default() + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|event| { + phase.matches_json(event.get("phase")) + && event["event"].as_str() == Some("stage_launched") + }) + .count() +} + +/// The pids whose process group (`/proc//stat` field 5) is `pgid`. +fn group_members(pgid: u32) -> Vec { + let Ok(entries) = std::fs::read_dir("/proc") else { + return Vec::new(); + }; + entries + .filter_map(|entry| entry.ok()?.file_name().to_str()?.parse::().ok()) + .filter(|pid| { + std::fs::read_to_string(format!("/proc/{pid}/stat")) + .ok() + .and_then(|stat| { + let rest = &stat[stat.rfind(')')? + 1..]; + rest.split_whitespace().nth(2)?.parse::().ok() + }) + == Some(pgid) + }) + .collect() +} + +/// Poll `done` every 25 ms until it holds or `within` elapses. +fn holds_within(within: Duration, mut done: impl FnMut() -> bool) -> bool { + let deadline = Instant::now() + within; + loop { + if done() { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(25)); + } +} + +/// Ends, on drop, the run a captured supervisor record names, whether or not +/// its state still loads (a test may corrupt it): a socket shutdown, then a +/// SIGKILL to the agent group when its leader is still the recorded process. +struct RecordReaper(SupervisorRecord); + +impl Drop for RecordReaper { + fn drop(&mut self) { + let _ = supervisor::request_shutdown(&self.0.socket_path, Duration::from_secs(15)); + let leader = self.0.agent_pgid; + if devflow_core::agent::agent_running(leader) + && devflow_core::agent::is_same_process(leader, self.0.agent_start_time) + { + common::signal_group(leader, libc::SIGKILL); + } + } +} + +/// One live supervised run of a [`triggered_agent`], started with +/// `devflow start --mode supervise --until plan`. Its reapers are declared +/// first so they drop before the home and repository they need. +struct LiveRun { + _record_reaper: RecordReaper, + _launch_reaper: common::LaunchedRunReaper, + home: common::HermeticHome, + _repo: tempfile::TempDir, + _fake_bin: common::FakeBin, + root: PathBuf, + phase: PhaseId, + trigger: PathBuf, + record: SupervisorRecord, +} + +impl LiveRun { + fn start(phase: PhaseId) -> Self { + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path().canonicalize().unwrap(); + init_supervised_repo(&root, phase); + let trigger = root.join(".git").join("devflow-test-trigger"); + let fake_bin = common::fake_bin_dir(&triggered_agent(&trigger)); + let launch_reaper = common::LaunchedRunReaper::new(&root, phase, vec![]); + let existing = std::env::var_os("PATH").unwrap_or_default(); + let path = format!("{}:{}", fake_bin.path.display(), existing.to_string_lossy()); + let output = common::hermetic_devflow(&home) + .args([ + "start", + "--phase", + &phase.to_string(), + "--agent", + "claude", + "--mode", + "supervise", + "--no-worktree", + "--until", + "plan", + ]) + .arg(&root) + .current_dir(&root) + .env("PATH", path) + .output() + .expect("run devflow start"); + assert!( + output.status.success(), + "start failed\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let record = devflow_core::workflow::load_state(&root, phase) + .expect("start saved state") + .supervisor + .expect("start records state.supervisor"); + Self { + _record_reaper: RecordReaper(record.clone()), + _launch_reaper: launch_reaper, + home, + _repo: repo, + _fake_bin: fake_bin, + root, + phase, + trigger, + record, + } + } + + /// `devflow stop --phase N` plus `extra`, run from the project root. + fn stop(&self, extra: &[&str]) -> std::process::Output { + common::hermetic_devflow(&self.home) + .args(["stop", "--phase", &self.phase.to_string()]) + .args(extra) + .current_dir(&self.root) + .output() + .expect("run devflow stop") + } + + fn state_file(&self) -> PathBuf { + devflow_core::workflow::state_path(&self.root, self.phase) + } + + fn probe(&self) -> Liveness { + supervisor::probe(&self.record.socket_path).expect("probe the recorded socket") + } +} + +/// `stop`'s stdout and stderr, for an assertion message. +fn describe(output: &std::process::Output) -> String { + format!( + "exit {:?}\nstdout: {}\nstderr: {}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +/// V-17 (999.138): a live run whose agent holds no per-phase lock is ended +/// through its supervisor's socket: the agent group is gone, the exit file +/// reads 143, no next stage is launched, the socket is Gone, and the state +/// records the stop. +#[test] +fn stop_ends_a_lock_free_live_run_through_the_socket() { + let run = LiveRun::start(PhaseId::new(140)); + assert!( + matches!( + devflow_core::lock::holder_status(&run.root, run.phase), + devflow_core::lock::HolderStatus::NoHolder + ), + "precondition: the running agent holds no per-phase lock" + ); + assert!( + matches!(run.probe(), Liveness::Alive(_)), + "precondition: the run is ALIVE" + ); + let pgid = run.record.agent_pgid; + assert!( + !group_members(pgid).is_empty(), + "precondition: the agent group is running" + ); + + let output = run.stop(&[]); + let text = describe(&output); + assert!(output.status.success(), "stop must succeed: {text}"); + assert!( + String::from_utf8_lossy(&output.stdout).contains(&format!( + "stopped phase {} through its supervisor", + run.phase + )), + "stop says it stopped the phase through its supervisor: {text}" + ); + assert!( + holds_within(Duration::from_secs(5), || group_members(pgid).is_empty()), + "the agent group {pgid} is gone" + ); + let exit = std::fs::read_to_string(devflow_core::agent_result::exit_code_path( + &run.root, run.phase, + )) + .expect("the supervisor records the agent's exit code"); + assert_eq!(exit.trim(), "143", "the agent ended by the stop"); + std::thread::sleep(Duration::from_secs(1)); + assert_eq!( + stage_launched_count(&run.root, run.phase), + 1, + "no next stage is launched after the stop" + ); + assert_eq!(run.probe(), Liveness::Gone, "the socket is gone"); + let state = devflow_core::workflow::load_state(&run.root, run.phase).unwrap(); + assert!(state.stopped, "state.stopped after the stop"); + assert!( + state + .stop_reason + .as_deref() + .is_some_and(|reason| reason.contains("devflow stop")), + "the stop reason names `devflow stop`: {:?}", + state.stop_reason + ); +} + +/// The V-17 control: the same fixture with no `stop`, whose agent is +/// released by its trigger, completes and advances, so a following +/// `stage_launched` appears. V-17's "no next stage" is therefore the stop's +/// doing, not a fixture that never advances. +#[test] +fn without_stop_a_lock_free_run_completes_and_advances() { + let run = LiveRun::start(PhaseId::new(141)); + assert_eq!(stage_launched_count(&run.root, run.phase), 1); + std::fs::write(&run.trigger, "go\n").unwrap(); + assert!( + holds_within(RUN_WAIT, || stage_launched_count(&run.root, run.phase) >= 2), + "a released agent must complete and launch the next stage:\n{}", + std::fs::read_to_string(devflow_core::events::events_path(&run.root)).unwrap_or_default() + ); +} + +/// D-09's repair verb: a live run whose state file will not load is ended +/// through its one sidecar-found socket. `stop` exits 0, says the stop was +/// not recorded because the state does not load (naming the file), and +/// leaves the state bytes unchanged. V-17 is its loadable-state control. +#[test] +fn stop_ends_a_live_run_whose_state_will_not_load() { + let run = LiveRun::start(PhaseId::new(142)); + let pgid = run.record.agent_pgid; + assert!( + !group_members(pgid).is_empty(), + "precondition: agent running" + ); + let state_file = run.state_file(); + std::fs::write(&state_file, b"{ this is not json").unwrap(); + let before = std::fs::read(&state_file).unwrap(); + + let output = run.stop(&[]); + let text = describe(&output); + assert!(output.status.success(), "stop must succeed: {text}"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(&format!( + "stopped phase {} through its supervisor", + run.phase + )), + "stop says it stopped the phase: {text}" + ); + assert!( + stdout.contains(&state_file.display().to_string()) + && stdout.contains("does not load") + && stdout.contains("was not recorded"), + "stop says `stopped` was not recorded and names the state file: {text}" + ); + assert!( + holds_within(Duration::from_secs(5), || group_members(pgid).is_empty()), + "the agent group {pgid} is gone" + ); + assert_eq!(run.probe(), Liveness::Gone, "the socket is gone"); + assert_eq!( + std::fs::read(&state_file).unwrap(), + before, + "the state that will not load is left byte for byte" + ); +} From d0c3dfe00c8f978efd731ba95fe996f45d811336 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 09:24:35 -0400 Subject: [PATCH 067/112] feat(50-14): stop a live run through its supervisor socket - stop probes the run's socket after the gate path: ALIVE gets a shutdown, GONE falls back to the lock holder, a probe error refuses - the socket comes from state.supervisor, or from the one sidecar-found socket when the state is missing or does not load; several are a refusal listing each --launch form - a handed_off reply reloads state and retries, 4 requests in all - a shutdown that errors re-probes once, so a supervisor dying under it reads STALE instead of failing the stop - a stop on an unloadable state exits 0 and says stopped was not recorded; STALE is refused until the backstop arm lands Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 394 ++++++++++++++++++++++++++--- 1 file changed, 362 insertions(+), 32 deletions(-) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 38b3b2e9..6fbb9eec 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -1981,30 +1981,347 @@ fn reap_stray_candidates( type AnsweredGate = (Stage, lock::HolderStatus, Option<(u32, Option)>); -/// End a running phase cleanly (23c) — the missing primitive -/// `23-ORPHAN-FORENSICS.md` names as the reason 54 processes accumulated -/// with no remedy but `kill(1)`. Answers `phase`'s open gate if it has one — -/// the primary path, since it is the only one that produces a clean final -/// state: the target unwinds through its own `abort()` and releases its -/// lock; otherwise signals the process recorded in `phase`'s per-phase lock -/// file (never `state.monitor_pid` — T-23-51: the monitor shell script's -/// `trap` closure only ever captures the **agent's** pid, never the -/// trailing `advance` invocation, so by the time `advance` is the shell's -/// foreground child, `monitor_pid` already names a process that exited long -/// ago; `lock::holder`'s recorded pid is the only correct target). +/// End a running phase cleanly (23c, D-04, D-06): the one teardown verb, at +/// every point of a run, that never claims a stop it did not perform. +/// +/// 1. An open gate with a confirmed live waiter is answered with an abort +/// ([`stop_via_gate`]); the waiter unwinds through its own `abort()`. +/// 2. Otherwise the run's supervisor socket is addressed +/// ([`stop_via_socket`]): the state's `supervisor` record names it, or, +/// when the state is missing or does not load, the one live sidecar-found +/// socket for this root and phase. Its probe picks the arm: ALIVE gets a +/// `shutdown`, which ends a run that holds no per-phase lock (999.138). +/// 3. Only when no live socket remains (GONE, or nothing recorded) does the +/// identity-confirmed per-phase lock holder get signalled +/// ([`stop_via_lock`]). +/// +/// No pid read from state is ever signalled. `stopped` is recorded only for +/// a stop performed or confirmed not live, and never into a state file that +/// does not load. pub(crate) fn stop(project_root: &Path, phase: PhaseId) -> Result<(), CliError> { + stop_in(project_root, phase, registry::cache_dir().as_deref()) +} + +/// [`stop`] with the cache dir whose `sock/` holds the sidecars it may +/// search, so tests never sweep the operator's real cache. +fn stop_in(project_root: &Path, phase: PhaseId, cache_dir: Option<&Path>) -> Result<(), CliError> { let (gate_answered, gate_holder) = stop_via_gate(project_root, phase)?; - let signal_sent = if gate_answered { - false - } else { - stop_via_lock(project_root, phase)? + if gate_answered { + return persist_stopped_state(project_root, phase, gate_holder, false); + } + match stop_via_socket(project_root, phase, cache_dir)? { + SocketStop::NoLiveSocket => { + let signal_sent = stop_via_lock(project_root, phase)?; + persist_stopped_state(project_root, phase, None, signal_sent) + } + // The supervisor may still be releasing the per-phase lock as it + // exits, so wait it out like a signalled holder. + SocketStop::Ended => persist_stopped_state(project_root, phase, None, true), + SocketStop::EndedStateUnloadable(state_file) => { + println!( + "stop: phase {phase}'s state file {} does not load, so `stopped` was not \ + recorded — repair or remove it before `devflow resume --phase {phase}`", + state_file.display() + ); + Ok(()) + } + } +} + +/// How long `stop` waits for a supervisor's final shutdown line: the +/// supervisor's own terminate-and-verify wait, twice, plus time to persist. +fn shutdown_final_timeout() -> std::time::Duration { + 2 * agent::TERMINATE_VERIFY_WAIT + std::time::Duration::from_secs(5) +} + +/// Shutdown requests `stop` sends in all when each is answered `handed_off`: +/// the first and 3 retries, each against the record a fresh state reload +/// names (crash row C13). +const MAX_SHUTDOWN_REQUESTS: usize = 4; + +/// What [`stop_via_socket`] did. +enum SocketStop { + /// No live supervisor socket: nothing recorded, or its probe says GONE. + NoLiveSocket, + /// The run was ended through its socket; record `stopped`. + Ended, + /// The run was ended through a sidecar-found socket, but the state file + /// at this path exists and does not load, so nothing can be recorded. + EndedStateUnloadable(PathBuf), +} + +/// The supervisor socket `stop` addresses for a phase, and its provenance. +struct LocatedSocket { + socket: PathBuf, + /// The state's record, when the socket came from it (the backstop + /// cross-checks the sidecar against it). + #[expect(dead_code, reason = "read by the STALE backstop arm")] + record: Option, + /// The state file, when it exists but does not load and the socket came + /// from its sidecar. + unloadable_state: Option, +} + +/// Find the socket for a bare `stop`: the loaded state's `supervisor` record +/// (none → `None`); or, when the state is missing or does not load, the one +/// socket of this root and phase that a sweep finds not GONE. Several such +/// sockets are a refusal listing each, since `stop` cannot tell which is the +/// run. +fn locate_phase_socket( + project_root: &Path, + phase: PhaseId, + cache_dir: Option<&Path>, +) -> Result, CliError> { + let unloadable_state = match workflow::load_state(project_root, phase) { + Ok(state) => { + return Ok(state.supervisor.map(|record| LocatedSocket { + socket: record.socket_path.clone(), + record: Some(record), + unloadable_state: None, + })); + } + Err(workflow::WorkflowError::MissingState(_)) => None, + Err(_) => Some(workflow::state_path(project_root, phase)), }; - persist_stopped_state( - project_root, - phase, - gate_answered.then_some(gate_holder).flatten(), - signal_sent, - ) + let Some(cache_dir) = cache_dir else { + return Ok(None); + }; + let live: Vec = + supervisor::sweep_for_in(cache_dir, project_root, phase) + .into_iter() + .filter(|entry| { + entry.socket_present && !matches!(entry.liveness, Ok(supervisor::Liveness::Gone)) + }) + .collect(); + match live.as_slice() { + [] => Ok(None), + [entry] => Ok(Some(LocatedSocket { + socket: entry.socket_path.clone(), + record: None, + unloadable_state, + })), + several => Err(several_sockets_refusal( + phase, + unloadable_state.as_deref(), + several, + )), + } +} + +/// The refusal for several live sockets of one root and phase: each with its +/// launch id, liveness and the `devflow stop --phase N --launch ` form +/// that addresses it alone. Nothing is signalled, removed or recorded. +fn several_sockets_refusal( + phase: PhaseId, + unloadable_state: Option<&Path>, + entries: &[supervisor::SweepEntry], +) -> CliError { + use std::fmt::Write as _; + let why = unloadable_state.map_or_else( + || "no state file names one".to_string(), + |path| format!("its state file {} does not load", path.display()), + ); + let mut text = format!( + "stop: phase {phase} has {} live supervisor sockets for this project and {why}, so stop \ + cannot tell which one is the run; nothing was signalled, removed or recorded. Stop each \ + one by its launch id:", + entries.len() + ); + for entry in entries { + let _ = write!(text, "\n {}", sweep_entry_stop_line(phase, entry)); + } + CliError::Message(text) +} + +/// One line of [`several_sockets_refusal`]. A socket with no usable sidecar +/// is named by the launch id in its file name (RV-05b); one whose name is +/// outside the DevFlow scheme has no `--launch` form, so it is listed by its +/// path and its ping reply's launch id. +fn sweep_entry_stop_line(phase: PhaseId, entry: &supervisor::SweepEntry) -> String { + let liveness = match &entry.liveness { + Ok(supervisor::Liveness::Alive(_)) => "ALIVE", + Ok(supervisor::Liveness::Stale) => "STALE", + Ok(supervisor::Liveness::Gone) => "GONE", + Err(_) => "probe failed", + }; + let sidecar_note = match (&entry.sidecar, &entry.sidecar_error) { + (Some(_), _) => "", + (None, Some(_)) => ", sidecar does not parse", + (None, None) => ", sidecar missing", + }; + let launch_id = entry + .sidecar + .as_ref() + .map(|sidecar| sidecar.launch_id.clone()) + .or_else(|| entry.launch_id_in_name.clone()); + match launch_id { + Some(id) => format!( + "launch {id} ({liveness}{sidecar_note}): devflow stop --phase {phase} --launch {id}" + ), + None => { + let pinged = match &entry.liveness { + Ok(supervisor::Liveness::Alive(reply)) => reply.launch_id.as_str(), + _ => "unknown", + }; + format!( + "socket {} ({liveness}{sidecar_note}, ping names launch {pinged}): its name is \ + outside the DevFlow socket scheme, so `--launch` cannot address it", + entry.socket_path.display() + ) + } + } +} + +/// The socket arm of [`stop`]. Probes the located socket and acts on its +/// state: ALIVE → `shutdown`; STALE → [`stop_stale_socket`]; GONE → nothing +/// here (the caller falls back to the lock holder); a probe error → a +/// refusal that reaches no teardown (R-6). +/// +/// A `handed_off` reply means the run moved to its successor: the state is +/// reloaded and the record it names is asked, up to +/// [`MAX_SHUTDOWN_REQUESTS`] requests in all; then `stop` fails, recording +/// nothing (crash row C13). +fn stop_via_socket( + project_root: &Path, + phase: PhaseId, + cache_dir: Option<&Path>, +) -> Result { + let mut handed_off = 0; + for _ in 0..MAX_SHUTDOWN_REQUESTS { + let Some(located) = locate_phase_socket(project_root, phase, cache_dir)? else { + return Ok(SocketStop::NoLiveSocket); + }; + let ended = match supervisor::probe(&located.socket) { + Ok(supervisor::Liveness::Gone) => return Ok(SocketStop::NoLiveSocket), + Err(error) => return Err(probe_refusal(phase, &located.socket, &error)), + Ok(supervisor::Liveness::Stale) => stop_stale_socket(phase, &located)?, + Ok(supervisor::Liveness::Alive(_)) => match shutdown_supervisor(phase, &located)? { + ShutdownResult::Ended => true, + ShutdownResult::Relocate => false, + ShutdownResult::HandedOff => { + handed_off += 1; + false + } + }, + }; + if ended { + return Ok(match located.unloadable_state { + Some(state_file) => SocketStop::EndedStateUnloadable(state_file), + None => SocketStop::Ended, + }); + } + } + Err(CliError::Message(format!( + "stop: phase {phase}'s supervisor answered `handed_off` to {handed_off} of \ + {MAX_SHUTDOWN_REQUESTS} shutdown requests (the first and 3 retries, each against the \ + record a fresh state reload named); nothing was stopped or recorded — run `devflow \ + status` and `devflow stop --phase {phase}` again" + ))) +} + +/// What one shutdown attempt in [`stop_via_socket`] came to. +enum ShutdownResult { + /// The supervisor (or, after it died, the backstop) ended the run. + Ended, + /// The supervisor had handed the run to its successor. + HandedOff, + /// The socket went away under the request: locate the run again. + Relocate, +} + +/// Ask the ALIVE supervisor at `located` to stop. +/// +/// A request that fails to reach an answer (a connect refused, a reset or a +/// broken pipe) may have met a supervisor dying under it: its listener is +/// released while the connection waits in the backlog, which +/// `request_shutdown` reports as an error (6d46008 made `probe` re-connect +/// for this, not `request_shutdown`). So the socket is probed once more: +/// STALE goes to [`stop_stale_socket`], GONE re-locates the run, and only a +/// supervisor that still answers, or a failed probe, makes the error stand. +fn shutdown_supervisor( + phase: PhaseId, + located: &LocatedSocket, +) -> Result { + let error = match supervisor::request_shutdown(&located.socket, shutdown_final_timeout()) { + Ok(supervisor::ShutdownOutcome::Stopped { + exit_code, + advance_suppressed, + }) => { + let advance = if advance_suppressed { + "advance suppressed" + } else { + "advance not suppressed" + }; + println!( + "stop: stopped phase {phase} through its supervisor (agent group ended, exit \ + code {exit_code}, {advance})" + ); + return Ok(ShutdownResult::Ended); + } + Ok(supervisor::ShutdownOutcome::HandedOff) => return Ok(ShutdownResult::HandedOff), + Ok(supervisor::ShutdownOutcome::Refused(reason)) => { + return Err(CliError::Message(format!( + "stop: phase {phase}'s supervisor at {} refused the shutdown ({reason}); nothing \ + was stopped or recorded", + located.socket.display() + ))); + } + Ok(other) => { + return Err(CliError::Message(format!( + "stop: phase {phase}'s supervisor gave an unknown shutdown answer ({other:?}); \ + nothing was recorded" + ))); + } + Err(error) => error, + }; + match supervisor::probe(&located.socket) { + Ok(supervisor::Liveness::Stale) => { + println!( + "stop: phase {phase}'s supervisor went away under the shutdown request ({error}); \ + its socket is now STALE" + ); + stop_stale_socket(phase, located).map(|_| ShutdownResult::Ended) + } + Ok(supervisor::Liveness::Gone) => Ok(ShutdownResult::Relocate), + _ => Err(CliError::Message(format!( + "stop: phase {phase}'s supervisor at {} did not answer the shutdown ({error}); \ + nothing was signalled or recorded", + located.socket.display() + ))), + } +} + +/// A probe that could not answer ALIVE, STALE or GONE: refused, naming the +/// error kind and the socket path, and never handed to the backstop (R-6). +fn probe_refusal(phase: PhaseId, socket: &Path, error: &supervisor::ProbeError) -> CliError { + CliError::Message(format!( + "stop: could not probe phase {phase}'s supervisor socket {} ({}: {error}); nothing was \ + signalled, removed or recorded", + socket.display(), + probe_error_kind(error) + )) +} + +/// A short name for a [`supervisor::ProbeError`]'s kind. +fn probe_error_kind(error: &supervisor::ProbeError) -> &'static str { + match error { + supervisor::ProbeError::Stat(_) => "stat failed", + supervisor::ProbeError::NotASocket(_) => "not a socket", + supervisor::ProbeError::ForeignOwner { .. } => "owned by another user", + supervisor::ProbeError::Connect(_) => "connect failed", + supervisor::ProbeError::NoReply(_) => "no valid ping reply", + _ => "probe failed", + } +} + +/// A STALE socket: its supervisor is gone. Refused here; nothing is +/// signalled, removed or recorded. +fn stop_stale_socket(phase: PhaseId, located: &LocatedSocket) -> Result { + Err(CliError::Message(format!( + "stop: phase {phase}'s supervisor socket {} is STALE (its supervisor is gone); nothing \ + was signalled, removed or recorded", + located.socket.display() + ))) } /// The primary path: answer `phase`'s open gate with a rejection whose note @@ -2143,15 +2460,19 @@ fn stop_with_existing_response( } } -/// The fallback: signal the process recorded in `phase`'s per-phase lock -/// file — the process actually running `advance()` and holding the lock -/// across the gate wait. Fail-closed on identity (T-23-52): a live pid that -/// does not look like a devflow process is refused, not signalled, since -/// the lock may be stale with a recycled pid. Never reads -/// `state.monitor_pid` — see [`stop`]'s doc comment. +/// The fallback once no live supervisor socket remains (GONE, or none +/// recorded): signal the process recorded in `phase`'s per-phase lock file, +/// such as a foreground `advance()` holding the lock. Fail-closed on +/// identity (T-23-52): the pid is signalled only when it matches the start +/// time the lock recorded, since the lock may be stale with a recycled pid. +/// The lock file is the only place this reads a pid from; state is never +/// one. fn stop_via_lock(project_root: &Path, phase: PhaseId) -> Result { let Some((pid_str, _path)) = lock::holder(project_root, phase) else { - println!("stop: no lock held for phase {phase} — nothing is running `advance()`"); + println!( + "stop: no live supervisor socket and no lock held for phase {phase} — nothing is \ + running" + ); return Ok(false); }; let Ok(pid) = pid_str.parse::() else { @@ -4979,6 +5300,13 @@ mod tests { ); } + /// [`stop`] for a root with no state file, whose socket search would + /// otherwise sweep the operator's real cache: an empty tempdir cache. + fn stop_with_an_empty_cache(root: &Path, phase: PhaseId) -> Result<(), CliError> { + let cache = tempfile::tempdir().unwrap(); + stop_in(root, phase, Some(cache.path())) + } + #[test] fn stop_via_gate_with_no_waiter_at_a_non_ship_gate_writes_nothing() { let dir = tempfile::tempdir().unwrap(); @@ -6637,7 +6965,7 @@ mod tests { stop_via_lock's identity guard treats as unconfirmable" ); - let err = stop(root, phase) + let err = stop_with_an_empty_cache(root, phase) .expect_err("a legacy lock with no recorded start time must be refused"); let message = err.to_string(); assert!( @@ -6681,7 +7009,8 @@ mod tests { std::fs::create_dir_all(lock_path.parent().unwrap()).unwrap(); std::fs::write(&lock_path, format!("{pid}\n{wrong_start}")).unwrap(); - let err = stop(root, phase).expect_err("a start-time mismatch must be refused"); + let err = stop_with_an_empty_cache(root, phase) + .expect_err("a start-time mismatch must be refused"); let message = err.to_string(); assert!( message.contains(&pid.to_string()), @@ -6731,7 +7060,7 @@ mod tests { std::fs::create_dir_all(lock_path.parent().unwrap()).unwrap(); std::fs::write(&lock_path, format!("{child_pid}\n{child_start}")).unwrap(); - let result = stop(root, phase); + let result = stop_with_an_empty_cache(root, phase); let _ = child.kill(); let _ = child.wait(); @@ -6788,7 +7117,8 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let root = dir.path(); - stop(root, PhaseId::new(201)).expect("stop against nothing present must succeed"); + stop_with_an_empty_cache(root, PhaseId::new(201)) + .expect("stop against nothing present must succeed"); } /// 14-CR-03: a capture file SHORTER than the follower's offset means the From 3618631528f4d6b7cfe05ed1baefb3574fc75754 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 09:27:17 -0400 Subject: [PATCH 068/112] test(50-14): add failing tests for stop on a STALE or unprobeable socket - a SIGKILLed supervisor's orphaned agent group is ended through the backstop, its socket and sidecar removed, stopped recorded - R-10: a supervisor killed at a gate is torn down to GONE with no gate response, the phase named resumable, the lock reclaimable - rename stop_never_signals_the_recorded_monitor_pid to stop_never_signals_a_recorded_pid_it_addresses_the_socket: a live pid planted in state is never signalled - a probe error is refused and never reaches the backstop Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/tests/stop_e2e.rs | 269 +++++++++++++++++++++++++-- 1 file changed, 254 insertions(+), 15 deletions(-) diff --git a/crates/devflow-cli/tests/stop_e2e.rs b/crates/devflow-cli/tests/stop_e2e.rs index 65f52abb..ee6e07d2 100644 --- a/crates/devflow-cli/tests/stop_e2e.rs +++ b/crates/devflow-cli/tests/stop_e2e.rs @@ -167,18 +167,32 @@ fn stop_writes_no_state_while_the_lock_holder_survives_the_signal() { common::kill_and_reap(&mut holder); } +/// T-50-38 (renamed from `stop_never_signals_the_recorded_monitor_pid`): +/// `stop` never signals a pid read from state; it addresses the socket. The +/// state plants a live test-owned process as `monitor_pid` and as the agent +/// group of a `supervisor` record whose socket file does not exist, so the +/// probe reads GONE: `stop` reports nothing running and the planted process +/// is still alive afterwards. #[test] -fn stop_never_signals_the_recorded_monitor_pid() { +fn stop_never_signals_a_recorded_pid_it_addresses_the_socket() { let home = common::HermeticHome::new(); let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(105); - let mut monitor = Command::new("sleep") + let mut planted = Command::new("sleep") .arg("60") .spawn() - .expect("spawn monitor"); + .expect("spawn the planted process"); + let start = devflow_core::agent::process_start_time(planted.id()).expect("start time"); let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); - state.monitor_pid = Some(monitor.id()); + state.monitor_pid = Some(planted.id()); + state.supervisor = Some(SupervisorRecord::new( + home.cache_dir().join("sock").join("missing.sock"), + "launch-missing".to_string(), + planted.id(), + start, + supervisor::boot_id().unwrap_or_default(), + )); devflow_core::workflow::save_state(&state).unwrap(); let output = common::hermetic_devflow(&home) @@ -186,14 +200,25 @@ fn stop_never_signals_the_recorded_monitor_pid() { .arg(root) .output() .expect("run devflow stop"); - assert!(output.status.success()); - assert!(monitor.try_wait().expect("poll monitor").is_none()); + let text = describe(&output); + assert!(output.status.success(), "{text}"); + assert!( + String::from_utf8_lossy(&output.stdout).contains("nothing is running"), + "a GONE socket and no lock read as nothing running: {text}" + ); + assert!( + planted + .try_wait() + .expect("poll the planted process") + .is_none(), + "the planted process named only by state is never signalled" + ); assert!( devflow_core::workflow::load_state(root, phase) .unwrap() .stopped ); - common::kill_and_reap(&mut monitor); + common::kill_and_reap(&mut planted); } #[test] @@ -1218,14 +1243,22 @@ esac ) } -/// A fake agent that keeps running until `trigger` exists, then commits one -/// empty commit, prints the success marker and exits 0. -fn triggered_agent(trigger: &Path) -> String { +/// What a [`triggered_agent`] does once released: one empty commit, the +/// success marker, exit 0. +const SUCCEEDS: &str = r#"git commit -q --allow-empty -m "fake stage work" +printf 'work done\nDEVFLOW_RESULT: {"status":"success","commits":1}\n' +exit 0"#; + +/// What a [`triggered_agent`] does once released: a failed Define, which the +/// never-silent path parks at a human gate. +const FAILS_TO_A_GATE: &str = r#"printf 'DEVFLOW_RESULT: {"status":"failed","reason":"gate fixture"}\n' +exit 0"#; + +/// A fake agent that keeps running until `trigger` exists, then runs +/// `released` ([`SUCCEEDS`] or [`FAILS_TO_A_GATE`]). +fn triggered_agent(trigger: &Path, released: &str) -> String { fake_claude(&format!( - r#"while [ ! -e '{trigger}' ]; do sleep 0.05; done -git commit -q --allow-empty -m "fake stage work" -printf 'work done\nDEVFLOW_RESULT: {{"status":"success","commits":1}}\n' -exit 0"#, + "while [ ! -e '{trigger}' ]; do sleep 0.05; done\n{released}", trigger = trigger.display() )) } @@ -1309,13 +1342,19 @@ struct LiveRun { } impl LiveRun { + /// A run whose agent succeeds once released. fn start(phase: PhaseId) -> Self { + Self::start_with(phase, SUCCEEDS) + } + + /// A run whose agent runs `released` once its trigger exists. + fn start_with(phase: PhaseId, released: &str) -> Self { let home = common::HermeticHome::new(); let repo = tempfile::tempdir().unwrap(); let root = repo.path().canonicalize().unwrap(); init_supervised_repo(&root, phase); let trigger = root.join(".git").join("devflow-test-trigger"); - let fake_bin = common::fake_bin_dir(&triggered_agent(&trigger)); + let fake_bin = common::fake_bin_dir(&triggered_agent(&trigger, released)); let launch_reaper = common::LaunchedRunReaper::new(&root, phase, vec![]); let existing = std::env::var_os("PATH").unwrap_or_default(); let path = format!("{}:{}", fake_bin.path.display(), existing.to_string_lossy()); @@ -1511,3 +1550,203 @@ fn stop_ends_a_live_run_whose_state_will_not_load() { "the state that will not load is left byte for byte" ); } + +/// SIGKILL a run's supervisor by the pid in its sidecar, after confirming +/// that pid is still the recorded process, and return once it is gone and +/// its socket probes STALE. Returns the sidecar. +fn kill_supervisor(run: &LiveRun) -> supervisor::Sidecar { + let sidecar = supervisor::read_sidecar(&run.record.socket_path.with_extension("json")) + .expect("read the sidecar"); + let pid = sidecar.supervisor_pid; + assert!( + devflow_core::agent::is_same_process(pid, sidecar.supervisor_start_time), + "the sidecar's supervisor {pid} is the recorded process" + ); + let signed = libc::pid_t::try_from(pid).expect("pid fits pid_t"); + assert!(signed > 1, "refusing to signal pid {signed}"); + // SAFETY: `signed > 1` names the supervisor just identity-checked. + assert_eq!( + unsafe { libc::kill(signed, libc::SIGKILL) }, + 0, + "SIGKILL {pid}" + ); + assert!( + holds_within(RUN_WAIT, || !devflow_core::agent::agent_running(pid)), + "the supervisor {pid} is gone after SIGKILL" + ); + assert!( + holds_within(Duration::from_secs(10), || matches!( + supervisor::probe(&run.record.socket_path), + Ok(Liveness::Stale) + )), + "a SIGKILLed supervisor leaves its socket STALE" + ); + sidecar +} + +/// A STALE socket goes through the 50-09 backstop: the supervisor of a live +/// run is SIGKILLed, its agent group outlives it, and `stop` ends that group +/// through the validated pidfd path, reports the members it ended, removes +/// the socket and sidecar, and records `stopped`. +#[test] +fn stop_tears_down_a_stale_supervisor_through_the_backstop() { + let run = LiveRun::start(PhaseId::new(143)); + let pgid = run.record.agent_pgid; + kill_supervisor(&run); + assert!( + !group_members(pgid).is_empty(), + "precondition: the orphaned agent group {pgid} outlives its supervisor" + ); + + let output = run.stop(&[]); + let text = describe(&output); + assert!(output.status.success(), "stop must succeed: {text}"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("ended its agent group") && stdout.contains(&pgid.to_string()), + "stop reports the group and the members it ended: {text}" + ); + assert!( + holds_within(Duration::from_secs(5), || group_members(pgid).is_empty()), + "the agent group {pgid} is gone" + ); + let socket = &run.record.socket_path; + assert!( + !socket.exists() && !socket.with_extension("json").exists(), + "the socket and sidecar are removed" + ); + assert_eq!(run.probe(), Liveness::Gone); + assert!( + devflow_core::workflow::load_state(&run.root, run.phase) + .unwrap() + .stopped, + "the performed stop is recorded" + ); +} + +/// R-10: a supervise-mode run parked at a gate, holding the per-phase lock, +/// whose supervisor is SIGKILLed. `stop` writes no gate response (no live +/// waiter), tears the STALE socket down to GONE (the agent had already +/// exited, so nothing is signalled), names `devflow resume --phase N` (R-8), +/// and leaves the lock reclaimable. +#[test] +fn stop_after_a_supervisor_killed_at_a_gate_reaps_and_reports_gone() { + let run = LiveRun::start_with(PhaseId::new(144), FAILS_TO_A_GATE); + std::fs::write(&run.trigger, "go\n").unwrap(); + let gate = Gates::gate_path(&run.root, run.phase, Stage::Define); + assert!( + holds_within(RUN_WAIT, || gate.exists() + && matches!( + devflow_core::lock::holder_status(&run.root, run.phase), + devflow_core::lock::HolderStatus::Live { .. } + )), + "precondition: the run parks at the Define gate holding the lock:\n{}", + std::fs::read_to_string(devflow_core::events::events_path(&run.root)).unwrap_or_default() + ); + kill_supervisor(&run); + + let output = run.stop(&[]); + let text = describe(&output); + assert!(output.status.success(), "stop must succeed: {text}"); + assert!( + String::from_utf8_lossy(&output.stdout).contains(&format!( + "the phase is resumable: run `devflow resume --phase {}`", + run.phase + )), + "a teardown that signalled nothing says the phase is resumable (R-8): {text}" + ); + assert!( + !Gates::response_path(&run.root, run.phase, Stage::Define).exists(), + "no gate response is written without a live waiter" + ); + assert_eq!(run.probe(), Liveness::Gone, "the socket is torn down"); + assert!( + matches!( + devflow_core::lock::holder_status(&run.root, run.phase), + devflow_core::lock::HolderStatus::NoHolder + | devflow_core::lock::HolderStatus::Recycled { .. } + ), + "the dead holder's lock is reclaimable" + ); + drop( + devflow_core::lock::acquire(&run.root, run.phase) + .expect("the per-phase lock can be taken after the stop"), + ); +} + +/// R-6: a probe error is reported and never reaches the backstop. The +/// recorded socket path holds a regular file; the sidecar beside it records +/// a live test-owned group with its real identity, which a backstop run +/// would end. `stop` exits non-zero naming the path problem, the group is +/// untouched, and nothing is removed or recorded. +#[test] +fn stop_reports_a_probe_error_and_never_runs_the_backstop() { + use std::os::unix::process::CommandExt as _; + let home = common::HermeticHome::new(); + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap(); + let phase = PhaseId::new(145); + let sock_dir = home.cache_dir().join("sock"); + std::fs::create_dir_all(&sock_dir).unwrap(); + let socket = sock_dir.join("not-a-socket.sock"); + std::fs::write(&socket, "a regular file\n").unwrap(); + let mut group = Command::new("sleep") + .arg("60") + .process_group(0) + .spawn() + .expect("spawn a test-owned group"); + let pgid = group.id(); + let start = devflow_core::agent::process_start_time(pgid).expect("start time"); + let boot = supervisor::boot_id().expect("a readable boot id"); + let sidecar = serde_json::json!({ + "launch_id": "launch-p", + "project_root": root, + "phase": phase, + "stage": "code", + "agent": "claude", + "launch": "stream", + "agent_pid": pgid, + "agent_pgid": pgid, + "agent_start_time": start, + "boot_id": boot, + "supervisor_pid": 1, + "supervisor_start_time": 0, + "created_at": "2026-10-18T00:00:00Z", + }); + std::fs::write(socket.with_extension("json"), sidecar.to_string()).unwrap(); + let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.clone()); + state.supervisor = Some(SupervisorRecord::new( + socket.clone(), + "launch-p".to_string(), + pgid, + start, + boot, + )); + devflow_core::workflow::save_state(&state).unwrap(); + let state_file = devflow_core::workflow::state_path(&root, phase); + let before = std::fs::read(&state_file).unwrap(); + + let output = common::hermetic_devflow(&home) + .args(["stop", "--phase", &phase.to_string(), "--root"]) + .arg(&root) + .output() + .expect("run devflow stop"); + let text = describe(&output); + assert!( + !output.status.success(), + "a probe error is a refusal: {text}" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains(&socket.display().to_string()) && stderr.contains("not a socket"), + "the refusal names the path and its problem: {text}" + ); + assert!( + group.try_wait().expect("poll the group").is_none(), + "the group the sidecar records is untouched" + ); + assert!(socket.is_file() && socket.with_extension("json").is_file()); + assert_eq!(std::fs::read(&state_file).unwrap(), before); + common::signal_group(pgid, libc::SIGKILL); + let _ = group.wait(); +} From 7fa533b0ce81d36e132a6c5bb47d6a58bc8f8555 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 09:28:32 -0400 Subject: [PATCH 069/112] feat(50-14): tear a STALE supervisor socket down through the backstop - stop hands a STALE socket's sidecar and the state record to the 50-09 backstop: an ended group lists its members, a group proven gone names the phase resumable (R-8), a refusal prints the backstop's text and exits non-zero without recording stopped - a STALE socket whose sidecar is missing or does not parse is refused with nothing signalled or removed - a supervisor that dies under its shutdown request now ends the same way Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 114 ++++++++++++++++++++++++++--- 1 file changed, 103 insertions(+), 11 deletions(-) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 6fbb9eec..04f58cef 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -2055,7 +2055,6 @@ struct LocatedSocket { socket: PathBuf, /// The state's record, when the socket came from it (the backstop /// cross-checks the sidecar against it). - #[expect(dead_code, reason = "read by the STALE backstop arm")] record: Option, /// The state file, when it exists but does not load and the socket came /// from its sidecar. @@ -2194,7 +2193,10 @@ fn stop_via_socket( let ended = match supervisor::probe(&located.socket) { Ok(supervisor::Liveness::Gone) => return Ok(SocketStop::NoLiveSocket), Err(error) => return Err(probe_refusal(phase, &located.socket, &error)), - Ok(supervisor::Liveness::Stale) => stop_stale_socket(phase, &located)?, + Ok(supervisor::Liveness::Stale) => { + stop_stale_socket(phase, &located)?; + true + } Ok(supervisor::Liveness::Alive(_)) => match shutdown_supervisor(phase, &located)? { ShutdownResult::Ended => true, ShutdownResult::Relocate => false, @@ -2280,7 +2282,7 @@ fn shutdown_supervisor( "stop: phase {phase}'s supervisor went away under the shutdown request ({error}); \ its socket is now STALE" ); - stop_stale_socket(phase, located).map(|_| ShutdownResult::Ended) + stop_stale_socket(phase, located).map(|()| ShutdownResult::Ended) } Ok(supervisor::Liveness::Gone) => Ok(ShutdownResult::Relocate), _ => Err(CliError::Message(format!( @@ -2314,14 +2316,104 @@ fn probe_error_kind(error: &supervisor::ProbeError) -> &'static str { } } -/// A STALE socket: its supervisor is gone. Refused here; nothing is -/// signalled, removed or recorded. -fn stop_stale_socket(phase: PhaseId, located: &LocatedSocket) -> Result { - Err(CliError::Message(format!( - "stop: phase {phase}'s supervisor socket {} is STALE (its supervisor is gone); nothing \ - was signalled, removed or recorded", - located.socket.display() - ))) +/// A STALE socket: its supervisor is gone, and its agent group may not be. +/// The sidecar beside the socket is read and handed, with the state's record +/// when there is one, to the 50-09 backstop, which validates the group's +/// identity before it signals anything. A sidecar that cannot be read is a +/// refusal: nothing else can validate the group. +fn stop_stale_socket(phase: PhaseId, located: &LocatedSocket) -> Result<(), CliError> { + let subject = format!("phase {phase}"); + let socket = supervisor::SocketPath::new(located.socket.clone()) + .map_err(|error| stale_refusal(&subject, &located.socket, &error.to_string()))?; + let sidecar = read_stale_sidecar(&subject, &socket)?; + run_backstop(phase, &subject, &socket, &sidecar, located.record.as_ref()) +} + +/// The sidecar of a STALE socket, or the refusal that names why it cannot be +/// used. +fn read_stale_sidecar( + subject: &str, + socket: &supervisor::SocketPath, +) -> Result { + supervisor::read_sidecar(&socket.sidecar_path()).map_err(|error| { + let why = match &error { + supervisor::SidecarError::Io(io) if io.kind() == std::io::ErrorKind::NotFound => { + "is missing".to_string() + } + supervisor::SidecarError::Json(parse) => format!("does not parse: {parse}"), + other => format!("could not be read: {other}"), + }; + stale_refusal(subject, socket.as_path(), &format!("its sidecar {why}")) + }) +} + +/// The refusal for a STALE socket whose agent group nothing can validate: +/// nothing is signalled or removed, and the manual repair is named. +fn stale_refusal(subject: &str, socket: &Path, why: &str) -> CliError { + CliError::Message(format!( + "stop: {subject}: socket {} is STALE and {why}; nothing recorded can validate its agent \ + group, so stop signals nothing — end its agent yourself and remove {} and {}", + socket.display(), + socket.display(), + socket.with_extension("json").display() + )) +} + +/// Run the backstop on a STALE socket and report what it did. A group it +/// ended, or one it proved gone (previous boot, empty group, recycled +/// leader, R-8), is a stop; a refusal prints the backstop's own text (the +/// members, the residual F1 limit, the inspection command) and fails. +fn run_backstop( + phase: PhaseId, + subject: &str, + socket: &supervisor::SocketPath, + sidecar: &supervisor::Sidecar, + record: Option<&SupervisorRecord>, +) -> Result<(), CliError> { + use supervisor::backstop::{BackstopDecision, BackstopOutcome}; + let outcome = + supervisor::backstop::teardown_stale(socket, sidecar, record).map_err(|error| { + CliError::Message(format!( + "stop: {subject}: the stale-socket backstop did not act ({error}); nothing was \ + recorded" + )) + })?; + match outcome { + BackstopOutcome::Killed { members, .. } => { + let pids: Vec = members.iter().map(|m| m.pid.to_string()).collect(); + println!( + "stop: {subject}: its supervisor is gone (STALE socket); ended its agent group {} \ + ({} member(s): pid {}) and removed {} and its sidecar", + sidecar.agent_pgid, + members.len(), + pids.join(", "), + socket.as_path().display() + ); + Ok(()) + } + BackstopOutcome::RemovedNothingSignalled(decision) => { + let why = match decision { + BackstopDecision::PreviousBoot => "previous boot", + BackstopDecision::EmptyGroup => "empty group", + BackstopDecision::LeaderRecycled => "recycled leader", + _ => "nothing to signal", + }; + println!( + "stop: {subject}: nothing was running ({why}); removed the stale socket {}; the \ + phase is resumable: run `devflow resume --phase {phase}`", + socket.as_path().display() + ); + Ok(()) + } + BackstopOutcome::Refused(decision) => Err(CliError::Message(format!( + "stop: {subject}: {}", + decision.describe(socket, sidecar) + ))), + other => Err(CliError::Message(format!( + "stop: {subject}: the stale-socket backstop gave an unknown outcome ({other:?}); \ + nothing was recorded" + ))), + } } /// The primary path: answer `phase`'s open gate with a rejection whose note From 7f02a050211807bdc439886dfb4e569ddc057d14 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 09:31:05 -0400 Subject: [PATCH 070/112] test(50-14): add failing tests for stop --launch and recover's live refusal - stop --launch tears down only that socket; a non-matching id and a phase mismatch change nothing; the bare stop control leaves the superseded socket alone - the several-sockets refusal on a corrupt state prints --launch lines that work when run as printed - RV-05b: an ALIVE socket whose sidecar does not parse is stopped by the launch id in its file name; a STALE one without a usable sidecar is refused - recover --clean refuses a phase whose supervisor is ALIVE Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/tests/stop_e2e.rs | 352 +++++++++++++++++++++++++++ 1 file changed, 352 insertions(+) diff --git a/crates/devflow-cli/tests/stop_e2e.rs b/crates/devflow-cli/tests/stop_e2e.rs index ee6e07d2..1a92cf66 100644 --- a/crates/devflow-cli/tests/stop_e2e.rs +++ b/crates/devflow-cli/tests/stop_e2e.rs @@ -1750,3 +1750,355 @@ fn stop_reports_a_probe_error_and_never_runs_the_backstop() { common::signal_group(pgid, libc::SIGKILL); let _ = group.wait(); } + +/// What a planted orphan socket has beside it. +enum OrphanSidecar { + /// The live run's sidecar, round-tripped through serde with this launch + /// id and the identity of a test child already reaped. + Parsed, + /// A sidecar file that does not parse. + Broken, + /// No sidecar file. + Missing, +} + +/// Plant a STALE socket for the run's root and phase with launch id `id` +/// (a listener bound and dropped) and the sidecar `kind` describes. Returns +/// the socket path. +fn plant_stale_orphan(run: &LiveRun, id: &str, kind: OrphanSidecar) -> PathBuf { + let socket = + supervisor::SocketPath::for_launch(&run.home.cache_dir(), &run.root, run.phase, id) + .expect("the orphan socket path fits sun_path"); + drop(supervisor::publish(&socket).expect("publish the orphan socket")); + let path = socket.as_path().to_path_buf(); + assert!( + matches!(supervisor::probe(&path), Ok(Liveness::Stale)), + "the planted orphan {id} is STALE" + ); + match kind { + OrphanSidecar::Parsed => { + let mut reaped = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a child to reap"); + let start = + devflow_core::agent::process_start_time(reaped.id()).expect("its start time"); + common::kill_and_reap(&mut reaped); + let live = supervisor::read_sidecar(&run.record.socket_path.with_extension("json")) + .expect("the live run's sidecar"); + let mut value = serde_json::to_value(&live).unwrap(); + value["launch_id"] = serde_json::json!(id); + value["agent_pid"] = serde_json::json!(reaped.id()); + value["agent_pgid"] = serde_json::json!(reaped.id()); + value["agent_start_time"] = serde_json::json!(start); + std::fs::write(socket.sidecar_path(), value.to_string()).unwrap(); + supervisor::read_sidecar(&socket.sidecar_path()).expect("the planted sidecar parses"); + } + OrphanSidecar::Broken => std::fs::write(socket.sidecar_path(), "{").unwrap(), + OrphanSidecar::Missing => {} + } + path +} + +/// The socket and sidecar of `socket` both exist. +fn launch_files_present(socket: &Path) -> bool { + socket.exists() && socket.with_extension("json").exists() +} + +/// The run's supervisor answers ping with its own launch id. +fn alive_as(run: &LiveRun) -> bool { + matches!(run.probe(), Liveness::Alive(reply) if reply.launch_id == run.record.launch_id) +} + +/// D-06 / T-50-53: `stop --launch ` tears down exactly the socket with +/// that launch id. A non-matching id and a phase mismatch change nothing; +/// the matching id removes the superseded orphan A and leaves the live run B +/// running, its state unstopped and no gate answered. +#[test] +fn stop_with_a_launch_id_tears_down_only_that_socket() { + let run = LiveRun::start(PhaseId::new(146)); + let orphan = plant_stale_orphan(&run, "orphan-a", OrphanSidecar::Parsed); + let other_phase = PhaseId::new(147).to_string(); + + let nope = run.stop(&["--launch", "nope"]); + assert!(nope.status.success(), "{}", describe(&nope)); + assert!( + String::from_utf8_lossy(&nope.stdout).contains("nothing to stop"), + "a non-matching id says nothing matched: {}", + describe(&nope) + ); + assert!(launch_files_present(&orphan) && alive_as(&run)); + + let mismatch = common::hermetic_devflow(&run.home) + .args(["stop", "--phase", &other_phase, "--launch", "orphan-a"]) + .current_dir(&run.root) + .output() + .expect("run devflow stop"); + assert!( + !mismatch.status.success(), + "a phase mismatch is refused: {}", + describe(&mismatch) + ); + assert!(launch_files_present(&orphan) && alive_as(&run)); + + let output = run.stop(&["--launch", "orphan-a"]); + let text = describe(&output); + assert!(output.status.success(), "{text}"); + assert!( + String::from_utf8_lossy(&output.stdout).contains("launch orphan-a"), + "stop names the launch it tore down: {text}" + ); + assert!( + !orphan.exists() && !orphan.with_extension("json").exists(), + "A's socket and sidecar are removed: {text}" + ); + assert!(alive_as(&run), "B still answers with its own launch id"); + assert!( + !group_members(run.record.agent_pgid).is_empty(), + "B's agent group is alive" + ); + let state = devflow_core::workflow::load_state(&run.root, run.phase).unwrap(); + assert!(!state.stopped, "a targeted stop records nothing"); + assert!( + !Gates::response_path(&run.root, run.phase, Stage::Define).exists(), + "a targeted stop answers no gate" + ); +} + +/// The control for the targeted form: the same fixture with a bare +/// `devflow stop --phase N` stops the run the state records (B) and leaves +/// the superseded socket A, so the two forms address different sockets. +#[test] +fn bare_stop_addresses_the_state_record_and_leaves_a_superseded_socket() { + let run = LiveRun::start(PhaseId::new(148)); + let orphan = plant_stale_orphan(&run, "orphan-a", OrphanSidecar::Parsed); + + let output = run.stop(&[]); + assert!(output.status.success(), "{}", describe(&output)); + assert_eq!(run.probe(), Liveness::Gone, "B is stopped"); + assert!( + devflow_core::workflow::load_state(&run.root, run.phase) + .unwrap() + .stopped + ); + assert!( + launch_files_present(&orphan), + "the superseded socket A is untouched" + ); +} + +/// D-06: `recover --clean` is a file cleaner and refuses a run whose +/// supervisor is ALIVE, deleting nothing. +#[test] +fn recover_clean_refuses_while_the_supervisor_is_alive() { + let run = LiveRun::start(PhaseId::new(149)); + let gate = + devflow_core::gates::Gates::write_gate(&run.root, run.phase, Stage::Define, "planted gate") + .expect("plant a gate file"); + let gate_bytes = std::fs::read(&gate).unwrap(); + let state_bytes = std::fs::read(run.state_file()).unwrap(); + + let output = common::hermetic_devflow(&run.home) + .args(["recover", "--clean", "--phase", &run.phase.to_string()]) + .current_dir(&run.root) + .output() + .expect("run devflow recover"); + let text = describe(&output); + assert!(!output.status.success(), "a live run is refused: {text}"); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains(&format!("devflow stop --phase {}", run.phase)), + "the refusal names the stop verb: {text}" + ); + assert_eq!(std::fs::read(run.state_file()).unwrap(), state_bytes); + assert_eq!(std::fs::read(&gate).unwrap(), gate_bytes); + assert!(alive_as(&run), "the run is untouched"); +} + +/// The `devflow stop ... --launch ` line `stop` printed for `id`, as +/// arguments after the program name. +fn printed_launch_line(output: &std::process::Output, id: &str) -> Vec { + let stderr = String::from_utf8_lossy(&output.stderr); + let wanted = format!("--launch {id}"); + let line = stderr + .lines() + .find(|line| line.trim_end().ends_with(&wanted)) + .unwrap_or_else(|| panic!("no printed line ends with `{wanted}`:\n{stderr}")); + let command = &line[line + .find("devflow stop") + .expect("the line holds the command")..]; + command + .split_whitespace() + .skip(1) + .map(str::to_string) + .collect() +} + +/// T-50-65: several live sockets of one root and phase with a state file +/// that does not load are a refusal listing each `--launch` form, acting on +/// neither. The printed line for B, run as printed, then stops B and leaves +/// the corrupt state bytes unchanged. +#[test] +fn stop_refuses_several_sockets_when_the_state_will_not_load() { + let run = LiveRun::start(PhaseId::new(150)); + let orphan = plant_stale_orphan(&run, "orphan-a", OrphanSidecar::Parsed); + std::fs::write(run.state_file(), b"{ this is not json").unwrap(); + let corrupt = std::fs::read(run.state_file()).unwrap(); + let b_id = run.record.launch_id.clone(); + + let refused = run.stop(&[]); + let text = describe(&refused); + assert!(!refused.status.success(), "several sockets: {text}"); + let stderr = String::from_utf8_lossy(&refused.stderr); + for id in [b_id.as_str(), "orphan-a"] { + assert!( + stderr.contains(&format!("devflow stop --phase {} --launch {id}", run.phase)), + "the refusal lists the --launch form for {id}: {text}" + ); + } + assert!(alive_as(&run) && launch_files_present(&orphan), "{text}"); + + let args = printed_launch_line(&refused, &b_id); + let output = common::hermetic_devflow(&run.home) + .args(&args) + .current_dir(&run.root) + .output() + .expect("run the printed line"); + assert!( + output.status.success(), + "the printed repair works: {args:?}\n{}", + describe(&output) + ); + assert_eq!(run.probe(), Liveness::Gone, "B is stopped"); + assert!(launch_files_present(&orphan), "A is untouched"); + assert_eq!(std::fs::read(run.state_file()).unwrap(), corrupt); +} + +/// RV-05b (V-39): with both the state and B's sidecar unparseable, the +/// refusal names B by the launch id in its file name; a non-matching id and +/// a wrong phase change nothing; the printed `--launch` line stops B and its +/// agent group, leaving A and the corrupt state bytes alone. +#[test] +fn stop_with_a_launch_id_stops_a_live_socket_whose_sidecar_will_not_parse() { + let run = LiveRun::start(PhaseId::new(151)); + let orphan = plant_stale_orphan(&run, "orphan-a", OrphanSidecar::Parsed); + let sidecar_path = run.record.socket_path.with_extension("json"); + let sidecar = supervisor::read_sidecar(&sidecar_path).expect("B's sidecar"); + let (pgid, start) = (sidecar.agent_pgid, sidecar.agent_start_time); + let b_id = run.record.launch_id.clone(); + assert!( + run.record + .socket_path + .file_stem() + .and_then(std::ffi::OsStr::to_str) + .is_some_and(|stem| stem.ends_with(&format!("-{b_id}"))), + "B's launch id is the last segment of its file name" + ); + std::fs::write(run.state_file(), b"{ this is not json").unwrap(); + std::fs::write(&sidecar_path, b"{ nor is this").unwrap(); + let corrupt = std::fs::read(run.state_file()).unwrap(); + + let refused = run.stop(&[]); + let text = describe(&refused); + assert!(!refused.status.success(), "{text}"); + let stderr = String::from_utf8_lossy(&refused.stderr); + let b_line = format!("devflow stop --phase {} --launch {b_id}", run.phase); + assert!( + stderr + .lines() + .any(|line| line.contains(&b_line) && line.contains("sidecar does not parse")), + "B is listed by its file name's launch id, marked unparseable: {text}" + ); + assert!( + stderr.contains(&format!( + "devflow stop --phase {} --launch orphan-a", + run.phase + )), + "{text}" + ); + assert!(alive_as(&run) && launch_files_present(&orphan)); + + let nope = run.stop(&["--launch", "nope"]); + assert!(nope.status.success(), "{}", describe(&nope)); + assert!(String::from_utf8_lossy(&nope.stdout).contains("nothing to stop")); + assert!(alive_as(&run), "a non-matching id leaves B running"); + + let wrong_phase = common::hermetic_devflow(&run.home) + .args([ + "stop", + "--phase", + &PhaseId::new(152).to_string(), + "--launch", + &b_id, + ]) + .current_dir(&run.root) + .output() + .expect("run devflow stop"); + assert!( + !wrong_phase.status.success(), + "B's file name names phase {}: {}", + run.phase, + describe(&wrong_phase) + ); + assert!(alive_as(&run), "a wrong phase leaves B running"); + + assert!( + devflow_core::agent::is_same_process(pgid, start), + "precondition: B's agent leader {pgid} is the recorded process" + ); + let args = printed_launch_line(&refused, &b_id); + let output = common::hermetic_devflow(&run.home) + .args(&args) + .current_dir(&run.root) + .output() + .expect("run the printed line"); + assert!(output.status.success(), "{args:?}\n{}", describe(&output)); + assert_eq!(run.probe(), Liveness::Gone, "B is stopped"); + let signed = libc::pid_t::try_from(pgid).expect("pgid fits pid_t"); + assert!( + holds_within(Duration::from_secs(5), || { + // SAFETY: signal 0 checks for existence and sends nothing. + let rc = unsafe { libc::kill(-signed, 0) }; + rc == -1 && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + }), + "B's agent group {pgid} is gone" + ); + assert!(launch_files_present(&orphan), "A is untouched"); + assert_eq!(std::fs::read(run.state_file()).unwrap(), corrupt); +} + +/// RV-05b (V-40): a STALE socket with no usable sidecar has no identity to +/// validate, so `--launch` refuses it with nothing signalled or removed, +/// for a sidecar that does not parse and for a missing one. Control: a STALE +/// socket whose sidecar parses, in the same fixture, is torn down. +#[test] +fn stop_with_a_launch_id_refuses_a_stale_socket_without_a_usable_sidecar() { + let run = LiveRun::start(PhaseId::new(153)); + let a = plant_stale_orphan(&run, "orphan-a", OrphanSidecar::Parsed); + let c = plant_stale_orphan(&run, "orphan-c", OrphanSidecar::Broken); + let d = plant_stale_orphan(&run, "orphan-d", OrphanSidecar::Missing); + + for (id, socket, why) in [ + ("orphan-c", &c, "does not parse"), + ("orphan-d", &d, "missing"), + ] { + let output = run.stop(&["--launch", id]); + let text = describe(&output); + assert!(!output.status.success(), "{id} is refused: {text}"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("STALE") && stderr.contains(why) && stderr.contains("signals nothing"), + "{id}: the refusal says STALE, why the sidecar is unusable, and that nothing is \ + signalled: {text}" + ); + assert!(socket.exists(), "{id}'s socket is kept"); + assert!(alive_as(&run), "B is untouched"); + } + assert_eq!(std::fs::read(c.with_extension("json")).unwrap(), b"{"); + assert!(!d.with_extension("json").exists()); + + let output = run.stop(&["--launch", "orphan-a"]); + assert!(output.status.success(), "{}", describe(&output)); + assert!(!a.exists() && !a.with_extension("json").exists()); + assert!(alive_as(&run), "B is untouched"); +} From 59c79408597a5ecb15a1e82d5b025ac09ece7f22 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 09:34:11 -0400 Subject: [PATCH 071/112] feat(50-14): add stop --launch and refuse recover --clean on a live run - stop --launch finds the one socket whose sidecar, or with no usable sidecar whose file name, carries the id; a phase or project mismatch changes nothing; the state's own launch runs the plain stop - otherwise only that socket is acted on: ALIVE shutdown (handed_off is reported, not retried), STALE backstop with its sidecar or a refusal without one, GONE nothing to stop; no gate, lock or state is touched - recover --clean --phase N refuses while the supervisor is ALIVE - rewrite the Stop help text and regenerate the help snapshot Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 270 +++++++++++++++++- crates/devflow-cli/src/main.rs | 27 +- .../tests/snapshots/devflow-help.txt | 2 +- 3 files changed, 285 insertions(+), 14 deletions(-) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 04f58cef..1d133987 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -2028,6 +2028,223 @@ fn stop_in(project_root: &Path, phase: PhaseId, cache_dir: Option<&Path>) -> Res } } +/// `devflow stop --phase N --launch ` (D-06): tear down only the +/// supervisor socket with that launch id, the form `devflow doctor` prints +/// for an orphan. The launch id is the address, so a deleted root's orphan +/// is reachable from any directory. +/// +/// The socket is the one whose sidecar carries ``, or, when its sidecar +/// is missing or does not parse, whose file name carries it (RV-05b). If this +/// project's state loads and records that same launch, the plain [`stop`] +/// runs. Otherwise only that socket is acted on: no gate is answered, no +/// lock holder signalled and no state written. +pub(crate) fn stop_launch( + project_root: &Path, + phase: PhaseId, + launch_id: &str, +) -> Result<(), CliError> { + let cache_dir = registry::cache_dir().ok_or_else(|| { + CliError::Message( + "stop: no cache directory (set HOME, XDG_CACHE_HOME or DEVFLOW_CACHE_DIR), so no \ + supervisor socket can be found by launch id" + .to_string(), + ) + })?; + stop_launch_in(project_root, phase, launch_id, &cache_dir) +} + +/// [`stop_launch`] against the sockets under `cache_dir`. +fn stop_launch_in( + project_root: &Path, + phase: PhaseId, + launch_id: &str, + cache_dir: &Path, +) -> Result<(), CliError> { + let Some(entry) = find_launch(cache_dir, launch_id)? else { + println!("stop: no supervisor socket with launch id {launch_id}; nothing to stop"); + return Ok(()); + }; + check_launch_belongs(project_root, phase, launch_id, &entry, cache_dir)?; + let recorded = workflow::load_state(project_root, phase).is_ok_and(|state| { + state + .supervisor + .is_some_and(|record| record.launch_id == launch_id) + }); + if recorded { + return stop_in(project_root, phase, Some(cache_dir)); + } + stop_one_launch(launch_id, &entry) +} + +/// The one socket under `cache_dir` whose sidecar carries `launch_id`, or, +/// with no usable sidecar, whose file name does. Two such sockets are a +/// refusal: the id no longer names one launch. +fn find_launch( + cache_dir: &Path, + launch_id: &str, +) -> Result, CliError> { + let mut matches: Vec = supervisor::sweep_in(cache_dir) + .into_iter() + .filter(|entry| match &entry.sidecar { + Some(sidecar) => sidecar.launch_id == launch_id, + None => entry.launch_id_in_name.as_deref() == Some(launch_id), + }) + .collect(); + if matches.len() > 1 { + let paths: Vec = matches + .iter() + .map(|entry| entry.socket_path.display().to_string()) + .collect(); + return Err(CliError::Message(format!( + "stop: launch id {launch_id} names {} sockets ({}); nothing was changed", + matches.len(), + paths.join(", ") + ))); + } + Ok(matches.pop()) +} + +/// Refuse a launch that is not this project's phase `phase`, changing +/// nothing. A parsed sidecar decides: its phase must be `phase`, and its +/// project root must be this one or no longer exist (a deleted root's +/// orphan). With no usable sidecar, the socket must be one +/// [`supervisor::sweep_for_in`] attributes to this root and phase by its +/// file name. +fn check_launch_belongs( + project_root: &Path, + phase: PhaseId, + launch_id: &str, + entry: &supervisor::SweepEntry, + cache_dir: &Path, +) -> Result<(), CliError> { + let refuse = |why: String| { + Err(CliError::Message(format!( + "stop: launch {launch_id} {why}; nothing was changed" + ))) + }; + match &entry.sidecar { + Some(sidecar) if sidecar.phase != phase => { + refuse(format!("belongs to phase {}, not {phase}", sidecar.phase)) + } + Some(sidecar) if sidecar.project_root != project_root && sidecar.project_root.is_dir() => { + refuse(format!( + "belongs to project {}, not {}; run stop from that project", + sidecar.project_root.display(), + project_root.display() + )) + } + Some(_) => Ok(()), + None => { + let ours = supervisor::sweep_for_in(cache_dir, project_root, phase) + .iter() + .any(|candidate| candidate.socket_path == entry.socket_path); + if ours { + Ok(()) + } else { + refuse(format!( + "has no usable sidecar, and its socket file name {} names another project \ + or phase than {} phase {phase}", + entry.socket_path.display(), + project_root.display() + )) + } + } + } +} + +/// Act on one launch's socket alone: ALIVE → `shutdown` (a `handed_off` +/// answer is reported, never retried against the successor); STALE → the +/// backstop with its parsed sidecar and no state record, or a refusal when +/// it has none; GONE → nothing to stop; a probe error → a refusal. +fn stop_one_launch(launch_id: &str, entry: &supervisor::SweepEntry) -> Result<(), CliError> { + let subject = format!("launch {launch_id}"); + let socket = &entry.socket_path; + match supervisor::probe(socket) { + Ok(supervisor::Liveness::Gone) => { + println!( + "stop: {subject}: socket {} is gone; nothing to stop", + socket.display() + ); + Ok(()) + } + Err(error) => Err(CliError::Message(format!( + "stop: {subject}: could not probe socket {} ({}: {error}); nothing was signalled, \ + removed or recorded", + socket.display(), + probe_error_kind(&error) + ))), + Ok(supervisor::Liveness::Stale) => stop_stale_launch(&subject, entry), + Ok(supervisor::Liveness::Alive(_)) => shutdown_one_launch(&subject, entry), + } +} + +/// [`stop_one_launch`]'s ALIVE arm. A request that errors is probed once +/// more, as [`shutdown_supervisor`] does: STALE goes to the backstop, GONE +/// leaves nothing to stop. +fn shutdown_one_launch(subject: &str, entry: &supervisor::SweepEntry) -> Result<(), CliError> { + let socket = &entry.socket_path; + let error = match supervisor::request_shutdown(socket, shutdown_final_timeout()) { + Ok(supervisor::ShutdownOutcome::Stopped { + exit_code, + advance_suppressed, + }) => { + let advance = if advance_suppressed { + "advance suppressed" + } else { + "advance not suppressed" + }; + println!( + "stop: stopped {subject} through its supervisor (agent group ended, exit code \ + {exit_code}, {advance})" + ); + return Ok(()); + } + Ok(supervisor::ShutdownOutcome::HandedOff) => { + println!("stop: {subject} already handed off to its successor; nothing stopped"); + return Ok(()); + } + Ok(other) => { + return Err(CliError::Message(format!( + "stop: {subject}: the supervisor did not stop ({other:?}); nothing was recorded" + ))); + } + Err(error) => error, + }; + match supervisor::probe(socket) { + Ok(supervisor::Liveness::Stale) => stop_stale_launch(subject, entry), + Ok(supervisor::Liveness::Gone) => { + println!( + "stop: {subject}: socket {} went away under the shutdown request ({error}); \ + nothing left to stop", + socket.display() + ); + Ok(()) + } + _ => Err(CliError::Message(format!( + "stop: {subject}: the supervisor at {} did not answer the shutdown ({error}); \ + nothing was signalled or recorded", + socket.display() + ))), + } +} + +/// [`stop_one_launch`]'s STALE arm: the backstop with the sidecar the sweep +/// parsed and no state record, or, with no usable sidecar, a refusal that +/// signals and removes nothing (RV-05b): there is no pgid, start time or +/// boot id to validate. +fn stop_stale_launch(subject: &str, entry: &supervisor::SweepEntry) -> Result<(), CliError> { + let Some(sidecar) = &entry.sidecar else { + let why = entry.sidecar_error.as_ref().map_or_else( + || "its sidecar is missing".to_string(), + |error| format!("its sidecar is unusable ({error})"), + ); + return Err(stale_refusal(subject, &entry.socket_path, &why)); + }; + let socket = supervisor::SocketPath::new(entry.socket_path.clone()) + .map_err(|error| stale_refusal(subject, &entry.socket_path, &error.to_string()))?; + run_backstop(subject, &socket, sidecar, None, None) +} + /// How long `stop` waits for a supervisor's final shutdown line: the /// supervisor's own terminate-and-verify wait, twice, plus time to persist. fn shutdown_final_timeout() -> std::time::Duration { @@ -2326,7 +2543,13 @@ fn stop_stale_socket(phase: PhaseId, located: &LocatedSocket) -> Result<(), CliE let socket = supervisor::SocketPath::new(located.socket.clone()) .map_err(|error| stale_refusal(&subject, &located.socket, &error.to_string()))?; let sidecar = read_stale_sidecar(&subject, &socket)?; - run_backstop(phase, &subject, &socket, &sidecar, located.record.as_ref()) + run_backstop( + &subject, + &socket, + &sidecar, + located.record.as_ref(), + Some(phase), + ) } /// The sidecar of a STALE socket, or the refusal that names why it cannot be @@ -2361,14 +2584,16 @@ fn stale_refusal(subject: &str, socket: &Path, why: &str) -> CliError { /// Run the backstop on a STALE socket and report what it did. A group it /// ended, or one it proved gone (previous boot, empty group, recycled -/// leader, R-8), is a stop; a refusal prints the backstop's own text (the +/// leader), is a stop; a refusal prints the backstop's own text (the /// members, the residual F1 limit, the inspection command) and fails. +/// `resumable` names the phase a teardown that signalled nothing leaves +/// resumable (R-8); a targeted stop of a superseded launch passes `None`. fn run_backstop( - phase: PhaseId, subject: &str, socket: &supervisor::SocketPath, sidecar: &supervisor::Sidecar, record: Option<&SupervisorRecord>, + resumable: Option, ) -> Result<(), CliError> { use supervisor::backstop::{BackstopDecision, BackstopOutcome}; let outcome = @@ -2398,9 +2623,12 @@ fn run_backstop( BackstopDecision::LeaderRecycled => "recycled leader", _ => "nothing to signal", }; + let resume = resumable.map_or_else(String::new, |phase| { + format!("; the phase is resumable: run `devflow resume --phase {phase}`") + }); println!( - "stop: {subject}: nothing was running ({why}); removed the stale socket {}; the \ - phase is resumable: run `devflow resume --phase {phase}`", + "stop: {subject}: nothing was running ({why}); removed the stale socket {} and its \ + sidecar{resume}", socket.as_path().display() ); Ok(()) @@ -3255,6 +3483,37 @@ fn print_open_branches(project_root: &Path) { } } +/// `recover --clean --phase N` is a file cleaner, not a teardown (D-06): it +/// refuses while the phase's supervisor answers ALIVE, found through the +/// state's record, or, when the state is missing or does not load, through +/// the sidecars of this root and phase. Nothing is deleted. +fn refuse_a_live_supervisor( + project_root: &Path, + phase: PhaseId, + cache_dir: Option<&Path>, +) -> Result<(), CliError> { + let is_alive = |liveness: &Result| { + matches!(liveness, Ok(supervisor::Liveness::Alive(_))) + }; + let alive = match workflow::load_state(project_root, phase) { + Ok(state) => state + .supervisor + .is_some_and(|record| is_alive(&supervisor::probe(&record.socket_path))), + Err(_) => cache_dir.is_some_and(|cache_dir| { + supervisor::sweep_for_in(cache_dir, project_root, phase) + .iter() + .any(|entry| is_alive(&entry.liveness)) + }), + }; + if alive { + return Err(CliError::Message(format!( + "phase {phase} has a live supervisor; run `devflow stop --phase {phase}` first — \ + recover --clean deleted nothing" + ))); + } + Ok(()) +} + pub(crate) fn recover_cmd( project_root: &Path, do_clean: bool, @@ -3267,6 +3526,7 @@ pub(crate) fn recover_cmd( // recorded, 999.128). A live, identity-confirmed holder still // refuses, so a wedged live run needs `devflow stop` first. Some(phase) => { + refuse_a_live_supervisor(project_root, phase, registry::cache_dir().as_deref())?; let report = match recover::clean_phase_report(project_root, phase) { Ok(report) => report, Err(recover::RecoverError::Lock(lock::LockError::Contended { diff --git a/crates/devflow-cli/src/main.rs b/crates/devflow-cli/src/main.rs index 0e04d60b..8b871c5d 100644 --- a/crates/devflow-cli/src/main.rs +++ b/crates/devflow-cli/src/main.rs @@ -26,7 +26,7 @@ mod commands; use commands::{ cleanup, doctor, evidence, gate_list, gate_respond, gate_show, gate_sweep, history_cmd, list, logs, recover_cmd, reference, release_check, release_verify, resolve_gate_target, start, - status, stop, test_cmd, + status, stop, stop_launch, test_cmd, }; use devflow_core::phase_id::PhaseId; @@ -352,18 +352,22 @@ enum Command { #[arg(default_value = ".")] project: PathBuf, }, - /// End a running phase cleanly (23c): answers its open gate with a - /// rejection if one is open — the target unwinds through its own abort - /// path, no signal sent — otherwise signals the process recorded in its - /// per-phase lock file (`.devflow/lock-{phase:02}`), never - /// `state.monitor_pid` (the PID `devflow status` displays, and the - /// wrong one — see `commands::stop`'s doc comment). Idempotent: safe to + /// End a running phase cleanly: answers its open gate with a rejection + /// when a live waiter holds it; otherwise addresses the run's supervisor + /// socket — `shutdown` when ALIVE, the guarded backstop when STALE, the + /// per-phase lock holder when GONE. `--launch ` tears down only the + /// socket with that launch id (the form `devflow doctor` prints for an + /// orphan). Never signals a pid recorded in state. Idempotent: safe to /// run against an already-stopped, never-started, or already-dead /// phase. Stop { /// Phase to stop. #[arg(long)] phase: PhaseId, + /// Tear down only the supervisor socket with this launch id; answers + /// no gate, signals no lock holder and writes no state. + #[arg(long)] + launch: Option, /// Project root. Overrides the positional argument when supplied. #[arg(long)] root: Option, @@ -740,9 +744,16 @@ fn run() -> Result<(), CliError> { } => ship_override(&project_root(project)?, phase, force), Command::Stop { phase, + launch, root, project, - } => stop(&project_root(root.unwrap_or(project))?, phase), + } => { + let root = project_root(root.unwrap_or(project))?; + match launch { + Some(launch_id) => stop_launch(&root, phase, &launch_id), + None => stop(&root, phase), + } + } Command::Evidence { phase, json, diff --git a/crates/devflow-cli/tests/snapshots/devflow-help.txt b/crates/devflow-cli/tests/snapshots/devflow-help.txt index ac8e35cc..8a07484c 100644 --- a/crates/devflow-cli/tests/snapshots/devflow-help.txt +++ b/crates/devflow-cli/tests/snapshots/devflow-help.txt @@ -18,7 +18,7 @@ Commands: doctor Audit the environment and report what's installed, missing, or broken release Read-only release-cut preflight: self-pin, develop/main divergence, crates.io publish order, and changelog-version agreement ship Manually drive a phase through Ship when the monitor that would have consumed its already-written Ship gate response is dead - stop End a running phase cleanly (23c): answers its open gate with a rejection if one is open — the target unwinds through its own abort path, no signal sent — otherwise signals the process recorded in its per-phase lock file (`.devflow/lock-{phase:02}`), never `state.monitor_pid` (the PID `devflow status` displays, and the wrong one — see `commands::stop`'s doc comment). Idempotent: safe to run against an already-stopped, never-started, or already-dead phase + stop End a running phase cleanly: answers its open gate with a rejection when a live waiter holds it; otherwise addresses the run's supervisor socket — `shutdown` when ALIVE, the guarded backstop when STALE, the per-phase lock holder when GONE. `--launch ` tears down only the socket with that launch id (the form `devflow doctor` prints for an orphan). Never signals a pid recorded in state. Idempotent: safe to run against an already-stopped, never-started, or already-dead phase evidence Report DevFlow's own structural record of whether a phase shipped (23-06) — a read-only oracle sourced from the append-only event log, not from any agent-authored attestation document. See `devflow_core::ship_evidence` for the full contract help Print this message or the help of the given subcommand(s) From bbc9fa9c7001b5f889475ba53cb098f77d795cce Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 09:37:17 -0400 Subject: [PATCH 072/112] test(50-14): prove --launch on a deleted root and the recorded launch - doctor's repair line for a deleted root's orphan, run from an unrelated directory with no project state, tears it down; another existing project's orphan is refused and kept - --launch with the state's own launch id runs the plain stop and records its stop reason Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/tests/stop_e2e.rs | 128 +++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/crates/devflow-cli/tests/stop_e2e.rs b/crates/devflow-cli/tests/stop_e2e.rs index 1a92cf66..784070e3 100644 --- a/crates/devflow-cli/tests/stop_e2e.rs +++ b/crates/devflow-cli/tests/stop_e2e.rs @@ -2102,3 +2102,131 @@ fn stop_with_a_launch_id_refuses_a_stale_socket_without_a_usable_sidecar() { assert!(!a.exists() && !a.with_extension("json").exists()); assert!(alive_as(&run), "B is untouched"); } + +/// Plant a STALE socket and a parsed sidecar for `(project_root, phase, +/// id)` under `home`'s cache, recording a test child already reaped as its +/// agent group. Returns the socket path. +fn plant_unattached_orphan( + home: &common::HermeticHome, + project_root: &Path, + phase: PhaseId, + id: &str, +) -> PathBuf { + let socket = supervisor::SocketPath::for_launch(&home.cache_dir(), project_root, phase, id) + .expect("the orphan socket path fits sun_path"); + drop(supervisor::publish(&socket).expect("publish the orphan socket")); + let mut reaped = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn a child to reap"); + let start = devflow_core::agent::process_start_time(reaped.id()).expect("its start time"); + common::kill_and_reap(&mut reaped); + let sidecar = serde_json::json!({ + "launch_id": id, + "project_root": project_root, + "phase": phase, + "stage": "code", + "agent": "claude", + "launch": "stream", + "agent_pid": reaped.id(), + "agent_pgid": reaped.id(), + "agent_start_time": start, + "boot_id": supervisor::boot_id().expect("a readable boot id"), + "supervisor_pid": 1, + "supervisor_start_time": 0, + "created_at": "2026-10-18T00:00:00Z", + }); + std::fs::write(socket.sidecar_path(), sidecar.to_string()).unwrap(); + supervisor::read_sidecar(&socket.sidecar_path()).expect("the planted sidecar parses"); + socket.as_path().to_path_buf() +} + +/// 50-12's repair for a deleted root's orphan works by launch id alone, +/// from any directory, with no project state: `devflow doctor` prints +/// `devflow stop --phase N --launch ` for it, and that line, run from an +/// unrelated directory, tears it down. Control: an orphan whose sidecar +/// names another project that still exists is refused there and kept. +#[test] +fn stop_with_a_launch_id_reaches_a_deleted_roots_orphan_from_any_directory() { + let home = common::HermeticHome::new(); + let dirs = tempfile::tempdir().unwrap(); + let base = dirs.path().canonicalize().unwrap(); + let (gone_root, other_root, elsewhere) = ( + base.join("gone"), + base.join("other"), + base.join("elsewhere"), + ); + for dir in [&gone_root, &other_root, &elsewhere] { + std::fs::create_dir(dir).unwrap(); + } + let phase = PhaseId::new(154); + let deleted = plant_unattached_orphan(&home, &gone_root, phase, "orphan-x"); + let foreign = plant_unattached_orphan(&home, &other_root, phase, "orphan-y"); + std::fs::remove_dir(&gone_root).unwrap(); + let repair = format!("devflow stop --phase {phase} --launch orphan-x"); + + let doctor = common::hermetic_devflow(&home) + .arg("doctor") + .current_dir(&elsewhere) + .output() + .expect("run devflow doctor"); + assert!( + String::from_utf8_lossy(&doctor.stdout).contains(&repair), + "doctor prints `{repair}` for the deleted root's orphan: {}", + describe(&doctor) + ); + + let refused = common::hermetic_devflow(&home) + .args([ + "stop", + "--phase", + &phase.to_string(), + "--launch", + "orphan-y", + ]) + .current_dir(&elsewhere) + .output() + .expect("run devflow stop"); + assert!( + !refused.status.success() + && String::from_utf8_lossy(&refused.stderr).contains("belongs to project"), + "another existing project's orphan is refused: {}", + describe(&refused) + ); + assert!(launch_files_present(&foreign), "the refused orphan is kept"); + + let args: Vec<&str> = repair.split_whitespace().skip(1).collect(); + let output = common::hermetic_devflow(&home) + .args(&args) + .current_dir(&elsewhere) + .output() + .expect("run the doctor's repair"); + assert!(output.status.success(), "{}", describe(&output)); + assert!( + !deleted.exists() && !deleted.with_extension("json").exists(), + "the deleted root's orphan is torn down: {}", + describe(&output) + ); +} + +/// `--launch` with the launch the state itself records runs the plain stop, +/// every arm included: the run is stopped through its socket and the plain +/// stop's own reason is recorded beside the supervisor's, which a targeted +/// stop (it writes no state) never adds. +#[test] +fn stop_with_the_recorded_launch_id_runs_the_plain_stop() { + let run = LiveRun::start(PhaseId::new(155)); + let output = run.stop(&["--launch", &run.record.launch_id]); + assert!(output.status.success(), "{}", describe(&output)); + assert_eq!(run.probe(), Liveness::Gone, "the recorded run is stopped"); + let state = devflow_core::workflow::load_state(&run.root, run.phase).unwrap(); + assert!(state.stopped, "the run is recorded stopped"); + assert!( + state + .stop_reason + .as_deref() + .is_some_and(|reason| reason.contains("stopped via `devflow stop`")), + "the plain stop's own reason is recorded: {:?}", + state.stop_reason + ); +} From 85e1dc771531e77fb0f75ecd3847f30879fa84e9 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 09:49:36 -0400 Subject: [PATCH 073/112] test(50-23): add failing tests for clearing the record on a failed launch - a_failed_monitor_spawn_clears_the_supervisor_record (over-long cache dir makes spawn_monitor fail; short-cache control records a new launch) - the early-failure and canary-refusal tests plant a prior record and require it cleared in memory and on disk - source scan: spawn_agent_and_record never clears the record itself and calls record_failed_launch; refuse_launch clears it Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_launch.rs | 277 ++++++++++++++++++++++ 1 file changed, 277 insertions(+) diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index e332a9aa..84e83ec6 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -4167,6 +4167,9 @@ mod tests { // A stale pid from a prior stage's now-dead monitor — this is what // must be cleared, not carried forward into the new stage. state.monitor_pid = Some(999_999); + // 50-23: the prior stage's supervisor record, which must be cleared + // too, or `status` reads GONE-while-active and names `devflow resume`. + state.supervisor = Some(fabricated_prior_record(root)); workflow::save_state(&state).unwrap(); let result = launch_stage_inner(&mut state, None, None); @@ -4180,11 +4183,19 @@ mod tests { "an early launch failure must clear the stale monitor_pid in-memory, not carry it \ forward from the previous stage" ); + assert_eq!( + state.supervisor, None, + "an early launch failure must clear the prior supervisor record in memory" + ); let reloaded = workflow::load_state(root, phase).unwrap(); assert_eq!( reloaded.monitor_pid, None, "the monitor_pid clear must be persisted to state.json, not just in-memory" ); + assert_eq!( + reloaded.supervisor, None, + "the supervisor record clear must be persisted to state.json" + ); } /// D-15 (44-02): a failed monitor relaunch must leave the phase cron @@ -4848,6 +4859,8 @@ mod tests { let mut state = canary_state(root, phase); // The previous stage's monitor pid, as `transition()` would leave it. state.monitor_pid = Some(4_294_967_000); + // 50-23: and the previous stage's supervisor record. + state.supervisor = Some(fabricated_prior_record(root)); workflow::save_state(&state).unwrap(); let calls = std::cell::Cell::new(0usize); @@ -4874,11 +4887,20 @@ mod tests { "a refused launch must not leave the previous stage's monitor pid standing — \ liveness() would report Stuck and point at `devflow resume`, which cannot help" ); + assert_eq!( + state.supervisor, None, + "a refused launch must not leave the previous stage's supervisor record standing: \ + it reads GONE-while-active and sends the operator to `devflow resume`" + ); let reloaded = workflow::load_state(root, phase).unwrap(); assert!( reloaded.monitor_pid.is_none(), "the cleared pid must be persisted, not only cleared in memory" ); + assert_eq!( + reloaded.supervisor, None, + "the cleared supervisor record must be persisted, not only cleared in memory" + ); assert_eq!( reloaded.canary, Some(canary::CanaryOutcome::Absent), @@ -8539,4 +8561,259 @@ mod tests { ); assert_eq!(leftover.launch_id, run.n.launch_id); } + + // ---- 50-23: every launch failure path has a defined end state ---- + + /// A previous stage's supervisor record whose socket does not exist and + /// whose agent pgid does not fit `pid_t`, so no reaper can signal it. + fn fabricated_prior_record(dir: &Path) -> SupervisorRecord { + SupervisorRecord::new( + dir.join("prior-stage-never-bound.sock"), + "priorstage0000000000".to_string(), + 4_294_967_000, + 1, + String::new(), + ) + } + + /// Names which run of `a_failed_monitor_spawn_clears_the_supervisor_record` + /// a child is (`fail` or `ok`). Not a DevFlow setting, so it carries no + /// `DEVFLOW_` prefix the doc check would require documented. + const SPAWN_CASE_ENV: &str = "P50_23_SPAWN_CASE"; + + /// WR-04's supervisor equivalent (T-50-62): a launch whose + /// `spawn_monitor` fails before spawning anything (its socket path is + /// over `sun_path`, so `SocketPath::for_launch` refuses it) clears the + /// previous stage's record in memory and on disk, and leaves no socket or + /// sidecar under the cache dir. Control: the same launch under a short + /// cache dir records a new record with a new launch id. + #[test] + fn a_failed_monitor_spawn_clears_the_supervisor_record() { + const NAME: &str = + "pipeline_launch::tests::a_failed_monitor_spawn_clears_the_supervisor_record"; + if !devflow_core::test_support::in_child_test(NAME) { + for case in ["fail", "ok"] { + let path_dir = go_waiting_claude_dir(); + let short = crate::test_support::hermetic_cache_dir(); + let cache = if case == "fail" { + let long = short.path().join("c".repeat(80)); + std::fs::create_dir_all(&long).unwrap(); + long + } else { + short.path().to_path_buf() + }; + let env: Vec<(&str, &std::ffi::OsStr)> = vec![ + ("DEVFLOW_CACHE_DIR", cache.as_os_str()), + (SPAWN_CASE_ENV, std::ffi::OsStr::new(case)), + ]; + let output = + devflow_core::test_support::run_test_in_child(NAME, path_dir.path(), &env); + devflow_core::test_support::assert_child_ran_exactly_one_passing_test( + &output, NAME, + ); + } + return; + } + let _guard = env_lock(); + let cache = PathBuf::from( + std::env::var_os("DEVFLOW_CACHE_DIR").expect("the parent passes DEVFLOW_CACHE_DIR"), + ); + let fail = std::env::var(SPAWN_CASE_ENV).expect("the parent names the case") == "fail"; + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + init_repo(root); + let phase = PhaseId::new(2301); + let mut state = State::new( + phase, + AgentKind::Claude, + Mode::Supervise, + root.to_path_buf(), + ); + state.legacy_claude_launch = true; + let prior = fabricated_prior_record(root); + state.supervisor = Some(prior.clone()); + workflow::save_state(&state).unwrap(); + + let result = launch_stage(&mut state, None, None); + let _reap_guard = ReapMonitorOnDrop::after_launch(&state); + let reloaded = workflow::load_state(root, phase).unwrap(); + if fail { + let err = result.expect_err("a socket path over sun_path cannot be bound"); + assert!( + err.to_string().contains("sun_path"), + "the error names the socket path bound: {err}" + ); + assert_eq!( + state.supervisor, None, + "the failed spawn clears the prior record in memory" + ); + assert_eq!( + reloaded.supervisor, None, + "the failed spawn clears the prior record on disk" + ); + let left: Vec = + std::fs::read_dir(devflow_core::registry::socket_dir_in(&cache)) + .map(|entries| entries.filter_map(|e| Some(e.ok()?.path())).collect()) + .unwrap_or_default(); + assert!(left.is_empty(), "no socket or sidecar is left: {left:?}"); + } else { + result.expect("control: the launch succeeds under a short cache dir"); + let record = state + .supervisor + .clone() + .expect("control: the launch records its supervisor"); + assert_ne!( + record.launch_id, prior.launch_id, + "control: a new launch id replaces the prior one" + ); + assert_eq!(reloaded.supervisor.as_ref(), Some(&record)); + let outcome = devflow_core::supervisor::request_shutdown( + &record.socket_path, + SUPERVISOR_STOP_WAIT, + ); + assert!( + matches!( + outcome, + Ok(devflow_core::supervisor::ShutdownOutcome::Stopped { .. }) + ), + "control: the launched supervisor stops through its socket: {outcome:?}" + ); + } + } + + /// `source`'s non-test lines: the part before the first line that is + /// exactly `mod tests {`, without its whole-line `//` comments. A + /// trailing comment stays, as part of its code line. + fn non_test_code_lines(source: &str) -> Vec<&str> { + source + .lines() + .take_while(|line| *line != "mod tests {") + .filter(|line| !line.trim_start().starts_with("//")) + .collect() + } + + /// The lines of the item whose header line starts with `header`, through + /// the next line starting with `closing`; `None` when no line starts with + /// `header`. + fn item_lines<'a>(lines: &[&'a str], header: &str, closing: &str) -> Option> { + let start = lines.iter().position(|line| line.starts_with(header))?; + let mut item = vec![lines[start]]; + for line in &lines[start + 1..] { + item.push(*line); + if line.starts_with(closing) { + break; + } + } + Some(item) + } + + const CLEAR_SPAWN_NOT_FOUND: &str = + "spawn_agent_and_record not found in the non-test code (or under 40 lines)"; + const CLEAR_SPAWN_CLEARS: &str = + "spawn_agent_and_record clears the supervisor record in its own body"; + const CLEAR_SPAWN_NO_HELPER: &str = "spawn_agent_and_record never calls record_failed_launch"; + const CLEAR_REFUSE_NOT_FOUND: &str = + "refuse_launch not found in the non-test code (or under 3 lines)"; + const CLEAR_REFUSE_NO_CLEAR: &str = "refuse_launch does not clear the supervisor record"; + + /// 50-13's rule as a pure scan over `source` (R6-G4): in the non-test, + /// comment-stripped code, `spawn_agent_and_record` never clears the + /// supervisor record itself (only its failure helper does) and calls + /// `record_failed_launch`, and `refuse_launch` clears the record. + fn launch_record_clear_violations(source: &str) -> Vec { + let lines = non_test_code_lines(source); + let has = |item: &[&str], needle: &str| item.iter().any(|line| line.contains(needle)); + let mut violations = Vec::new(); + match item_lines(&lines, "fn spawn_agent_and_record(", "}").filter(|item| item.len() >= 40) + { + None => violations.push(CLEAR_SPAWN_NOT_FOUND.to_string()), + Some(item) => { + if has(&item, "supervisor = None") { + violations.push(CLEAR_SPAWN_CLEARS.to_string()); + } + if !has(&item, "record_failed_launch(") { + violations.push(CLEAR_SPAWN_NO_HELPER.to_string()); + } + } + } + match item_lines(&lines, "fn refuse_launch(", "}").filter(|item| item.len() >= 3) { + None => violations.push(CLEAR_REFUSE_NOT_FOUND.to_string()), + Some(item) if !has(&item, "supervisor = None") => { + violations.push(CLEAR_REFUSE_NO_CLEAR.to_string()); + } + Some(_) => {} + } + violations + } + + /// T-50-62 (permanent source scan, R6-G4): the supervisor record is + /// cleared only on a launch failure path. The real file yields no + /// violation; planted sources, each padded to the floors so it carries + /// one defect, yield exactly that defect's violation. + #[test] + fn the_supervisor_record_is_cleared_only_on_launch_failure_paths() { + let real = launch_record_clear_violations(include_str!("pipeline_launch.rs")); + assert!(real.is_empty(), "pipeline_launch.rs: {real:?}"); + + let pad = |n: usize| " let _filler = 0;\n".repeat(n); + let spawn = |body: &str| format!("fn spawn_agent_and_record(\n{}{body}}}\n", pad(40)); + let refuse = |body: &str| { + format!( + "fn refuse_launch(state: &mut State) {{\n{}{body}}}\n", + pad(2) + ) + }; + let good_spawn = spawn(" return Err(failed(record_failed_launch(state), err));\n"); + let good_refuse = refuse(" state.supervisor = None;\n"); + let only = |violation: &str| vec![violation.to_string()]; + + assert_eq!( + launch_record_clear_violations(&format!("{good_spawn}{good_refuse}")), + Vec::::new(), + "a correct layout" + ); + assert_eq!( + launch_record_clear_violations(&format!( + "{}{good_refuse}", + spawn(" record_failed_launch(state)?;\n state.supervisor = None;\n") + )), + only(CLEAR_SPAWN_CLEARS) + ); + assert_eq!( + launch_record_clear_violations(&format!( + "{}{good_refuse}", + spawn(" record_failed_launch(state)?; // never: state.supervisor = None\n") + )), + only(CLEAR_SPAWN_CLEARS), + "a trailing comment is part of its code line" + ); + assert_eq!( + launch_record_clear_violations(&format!( + "{}{good_refuse}", + spawn(" // record_failed_launch(state)?;\n") + )), + only(CLEAR_SPAWN_NO_HELPER), + "a commented-out call is not a call" + ); + assert_eq!( + launch_record_clear_violations(&format!("{good_spawn}{}", refuse(""))), + only(CLEAR_REFUSE_NO_CLEAR) + ); + assert_eq!( + launch_record_clear_violations(&format!("mod tests {{\n{good_spawn}{good_refuse}")), + vec![ + CLEAR_SPAWN_NOT_FOUND.to_string(), + CLEAR_REFUSE_NOT_FOUND.to_string() + ], + "functions only after the test-module cut are not found" + ); + assert_eq!( + launch_record_clear_violations(&format!( + "fn spawn_agent_and_record(\n{} record_failed_launch(state)?;\n}}\n{good_refuse}", + pad(5) + )), + only(CLEAR_SPAWN_NOT_FOUND), + "a body under its floor" + ); + } } From 91d252c9182f6e4c4b40c0d85d88223974a349b8 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 09:51:32 -0400 Subject: [PATCH 074/112] feat(50-23): clear the supervisor record on a refused or failed launch - record_failed_launch clears state.supervisor and saves (WR-04's rationale moved to the socket record) - spawn_agent_and_record routes every pre-publish error (binary check, capture archive, launch context, spawn_monitor, safe-point saves) through it; the success path never clears the record - refuse_launch clears the record beside the monitor pid Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_launch.rs | 198 +++++++++++++++------- 1 file changed, 134 insertions(+), 64 deletions(-) diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index 84e83ec6..2a5c63c3 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -575,14 +575,16 @@ where /// Abort a launch the canary refused, leaving no half-launched stage behind. /// -/// Clearing `monitor_pid` applies WR-04's rationale (see -/// [`spawn_agent_and_record`]) to an aborted launch: `transition()` has already +/// Clearing `monitor_pid` and `supervisor` applies WR-04's rationale (see +/// [`record_failed_launch`]) to an aborted launch: `transition()` has already /// advanced `state.stage` and saved it by the time this runs, so a refusal that -/// left the PREVIOUS stage's monitor pid standing would make `liveness()` -/// report `Stuck → devflow resume` and send the operator after the wrong -/// problem entirely. The right remedy is the message below. +/// left the PREVIOUS stage's monitor pid or supervisor record standing would +/// read as a run gone while active and send the operator to `devflow resume`, +/// after the wrong problem entirely. With neither, the run reads not launched, +/// and the right remedy is the message below. fn refuse_launch(state: &mut State, message: String) -> Result<(), CliError> { state.monitor_pid = None; + state.supervisor = None; workflow::save_state(state)?; Err(CliError::Message(message)) } @@ -1487,6 +1489,24 @@ pub(crate) fn launch_context() -> Result { )) } +/// End a launch that did not happen (50-23, T-50-62): clear +/// `state.supervisor` and save. +/// +/// WR-04's rationale, moved from the monitor pid to the socket record: +/// `transition()` has already advanced and saved `state.stage` by the time a +/// launch runs, so a record still naming the previous stage's supervisor, +/// which is ending or gone, reads GONE-while-active and sends the operator to +/// `devflow resume`, the wrong remedy. With no record the run reads not +/// launched, and the launch's own error message is the remedy. +/// +/// Called only on a failure path, never before the save that records a new +/// record (50-13's rule). +fn record_failed_launch(state: &mut State) -> Result<(), CliError> { + state.supervisor = None; + workflow::save_state(state)?; + Ok(()) +} + /// The tail of [`launch_stage_inner`]: clear the stale monitor pid, validate /// the agent binary, archive the prior capture, spawn the monitor, and /// record the launch. Extracted (28-03, Task 2) so @@ -1518,66 +1538,26 @@ fn spawn_agent_and_record( state.monitor_pid = None; workflow::save_state(state)?; - ensure_agent_binary(program)?; - - // 17d (Task 2, D-17-D-19) originally placed the self-dogfood - // build-staleness gate here, before spawn_monitor. 25b (D-03) moved it - // out: it is now adjudicated exactly once, in `commands::start`, after - // `state.worktree_path` is set and before this whole launch path is ever - // entered — so a phase that modifies DevFlow's own source can progress - // past every stage boundary instead of being re-blocked at each one. See - // `commands::start` for the new call site and its accompanying D-04/D-05 - // trade-off notes. - if let Some(stamp) = agent_result::archive_phase_files( - &state.project_root, - state - .worktree_path - .as_deref() - .unwrap_or(&state.project_root), - state.phase, - capture_retention(&state.project_root), - ) - .map_err(|err| { - CliError::Message(format!( - "could not archive phase {} capture before rollover: {err}", - state.phase - )) - })? { - events::emit( - &state.project_root, - state.phase, - "capture_archived", - serde_json::json!({ - "stage": archived_stage.unwrap_or(state.stage).to_string(), - "to_stage": state.stage.to_string(), - "stamp": stamp, - }), - ); - } - let ctx = launch_context()?; - let launch_label = match &launch { - monitor::MonitorLaunch::PipeOwning { .. } => "stream", - monitor::MonitorLaunch::Legacy => "capture", + // 50-23 (T-50-62): every error before the publish save ends the launch + // with no supervisor record (see `record_failed_launch`); the success + // path below never clears the record before the save that replaces it. + let SpawnedLaunch { + published, + ctx, + launch_label, + } = match spawn_next_supervisor(state, program, args, extra_env, archived_stage, launch) { + Ok(Some(spawned)) => spawned, + // A stop honoured at safe point 2 or 3 (50-13), already recorded. + Ok(None) => return Ok(()), + Err(err) => { + return Err(match record_failed_launch(state) { + Ok(()) => err, + Err(clear_err) => CliError::Message(format!( + "{err}; clearing the supervisor record after it also failed: {clear_err}" + )), + }); + } }; - #[cfg(test)] - test_hold("before_spawn"); - // Safe point 2 (50-13): a stop taken during the advance ends it before - // the next stage's supervisor is spawned. - if stopped_at_safe_point(state, 2)? { - return Ok(()); - } - let published = monitor::spawn_monitor(state, program, args, extra_env, launch, &ctx) - .map_err(|err| CliError::Message(format!("could not spawn monitor: {err}")))?; - #[cfg(test)] - test_hold("before_publish"); - // Safe point 3 (50-13): the next supervisor answered its ping but is not - // published. On a stop, shut it down through its socket and never save - // its record; the state keeps this run's record, marked stopped. - if with_stop_token(StopToken::safe_point) == Some(SafePoint::StopHere) { - shut_down_unpublished(&published.record); - record_advance_stop(state, 3)?; - return Ok(()); - } let pid = published.supervisor_pid; let socket_path = published.record.socket_path.display().to_string(); let launch_id = published.record.launch_id.clone(); @@ -1647,6 +1627,96 @@ fn spawn_agent_and_record( Ok(()) } +/// What [`spawn_next_supervisor`] hands back to [`spawn_agent_and_record`] +/// for the publish: the ping-confirmed supervisor, the launch context it +/// was spawned under, and the launch shape's label. +struct SpawnedLaunch { + published: monitor::Published, + ctx: monitor::LaunchContext, + launch_label: &'static str, +} + +/// The fallible steps of [`spawn_agent_and_record`] before the publish save: +/// validate the agent binary, archive the prior capture, resolve the launch +/// context, and spawn the next supervisor, with 50-13's safe points 2 and 3 +/// around the spawn. `Ok(None)` when a stop was honoured at one of them (the +/// stop is already recorded). Every `Err` is a launch that did not happen, +/// and its caller clears the supervisor record ([`record_failed_launch`]). +fn spawn_next_supervisor( + state: &mut State, + program: &str, + args: &[String], + extra_env: &[(String, String)], + archived_stage: Option, + launch: monitor::MonitorLaunch, +) -> Result, CliError> { + ensure_agent_binary(program)?; + + // 17d (Task 2, D-17-D-19) originally placed the self-dogfood + // build-staleness gate here, before spawn_monitor. 25b (D-03) moved it + // out: it is now adjudicated exactly once, in `commands::start`, after + // `state.worktree_path` is set and before this whole launch path is ever + // entered — so a phase that modifies DevFlow's own source can progress + // past every stage boundary instead of being re-blocked at each one. See + // `commands::start` for the new call site and its accompanying D-04/D-05 + // trade-off notes. + if let Some(stamp) = agent_result::archive_phase_files( + &state.project_root, + state + .worktree_path + .as_deref() + .unwrap_or(&state.project_root), + state.phase, + capture_retention(&state.project_root), + ) + .map_err(|err| { + CliError::Message(format!( + "could not archive phase {} capture before rollover: {err}", + state.phase + )) + })? { + events::emit( + &state.project_root, + state.phase, + "capture_archived", + serde_json::json!({ + "stage": archived_stage.unwrap_or(state.stage).to_string(), + "to_stage": state.stage.to_string(), + "stamp": stamp, + }), + ); + } + let ctx = launch_context()?; + let launch_label = match &launch { + monitor::MonitorLaunch::PipeOwning { .. } => "stream", + monitor::MonitorLaunch::Legacy => "capture", + }; + #[cfg(test)] + test_hold("before_spawn"); + // Safe point 2 (50-13): a stop taken during the advance ends it before + // the next stage's supervisor is spawned. + if stopped_at_safe_point(state, 2)? { + return Ok(None); + } + let published = monitor::spawn_monitor(state, program, args, extra_env, launch, &ctx) + .map_err(|err| CliError::Message(format!("could not spawn monitor: {err}")))?; + #[cfg(test)] + test_hold("before_publish"); + // Safe point 3 (50-13): the next supervisor answered its ping but is not + // published. On a stop, shut it down through its socket and never save + // its record; the state keeps this run's record, marked stopped. + if with_stop_token(StopToken::safe_point) == Some(SafePoint::StopHere) { + shut_down_unpublished(&published.record); + record_advance_stop(state, 3)?; + return Ok(None); + } + Ok(Some(SpawnedLaunch { + published, + ctx, + launch_label, + })) +} + /// Render the re-scan gate's context: which plan files carry a human-only /// checkpoint DevFlow did not record at this phase's Code preflight. /// From be9bd7af8fcec6a7b3121b8192d3a7627eaba8ed Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 09:56:14 -0400 Subject: [PATCH 075/112] test(50-23): add failing tests for the publish save and its stops - failed-save trio: an unconfirmed shutdown names stop --launch, a confirmed one shuts the new supervisor down, a good save hands off - held-publish trio: a stop sent straight to N+1 is recorded, one still in its kill grace is recorded by N+1 itself (sync-point ordered, with a reversed-release control run), and no stop hands off - runtime.rs source scan for the test-only stopped-outcome hold, and its reasoned doc-check allowlist entry - publish_record scaffold keeps today's single save (RED only) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_launch.rs | 561 +++++++++++++++++- crates/devflow-core/src/supervisor/runtime.rs | 179 ++++++ doc-check-allowlist.toml | 5 + 3 files changed, 741 insertions(+), 4 deletions(-) diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index 2a5c63c3..5cdefb5c 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -1575,10 +1575,11 @@ fn spawn_agent_and_record( // between the two always finds a live address. #[cfg(test)] test_hold("before_record_save"); - state.monitor_pid = Some(pid); - state.supervisor = Some(published.record.clone()); - workflow::save_state(state)?; - record_successor(published.record); + let record = published.record.clone(); + match publish_record(state, published, workflow::save_state)? { + PublishOutcome::HandedOff => record_successor(record), + PublishOutcome::SuccessorAlreadyEnded => {} + } #[cfg(test)] test_hold("after_publish"); match devflow_core::ship::consume_cron_instructions(&state.project_root, state.phase) { @@ -1627,6 +1628,41 @@ fn spawn_agent_and_record( Ok(()) } +/// The stop reason a launcher records when the next stage's supervisor it +/// just published had already ended (50-23, T-50-64). +#[cfg_attr( + not(test), + expect(dead_code, reason = "50-23 Task 2 RED scaffold; GREEN records it") +)] +pub(crate) const NEXT_STOPPED_BEFORE_SAVE_REASON: &str = + "the next stage's supervisor was stopped before its record was saved"; + +/// How [`publish_record`] ended. +enum PublishOutcome { + /// The record is saved and the next supervisor is running it. + HandedOff, + /// The record is saved, but the next supervisor had already ended. + #[cfg_attr( + not(test), + expect(dead_code, reason = "50-23 Task 2 RED scaffold; GREEN constructs it") + )] + #[cfg_attr(test, allow(dead_code))] + SuccessorAlreadyEnded, +} + +/// Record the next stage's supervisor in `state` with one `save` (50-23). +/// RED scaffold: today's behaviour, a single save whose error is returned. +fn publish_record>( + state: &mut State, + published: monitor::Published, + mut save: impl FnMut(&State) -> Result<(), E>, +) -> Result { + state.monitor_pid = Some(published.supervisor_pid); + state.supervisor = Some(published.record); + save(state).map_err(Into::into)?; + Ok(PublishOutcome::HandedOff) +} + /// What [`spawn_next_supervisor`] hands back to [`spawn_agent_and_record`] /// for the publish: the ping-confirmed supervisor, the launch context it /// was spawned under, and the launch shape's label. @@ -8886,4 +8922,521 @@ mod tests { "a body under its floor" ); } + + /// A launch context's supervisor, spawned straight through + /// `spawn_monitor` on the supervise-only harness entry for a fake agent + /// that waits for `go` (never created), with a reaper for it. The state + /// on disk holds `prior` and is left untouched. + struct PublishFixture { + _dir: tempfile::TempDir, + root: PathBuf, + phase: PhaseId, + state: State, + prior: SupervisorRecord, + published: Option, + reaper: Option, + } + + impl PublishFixture { + fn spawn(phase: PhaseId) -> Self { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_path_buf(); + init_repo(&root); + let mut state = State::new(phase, AgentKind::Claude, Mode::Supervise, root.clone()); + state.legacy_claude_launch = true; + let prior = fabricated_prior_record(&root); + state.supervisor = Some(prior.clone()); + workflow::save_state(&state).unwrap(); + let ctx = launch_context().expect("the parent passes DEVFLOW_CACHE_DIR"); + let published = monitor::spawn_monitor( + &state, + "claude", + &[], + &[], + monitor::MonitorLaunch::Legacy, + &ctx, + ) + .expect("the next supervisor publishes"); + let mut holder = state.clone(); + holder.supervisor = Some(published.record.clone()); + let reaper = Some(ReapMonitorOnDrop::after_launch(&holder)); + Self { + _dir: dir, + root, + phase, + state, + prior, + published: Some(published), + reaper, + } + } + + fn take(&mut self) -> monitor::Published { + self.published.take().expect("published once") + } + + fn reloaded(&self) -> State { + workflow::load_state(&self.root, self.phase).expect("the state loads") + } + } + + impl Drop for PublishFixture { + fn drop(&mut self) { + drop(self.reaper.take()); + } + } + + /// A save that always fails, as a full disk would. + fn failing_save(_: &State) -> Result<(), CliError> { + Err(CliError::Workflow( + devflow_core::workflow::WorkflowError::Io(std::io::Error::other( + "injected save failure: no space left on device", + )), + )) + } + + fn probes_alive(socket: &Path) -> bool { + matches!( + devflow_core::supervisor::probe(socket), + Ok(devflow_core::supervisor::Liveness::Alive(_)) + ) + } + + fn probes_gone(socket: &Path) -> bool { + devflow_core::supervisor::probe(socket) + .is_ok_and(|liveness| liveness == devflow_core::supervisor::Liveness::Gone) + } + + /// The parent half of a publish-save test: a child on the supervise-only + /// entry whose fake agent waits for `go` in its working directory. + fn run_publish_child(name: &str) { + let path_dir = go_waiting_claude_dir(); + let output = run_hermetic_test_in_child(name, path_dir.path(), &[]); + devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, name); + } + + /// RV-08 (V-38, T-50-63): a failed publish save whose shutdown of the new + /// supervisor is not confirmed (its recorded socket refuses the connect) + /// keeps the new record in memory, saves nothing over the file's record, + /// and names `devflow stop --phase N --launch `. The new supervisor + /// is still running. Its opposite is the next test, whose confirmed + /// shutdown clears the record and names no repair. + #[test] + fn a_failed_publish_save_whose_shutdown_is_not_confirmed_names_the_stop_launch_repair() { + const NAME: &str = "pipeline_launch::tests::a_failed_publish_save_whose_shutdown_is_not_confirmed_names_the_stop_launch_repair"; + if !devflow_core::test_support::in_child_test(NAME) { + run_publish_child(NAME); + return; + } + let _guard = env_lock(); + let mut fixture = PublishFixture::spawn(PhaseId::new(2302)); + let mut published = fixture.take(); + let real = published.record.socket_path.clone(); + let new_id = published.record.launch_id.clone(); + let refused = fixture.root.join("refused.sock"); + drop(std::os::unix::net::UnixListener::bind(&refused).expect("bind then drop")); + published.record.socket_path = refused.clone(); + + let err = publish_record(&mut fixture.state, published, failing_save) + .err() + .expect("a failed save is an error"); + let text = err.to_string(); + assert!( + text.contains("injected save failure"), + "names the save error: {text}" + ); + let repair = format!("devflow stop --phase {} --launch {new_id}", fixture.phase); + assert!(text.contains(&repair), "names `{repair}`: {text}"); + assert!( + text.contains(&refused.display().to_string()), + "names the socket: {text}" + ); + assert_eq!( + fixture.state.supervisor.as_ref().map(|r| &r.launch_id), + Some(&new_id), + "the in-memory record still names the new launch" + ); + assert_eq!( + fixture.reloaded().supervisor.as_ref(), + Some(&fixture.prior), + "nothing was saved over the file's record" + ); + assert!(probes_alive(&real), "the new supervisor is still running"); + let outcome = devflow_core::supervisor::request_shutdown(&real, SUPERVISOR_STOP_WAIT); + assert!( + matches!( + outcome, + Ok(devflow_core::supervisor::ShutdownOutcome::Stopped { .. }) + ), + "the operator's handle still ends it: {outcome:?}" + ); + } + + /// R-1 step 4 (T-50-63): a failed publish save shuts the new supervisor + /// down through its socket before returning the error: its socket reads + /// Gone, its agent group is gone, and the record does not name it. + #[test] + fn a_failed_publish_save_shuts_the_new_supervisor_down() { + const NAME: &str = + "pipeline_launch::tests::a_failed_publish_save_shuts_the_new_supervisor_down"; + if !devflow_core::test_support::in_child_test(NAME) { + run_publish_child(NAME); + return; + } + let _guard = env_lock(); + let mut fixture = PublishFixture::spawn(PhaseId::new(2303)); + let published = fixture.take(); + let record = published.record.clone(); + + let err = publish_record(&mut fixture.state, published, failing_save) + .err() + .expect("a failed save is an error"); + let text = err.to_string(); + assert!( + text.contains("injected save failure"), + "names the save error: {text}" + ); + assert!( + text.contains("shut down"), + "names the shutdown of the new supervisor: {text}" + ); + assert!( + !text.contains("--launch"), + "a confirmed shutdown needs no repair line: {text}" + ); + assert!( + poll_until(Duration::from_secs(5), || probes_gone(&record.socket_path)), + "the new supervisor's socket reads Gone" + ); + assert!( + poll_until(Duration::from_secs(5), || !group_alive(record.agent_pgid)), + "the new supervisor's agent group is gone" + ); + assert_ne!( + fixture.state.supervisor.as_ref().map(|r| &r.launch_id), + Some(&record.launch_id), + "the record does not name the shut-down supervisor" + ); + } + + /// The control for the failed-save pair: with `workflow::save_state` the + /// publish hands off, the reloaded state names the new launch and the + /// new supervisor probes Alive. + #[test] + fn a_successful_publish_save_records_and_keeps_the_new_supervisor() { + const NAME: &str = "pipeline_launch::tests::a_successful_publish_save_records_and_keeps_the_new_supervisor"; + if !devflow_core::test_support::in_child_test(NAME) { + run_publish_child(NAME); + return; + } + let _guard = env_lock(); + let mut fixture = PublishFixture::spawn(PhaseId::new(2304)); + let published = fixture.take(); + let record = published.record.clone(); + + let outcome = publish_record(&mut fixture.state, published, workflow::save_state) + .expect("control: the save succeeds"); + assert!( + matches!(outcome, PublishOutcome::HandedOff), + "control: a running successor is handed off" + ); + assert_eq!( + fixture.reloaded().supervisor.as_ref(), + Some(&record), + "control: the reloaded state names the new launch" + ); + assert!(!fixture.reloaded().stopped, "control: not stopped"); + assert!( + probes_alive(&record.socket_path), + "control: the new supervisor is running" + ); + let shutdown = + devflow_core::supervisor::request_shutdown(&record.socket_path, SUPERVISOR_STOP_WAIT); + assert!( + matches!( + shutdown, + Ok(devflow_core::supervisor::ShutdownOutcome::Stopped { .. }) + ), + "{shutdown:?}" + ); + } + + /// The next supervisor a held stage N spawned: the one sweep entry whose + /// launch id is not N's, with its sidecar. + fn next_supervisor(run: &HandoffRun) -> (PathBuf, devflow_core::supervisor::Sidecar) { + let found: Vec<_> = devflow_core::supervisor::sweep_in(&run.cache) + .into_iter() + .filter_map(|entry| { + let sidecar = entry.sidecar?; + (sidecar.launch_id != run.n.launch_id).then_some((entry.socket_path, sidecar)) + }) + .collect(); + match <[_; 1]>::try_from(found) { + Ok([one]) => one, + Err(other) => panic!("exactly one next supervisor at the hold: {other:?}"), + } + } + + /// Release N, then wait for its socket to read Gone (its save, probe and + /// `finish` are done). + fn release_and_wait_for_n(run: &HandoffRun) { + run.release(); + assert!( + poll_until(SUPERVISOR_STOP_WAIT, || probes_gone(&run.n.socket_path)), + "N finishes once released: {}", + run.events() + ); + } + + /// T-50-64: a stop sent straight to N+1 while N is held before its + /// publish save (so N+1 ends on 50-13's no-write path, the state still + /// naming N) is recorded by N's post-save re-probe: `stopped` with this + /// plan's reason, the state naming N+1, which reads Gone with no agent + /// left. + #[test] + fn a_stop_sent_straight_to_the_next_supervisor_before_its_record_is_saved_is_recorded() { + const NAME: &str = "pipeline_launch::tests::a_stop_sent_straight_to_the_next_supervisor_before_its_record_is_saved_is_recorded"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, Some("before_publish"), SUCCESS_MARKER); + return; + } + let _guard = env_lock(); + let run = HandoffRun::launch(PhaseId::new(2305)); + run.go(); + run.wait_held(); + let (next_socket, next) = next_supervisor(&run); + let outcome = + devflow_core::supervisor::request_shutdown(&next_socket, SUPERVISOR_STOP_WAIT); + assert!( + matches!( + outcome, + Ok(devflow_core::supervisor::ShutdownOutcome::Stopped { .. }) + ), + "N+1 stops: {outcome:?}" + ); + release_and_wait_for_n(&run); + + let state = run.state(); + assert_eq!( + state.supervisor.as_ref().map(|r| &r.launch_id), + Some(&next.launch_id), + "the state names N+1" + ); + assert!( + state.stopped, + "the stop is recorded (stop_reason {:?})", + state.stop_reason + ); + assert_eq!( + state.stop_reason.as_deref(), + Some(NEXT_STOPPED_BEFORE_SAVE_REASON) + ); + assert!(probes_gone(&next_socket), "N+1 reads Gone"); + assert!( + poll_until(Duration::from_secs(5), || !group_alive(next.agent_pgid)), + "no agent of N+1 survives" + ); + } + + /// The control: the same held publish with no stop hands off, so the + /// state names a running N+1 and is not stopped. + #[test] + fn without_a_stop_to_the_next_supervisor_the_held_publish_hands_off() { + const NAME: &str = "pipeline_launch::tests::without_a_stop_to_the_next_supervisor_the_held_publish_hands_off"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, Some("before_publish"), SUCCESS_MARKER); + return; + } + let _guard = env_lock(); + let run = HandoffRun::launch(PhaseId::new(2306)); + run.go(); + run.wait_held(); + let (next_socket, next) = next_supervisor(&run); + release_and_wait_for_n(&run); + + let state = run.state(); + assert_eq!( + state.supervisor.as_ref().map(|r| &r.launch_id), + Some(&next.launch_id), + "control: the state names N+1" + ); + assert!( + !state.stopped, + "control: not stopped ({:?})", + state.stop_reason + ); + assert!(probes_alive(&next_socket), "control: N+1 runs on"); + } + + /// The stopped-outcome hold variable (R-6E): the path a stopped + /// supervisor waits on between its agent's exit and its outcome decision. + const STOPPED_OUTCOME_HOLD_ENV: &str = "DEVFLOW_TEST_STOPPED_OUTCOME_HOLD"; + + /// Names which run of the kill-grace test a child is (`a` or `b`). Not a + /// DevFlow setting, so no `DEVFLOW_` prefix. + const KILL_GRACE_RUN_ENV: &str = "P50_23_KILL_GRACE_RUN"; + + /// The marker N+1's fake agent leaves when SIGTERM reaches it. + const TERM_SEEN: &str = "term-seen"; + + /// [`handoff_claude_dir`] with a stage N+1 agent that traps SIGTERM by + /// leaving [`TERM_SEEN`] in its working directory and running on, so its + /// supervisor's kill grace ends in SIGKILL. + fn term_trapping_handoff_claude_dir() -> tempfile::TempDir { + let path_dir = handoff_claude_dir(SUCCESS_MARKER); + let script = format!( + "#!/bin/sh\ncase \"$1\" in --version) exit 0 ;; esac\n\ + if [ ! -e first-done ]; then\n : > first-done\n gate=go\n\ + else\n trap ': > {TERM_SEEN}' TERM\n gate=go2\nfi\n\ + i=0\nwhile [ ! -e \"$gate\" ] && [ \"$i\" -lt 1200 ]; do sleep 0.05; i=$((i+1)); done\n\ + printf '%s\\n' '{SUCCESS_MARKER}'\nexit 0\n" + ); + std::fs::write(path_dir.path().join("claude"), script).unwrap(); + path_dir + } + + /// Creates its path when dropped, so a failing kill-grace run never + /// leaves N+1 parked at the stopped-outcome hold. + struct ReleaseOnDrop(PathBuf); + + impl Drop for ReleaseOnDrop { + fn drop(&mut self) { + let _ = std::fs::write(&self.0, ""); + } + } + + /// Wait (bounded) for `path` to exist. + fn wait_for_file(path: &Path, within: Duration) -> bool { + poll_until(within, || path.exists()) + } + + /// R6-E (V-50, T-50-64): a stop N+1 has accepted but not carried out + /// when N saves its record is recorded by N+1 itself. Ordered only by + /// events: N held at `before_publish`, the stop sent to N+1, N+1's + /// agent marker, N+1 parked at the stopped-outcome hold (owner still + /// `agent_running`), then the two releases. Run (a) releases N first: + /// N saves, probes N+1 (running) and hands off with no stop written, + /// then N+1 records its own stop. Run (b), the in-test control, releases + /// N+1 first: it ends on the no-write path and N's re-probe records this + /// plan's reason. The release order alone selects the end state. + #[test] + fn a_next_supervisor_still_in_its_kill_grace_at_publish_records_its_own_stop() { + const NAME: &str = "pipeline_launch::tests::a_next_supervisor_still_in_its_kill_grace_at_publish_records_its_own_stop"; + if !devflow_core::test_support::in_child_test(NAME) { + for run in ["a", "b"] { + let path_dir = term_trapping_handoff_claude_dir(); + let hold_dir = tempfile::Builder::new() + .prefix("dfh") + .tempdir_in("/tmp") + .unwrap(); + let n_hold = std::ffi::OsString::from(format!( + "before_publish:{}", + hold_dir.path().join("release").display() + )); + let k_file = hold_dir.path().join("k-release"); + let env: Vec<(&str, &std::ffi::OsStr)> = vec![ + (TEST_MONITOR_FULL_ENV, std::ffi::OsStr::new("1")), + ("DEVFLOW_GATE_TIMEOUT_SECS", std::ffi::OsStr::new("120")), + (HOLD_ENV, n_hold.as_os_str()), + (STOPPED_OUTCOME_HOLD_ENV, k_file.as_os_str()), + (KILL_GRACE_RUN_ENV, std::ffi::OsStr::new(run)), + ]; + let output = run_hermetic_test_in_child(NAME, path_dir.path(), &env); + devflow_core::test_support::assert_child_ran_exactly_one_passing_test( + &output, NAME, + ); + } + return; + } + let _guard = env_lock(); + let release_first_n = std::env::var(KILL_GRACE_RUN_ENV).expect("the run") == "a"; + let k_file = PathBuf::from( + std::env::var_os(STOPPED_OUTCOME_HOLD_ENV).expect("the parent passes the hold"), + ); + let k_held = PathBuf::from(format!("{}.held", k_file.display())); + let step = SUPERVISOR_STOP_WAIT; + let run = HandoffRun::launch(PhaseId::new(if release_first_n { 2307 } else { 2308 })); + let _release_k = ReleaseOnDrop(k_file.clone()); + run.go(); + run.wait_held(); + let (next_socket, next) = next_supervisor(&run); + let stopper = { + let socket = next_socket.clone(); + std::thread::spawn(move || { + devflow_core::supervisor::request_shutdown(&socket, Duration::from_secs(60)) + }) + }; + assert!( + wait_for_file(&run.root.join(TERM_SEEN), Duration::from_secs(5)), + "N+1 accepted the stop and signalled its agent group" + ); + assert!( + wait_for_file(&k_held, step), + "stopped-outcome hold never reached" + ); + let mut owner = None; + let answered = poll_until(step, || { + match devflow_core::supervisor::probe(&next_socket) { + Ok(devflow_core::supervisor::Liveness::Alive(reply)) => { + owner = Some(reply.owner); + true + } + Ok(other) => panic!("N+1 must be Alive at the hold, found {other:?}"), + Err(_) => false, + } + }); + assert!(answered, "N+1 answers a ping while parked at the hold"); + assert_eq!( + owner.as_deref(), + Some("agent_running"), + "expected owner agent_running, found {owner:?}" + ); + + let ended = |socket: &Path| poll_until(step, || probes_gone(socket)); + if release_first_n { + release_and_wait_for_n(&run); + let state = run.state(); + assert_eq!( + state.supervisor.as_ref().map(|r| &r.launch_id), + Some(&next.launch_id), + "N saved N+1's record" + ); + assert!( + !state.stopped, + "N handed off and wrote no stop: found stopped {} with stop_reason {:?}", + state.stopped, state.stop_reason + ); + std::fs::write(&k_file, "").unwrap(); + assert!(ended(&next_socket), "N+1 finishes once released"); + } else { + std::fs::write(&k_file, "").unwrap(); + assert!(ended(&next_socket), "N+1 finishes once released"); + release_and_wait_for_n(&run); + } + let outcome = stopper.join().expect("the stopper thread"); + assert!( + matches!( + outcome, + Ok(devflow_core::supervisor::ShutdownOutcome::Stopped { .. }) + ), + "the stop sent to N+1 reads Stopped: {outcome:?}" + ); + let state = run.state(); + assert_eq!( + state.supervisor.as_ref().map(|r| &r.launch_id), + Some(&next.launch_id), + "the state names N+1" + ); + assert!(state.stopped, "the stop is recorded"); + let expected = if release_first_n { + SUPERVISOR_STOP_REASON + } else { + NEXT_STOPPED_BEFORE_SAVE_REASON + }; + assert_eq!(state.stop_reason.as_deref(), Some(expected)); + assert!( + poll_until(Duration::from_secs(5), || !group_alive(next.agent_pgid)), + "no agent of N+1 survives" + ); + } } diff --git a/crates/devflow-core/src/supervisor/runtime.rs b/crates/devflow-core/src/supervisor/runtime.rs index 5c088d1e..d51eada0 100644 --- a/crates/devflow-core/src/supervisor/runtime.rs +++ b/crates/devflow-core/src/supervisor/runtime.rs @@ -1646,4 +1646,183 @@ mod tests { supervisor.finish(); assert!(matches!(probe(&path), Ok(Liveness::Gone))); } + + // ---- 50-23 (R6-E): the stopped-outcome test hold stays test-only ---- + + /// `source`'s non-test lines: the part before the first line that is + /// exactly `mod tests {`, without its whole-line `//` comments. + fn non_test_code_lines(source: &str) -> Vec<&str> { + source + .lines() + .take_while(|line| *line != "mod tests {") + .filter(|line| !line.trim_start().starts_with("//")) + .collect() + } + + /// The lines of the item whose header line starts with `header`, through + /// the next line starting with `closing`; `None` when no line starts with + /// `header`. + fn item_lines<'a>(lines: &[&'a str], header: &str, closing: &str) -> Option> { + let start = lines.iter().position(|line| line.starts_with(header))?; + let mut item = vec![lines[start]]; + for line in &lines[start + 1..] { + item.push(*line); + if line.starts_with(closing) { + break; + } + } + Some(item) + } + + const HOLD_NO_LITERAL_READ: &str = + "no literal std::env::var_os/var read of the stopped-outcome hold variable"; + const HOLD_FN_NOT_ONCE: &str = "fn stopped_outcome_test_hold( is not defined exactly once"; + const HOLD_NOT_GATED: &str = "stopped_outcome_test_hold is not directly under #[cfg(any(test, feature = \"test-support\"))]"; + const WAIT_AGENT_NOT_FOUND: &str = "Supervisor::wait_agent not found (or under 5 lines)"; + const WAIT_AGENT_NO_HOLD: &str = + "Supervisor::wait_agent does not call stopped_outcome_test_hold"; + + /// R6-E as a pure scan over `source`'s non-test, comment-stripped code: + /// the hold variable is read literally (the form the doc check sees), + /// `stopped_outcome_test_hold` is defined once directly under the + /// test-support gate, and `wait_agent` calls it. + fn stopped_outcome_hold_violations(source: &str) -> Vec { + let lines = non_test_code_lines(source); + let mut violations = Vec::new(); + let reads = [ + r#"std::env::var_os("DEVFLOW_TEST_STOPPED_OUTCOME_HOLD")"#, + r#"std::env::var("DEVFLOW_TEST_STOPPED_OUTCOME_HOLD")"#, + ]; + if !lines + .iter() + .any(|line| reads.iter().any(|read| line.contains(read))) + { + violations.push(HOLD_NO_LITERAL_READ.to_string()); + } + let defined: Vec = lines + .iter() + .enumerate() + .filter(|(_, line)| { + line.trim_start() + .starts_with("fn stopped_outcome_test_hold(") + }) + .map(|(at, _)| at) + .collect(); + match defined.as_slice() { + [at] => { + let gated = at.checked_sub(1).is_some_and(|above| { + lines[above].ends_with(r#"#[cfg(any(test, feature = "test-support"))]"#) + }); + if !gated { + violations.push(HOLD_NOT_GATED.to_string()); + } + } + _ => violations.push(HOLD_FN_NOT_ONCE.to_string()), + } + match item_lines(&lines, " pub fn wait_agent(", " }").filter(|item| item.len() >= 5) { + None => violations.push(WAIT_AGENT_NOT_FOUND.to_string()), + Some(item) => { + if !item + .iter() + .any(|line| line.contains("stopped_outcome_test_hold(")) + { + violations.push(WAIT_AGENT_NO_HOLD.to_string()); + } + } + } + violations + } + + /// T-50-85 (permanent source scan, R6-G4, R6-E): a release build never + /// compiles the stopped-outcome hold. The real file yields no violation; + /// each planted source yields the violation it plants. + #[test] + fn the_stopped_outcome_hold_is_test_only_and_read_literally() { + let source = include_str!("runtime.rs"); + assert!( + source.lines().any(|line| line == "mod tests {"), + "runtime.rs must keep its `mod tests {{` line, or the scan would read its own controls" + ); + let real = stopped_outcome_hold_violations(source); + assert!(real.is_empty(), "runtime.rs: {real:?}"); + + let gate = "#[cfg(any(test, feature = \"test-support\"))]\n"; + let read = " let _path = std::env::var_os(\"DEVFLOW_TEST_STOPPED_OUTCOME_HOLD\");\n"; + let hold = |attribute: &str, body: &str| { + format!("{attribute}fn stopped_outcome_test_hold(stop_requested: bool) {{\n{body}}}\n") + }; + let pad = " let _filler = 0;\n".repeat(4); + let call = " stopped_outcome_test_hold(true);\n"; + let wait_agent = |body: &str| { + format!("impl Supervisor {{\n pub fn wait_agent(&self) {{\n{pad}{body} }}\n") + }; + let other = |body: &str| format!(" pub fn token(&self) {{\n{body} }}\n}}\n"); + let good = format!("{}{}{}", hold(gate, read), wait_agent(call), other("")); + let only = |violation: &str| vec![violation.to_string()]; + + assert_eq!( + stopped_outcome_hold_violations(&good), + Vec::::new(), + "a correct layout" + ); + assert_eq!( + stopped_outcome_hold_violations(&format!( + "{}{}{}", + hold("#[cfg(test)]\n", read), + wait_agent(call), + other("") + )), + only(HOLD_NOT_GATED), + "gated by cfg(test) only" + ); + assert_eq!( + stopped_outcome_hold_violations(&format!( + "{}{}{}", + hold("", read), + wait_agent(call), + other("") + )), + only(HOLD_NOT_GATED), + "no attribute" + ); + assert_eq!( + stopped_outcome_hold_violations(&format!( + "const HOLD: &str = \"hold\";\n{}{}{}", + hold(gate, " let _path = std::env::var_os(HOLD);\n"), + wait_agent(call), + other("") + )), + only(HOLD_NO_LITERAL_READ), + "a read through a constant" + ); + assert_eq!( + stopped_outcome_hold_violations(&format!( + "{}{}{}", + hold(gate, read), + wait_agent(" // stopped_outcome_test_hold(true);\n"), + other("") + )), + only(WAIT_AGENT_NO_HOLD), + "a commented-out call" + ); + assert_eq!( + stopped_outcome_hold_violations(&format!( + "{}{}{}", + hold(gate, read), + wait_agent(""), + other(call) + )), + only(WAIT_AGENT_NO_HOLD), + "the call moved to another method" + ); + assert_eq!( + stopped_outcome_hold_violations(&format!("mod tests {{\n{good}")), + vec![ + HOLD_NO_LITERAL_READ.to_string(), + HOLD_FN_NOT_ONCE.to_string(), + WAIT_AGENT_NOT_FOUND.to_string() + ], + "the read, the hold and a decoy wait_agent only after the test-module cut" + ); + } } diff --git a/doc-check-allowlist.toml b/doc-check-allowlist.toml index b8e0a720..6da41482 100644 --- a/doc-check-allowlist.toml +++ b/doc-check-allowlist.toml @@ -64,3 +64,8 @@ reason = "Test-only: selects the advance-running monitor harness entry for #[cfg kind = "source_to_docs" token = "DEVFLOW_TEST_HOLD" reason = "Test-only: #[cfg(test)] hold points in the devflow-core monitor (50-24) and the CLI stage handoff (50-13, 50-23) wait while it names : and that path does not exist, so tests can land a fault in a chosen window; compiled out of release builds, so it is not an operator setting." + +[[exceptions]] +kind = "source_to_docs" +token = "DEVFLOW_TEST_STOPPED_OUTCOME_HOLD" +reason = "Test-only: under the devflow-core test-support feature, Supervisor::wait_agent waits while it names a path that does not exist, after a stopped agent exits and before the stopped outcome is decided, so a CLI test can order that decision against the next stage's publish; compiled out of release builds, so it is not an operator setting (Phase 50, plan 50-23)." From cd9235cec32bfc9d9e81be98a15bb28eb4cdc785 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 10:00:40 -0400 Subject: [PATCH 076/112] feat(50-23): shut down an unrecorded successor and record a stop it took - publish_record: one save; a failed save shuts the new supervisor down through its socket, and an unconfirmed shutdown keeps the record and names devflow stop --phase N --launch (RV-08) - a successful save re-probes the successor; one that already ended is recorded stopped with its own reason, a crashed (STALE) one is kept - test-only stopped_outcome_test_hold in Supervisor::wait_agent, gated by cfg(any(test, feature = "test-support")) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_launch.rs | 139 ++++++++++++++++-- crates/devflow-core/src/supervisor/runtime.rs | 39 +++++ 2 files changed, 164 insertions(+), 14 deletions(-) diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index 5cdefb5c..0cce1957 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -1630,37 +1630,148 @@ fn spawn_agent_and_record( /// The stop reason a launcher records when the next stage's supervisor it /// just published had already ended (50-23, T-50-64). -#[cfg_attr( - not(test), - expect(dead_code, reason = "50-23 Task 2 RED scaffold; GREEN records it") -)] pub(crate) const NEXT_STOPPED_BEFORE_SAVE_REASON: &str = "the next stage's supervisor was stopped before its record was saved"; +/// How long [`successor_runs_on`] keeps re-probing a STALE successor: a +/// finishing supervisor drops its listener just before it unlinks. +const STALE_REPROBE_WINDOW: Duration = Duration::from_millis(200); + +/// The interval of [`successor_runs_on`]'s STALE re-probe. +const STALE_REPROBE_INTERVAL: Duration = Duration::from_millis(20); + /// How [`publish_record`] ended. enum PublishOutcome { /// The record is saved and the next supervisor is running it. HandedOff, - /// The record is saved, but the next supervisor had already ended. - #[cfg_attr( - not(test), - expect(dead_code, reason = "50-23 Task 2 RED scaffold; GREEN constructs it") - )] - #[cfg_attr(test, allow(dead_code))] + /// The record is saved, but the next supervisor had already ended, so + /// the run is recorded `stopped`. SuccessorAlreadyEnded, } -/// Record the next stage's supervisor in `state` with one `save` (50-23). -/// RED scaffold: today's behaviour, a single save whose error is returned. +/// Record the next stage's supervisor (`published`) in `state` with one +/// `save`, the publish (50-13, R-1 step 4; 50-23). +/// +/// - The save fails: shut the new supervisor down through its socket +/// (bounded by [`SUPERVISOR_STOP_WAIT`]), so no live supervisor is left +/// unrecorded, and return the error ([`shut_down_unrecorded`]). +/// - The save succeeds: probe the new socket once ([`successor_runs_on`]). +/// A successor still running is handed off. One that already ended (a +/// stop sent straight to it takes 50-13's no-write path while the state +/// still names this launcher's run) is recorded here: `stopped` with +/// [`NEXT_STOPPED_BEFORE_SAVE_REASON`], saved again under the per-phase +/// lock the caller holds. +/// +/// A successor that has accepted a stop but is still in its kill grace +/// answers owner `agent_running` and is handed off: this save lands before +/// that probe, and it loads the state only after its outcome is decided, so +/// it finds its own launch id and records its own stop. +/// +/// Signals nothing by pid: every step goes through `published`'s socket. fn publish_record>( state: &mut State, published: monitor::Published, mut save: impl FnMut(&State) -> Result<(), E>, ) -> Result { + let record = published.record; state.monitor_pid = Some(published.supervisor_pid); - state.supervisor = Some(published.record); + state.supervisor = Some(record.clone()); + if let Err(err) = save(state) { + return Err(shut_down_unrecorded(state, &record, err.into())); + } + if successor_runs_on(&record.socket_path) { + return Ok(PublishOutcome::HandedOff); + } + state.stopped = true; + state.stop_reason = Some(NEXT_STOPPED_BEFORE_SAVE_REASON.to_string()); save(state).map_err(Into::into)?; - Ok(PublishOutcome::HandedOff) + info!( + "phase {}: {NEXT_STOPPED_BEFORE_SAVE_REASON} (launch {})", + state.phase, record.launch_id + ); + Ok(PublishOutcome::SuccessorAlreadyEnded) +} + +/// After a failed publish save (`save_err`): ask the unrecorded supervisor +/// at `record` to stop, and return the launch's error. +/// +/// A confirmed `Stopped` clears the record (best effort) and the error says +/// the supervisor was shut down. Anything else (RV-08: `HandedOff`, +/// `Refused`, a probe error or a timeout) means it may still be running: +/// nothing is cleared or saved, so the in-memory record keeps naming it and +/// no save erases the handle the file holds, and the error names its socket, +/// its launch id and the `devflow stop --phase N --launch ` repair, which +/// finds it by its sidecar whatever the state says (50-14). +fn shut_down_unrecorded( + state: &mut State, + record: &SupervisorRecord, + save_err: CliError, +) -> CliError { + let failed = format!( + "could not save the record of the next stage's supervisor (launch {}): {save_err}", + record.launch_id + ); + match devflow_core::supervisor::request_shutdown(&record.socket_path, SUPERVISOR_STOP_WAIT) { + Ok(devflow_core::supervisor::ShutdownOutcome::Stopped { .. }) => { + let cleared = record_failed_launch(state) + .err() + .map(|err| format!("; clearing its record also failed: {err}")) + .unwrap_or_default(); + CliError::Message(format!( + "{failed}; that supervisor was shut down through its socket{cleared}" + )) + } + other => { + let outcome = match other { + Ok(outcome) => format!("{outcome:?}"), + Err(err) => err.to_string(), + }; + CliError::Message(format!( + "{failed}; shutting it down at {} did not confirm a stop ({outcome}), so it may \ + still be running; run `devflow stop --phase {} --launch {}` to end it", + record.socket_path.display(), + state.phase, + record.launch_id + )) + } + } +} + +/// Whether the successor whose record was just saved is still running and +/// can be handed off: Alive with owner `agent_running`, `advancing` or +/// `gate_wait`. Gone, or Alive with any other owner (it finished without +/// advancing), is `false`. STALE is re-probed for +/// [`STALE_REPROBE_WINDOW`]: Gone within it is `false`, and a socket still +/// STALE is `true` (a crashed successor stays recorded and reads STALE, a +/// real finding with its own repair). A probe error is `true`: the record +/// stays and `status` reports the error. +fn successor_runs_on(socket: &Path) -> bool { + use devflow_core::supervisor::{Liveness, probe}; + let running = [ + OwnerPhase::AgentRunning, + OwnerPhase::Advancing, + OwnerPhase::GateWait, + ]; + let deadline = std::time::Instant::now() + STALE_REPROBE_WINDOW; + loop { + match probe(socket) { + Ok(Liveness::Alive(reply)) => { + return running.iter().any(|phase| phase.as_str() == reply.owner); + } + Ok(Liveness::Gone) => return false, + Ok(Liveness::Stale) if std::time::Instant::now() < deadline => { + std::thread::sleep(STALE_REPROBE_INTERVAL); + } + Ok(Liveness::Stale) => return true, + Err(err) => { + warn!( + "could not probe the next supervisor at {} after its publish: {err}", + socket.display() + ); + return true; + } + } + } } /// What [`spawn_next_supervisor`] hands back to [`spawn_agent_and_record`] diff --git a/crates/devflow-core/src/supervisor/runtime.rs b/crates/devflow-core/src/supervisor/runtime.rs index d51eada0..f4f9b4f4 100644 --- a/crates/devflow-core/src/supervisor/runtime.rs +++ b/crates/devflow-core/src/supervisor/runtime.rs @@ -329,6 +329,40 @@ fn wait_for_exit_without_reaping(pid: u32) -> io::Result<()> { } } +/// How long [`stopped_outcome_test_hold`] waits for its release file: above +/// the bounded waits of the test that uses it, combined. +#[cfg(any(test, feature = "test-support"))] +const STOPPED_OUTCOME_HOLD_WAIT: std::time::Duration = std::time::Duration::from_secs(60); + +/// Test-only hold (50-23, R6-E). When `stop_requested` and +/// `DEVFLOW_TEST_STOPPED_OUTCOME_HOLD` names a path: create `.held`, +/// then wait, polling every 20 ms and bounded by +/// [`STOPPED_OUTCOME_HOLD_WAIT`], until `` exists. [`Supervisor::wait_agent`] +/// calls it after the agent's exit is observed and before the owner lock is +/// taken for the exit decision, so a parked supervisor still answers pings +/// with owner `agent_running` and its outcome is undecided: a CLI test can +/// order that decision against the next stage's publish. Returns at once +/// for a supervisor whose stop was not requested. Compiled only under +/// `cfg(test)` or the `test-support` feature, together with its one call; a +/// release build has neither. +#[cfg(any(test, feature = "test-support"))] +fn stopped_outcome_test_hold(stop_requested: bool) { + if !stop_requested { + return; + } + let Some(release) = std::env::var_os("DEVFLOW_TEST_STOPPED_OUTCOME_HOLD") else { + return; + }; + let release = std::path::PathBuf::from(release); + let mut held = release.clone().into_os_string(); + held.push(".held"); + let _ = std::fs::write(&held, ""); + let deadline = std::time::Instant::now() + STOPPED_OUTCOME_HOLD_WAIT; + while !release.exists() && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + /// An exit status as a code: the exit code, or `128 + n` for a death by /// signal `n` (the `monitor` convention: SIGTERM is 143, SIGKILL 137). pub(crate) fn exit_code(status: std::process::ExitStatus) -> i32 { @@ -514,6 +548,11 @@ impl Supervisor { /// (RESEARCH Pitfall 9). pub fn wait_agent(&self, child: &mut std::process::Child) -> io::Result { wait_for_exit_without_reaping(child.id())?; + // Test-only (50-23, R6-E): park a stopped supervisor between the + // observed exit and its outcome decision. Compiled out of release + // builds. + #[cfg(any(test, feature = "test-support"))] + stopped_outcome_test_hold(self.token.stop_requested()); let pgid = self.agent_pgid; self.token.observe_exit(|stop_requested| { if stop_requested { From 695d4837156662f810ff8c16b5beb133d64fc699 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 10:03:27 -0400 Subject: [PATCH 077/112] test(50-23): add failing tests for forwarding a late stop to N+1 - V-35/V-48: SIGTERM at before_record_save, after_publish and before_handoff_mark, and a socket stop answered stopped, each end with N+1 stopped and its own stop recorded; no-stop control runs on - source scan: run_monitor takes the successor, holds, marks handed off, then checks for a stop, in that order (planted controls) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_launch.rs | 264 ++++++++++++++++++++++ 1 file changed, 264 insertions(+) diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index 0cce1957..4bfc3939 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -9550,4 +9550,268 @@ mod tests { "no agent of N+1 survives" ); } + + // ---- 50-23 Task 3: a stop after safe point 3 is forwarded (RV-04) ---- + + /// Names which run of the forward test a child is (`a` to `d`). Not a + /// DevFlow setting, so no `DEVFLOW_` prefix. + const FORWARD_RUN_ENV: &str = "P50_23_FORWARD_RUN"; + + /// The parent half of a forward test: [`run_handoff_child`] with the run + /// label passed to the child. + fn run_forward_child(name: &str, hold: &str, label: &str) { + let path_dir = handoff_claude_dir(SUCCESS_MARKER); + let hold_dir = tempfile::Builder::new() + .prefix("dfh") + .tempdir_in("/tmp") + .unwrap(); + let hold_value = std::ffi::OsString::from(format!( + "{hold}:{}", + hold_dir.path().join("release").display() + )); + let env: Vec<(&str, &std::ffi::OsStr)> = vec![ + (TEST_MONITOR_FULL_ENV, std::ffi::OsStr::new("1")), + ("DEVFLOW_GATE_TIMEOUT_SECS", std::ffi::OsStr::new("120")), + (HOLD_ENV, hold_value.as_os_str()), + (FORWARD_RUN_ENV, std::ffi::OsStr::new(label)), + ]; + let output = run_hermetic_test_in_child(name, path_dir.path(), &env); + devflow_core::test_support::assert_child_ran_exactly_one_passing_test(&output, name); + } + + /// RV-04 (V-35, T-50-73) and R5-I1 (V-48, T-50-83): a stop that reaches + /// stage N after safe point 3 (N is still `Advancing`, so it is flagged) + /// is forwarded by N to the successor it recorded, and N+1 stops and + /// records its own stop. Four runs, each a fresh child: (a) SIGTERM while + /// N is held at `before_record_save`; (b) at `after_publish`; (c) a + /// socket stop at `before_record_save`, whose caller is answered + /// `stopped`, not `handed_off`; (d) SIGTERM while N is held at + /// `before_handoff_mark`, after its advance returned and before it marks + /// itself handed off. + #[test] + fn a_stop_reaching_the_supervisor_after_safe_point_3_is_forwarded_to_the_next_supervisor() { + const NAME: &str = "pipeline_launch::tests::a_stop_reaching_the_supervisor_after_safe_point_3_is_forwarded_to_the_next_supervisor"; + if !devflow_core::test_support::in_child_test(NAME) { + for (label, hold) in [ + ("a", "before_record_save"), + ("b", "after_publish"), + ("c", "before_record_save"), + ("d", "before_handoff_mark"), + ] { + run_forward_child(NAME, hold, label); + } + return; + } + let _guard = env_lock(); + let label = std::env::var(FORWARD_RUN_ENV).expect("the parent names the run"); + let phase = match label.as_str() { + "a" => 2309, + "b" => 2310, + "c" => 2311, + _ => 2312, + }; + let run = HandoffRun::launch(PhaseId::new(phase)); + run.go(); + run.wait_held(); + if label == "c" { + let (accepted, last) = + shutdown_accepted_then(&run.n.socket_path, || run.release(), HANDOFF_STEP_WAIT); + assert_eq!(accepted["owner"], "advancing", "run {label}: {accepted}"); + assert_eq!( + last["stopped"], true, + "run {label}: N answers the forwarded stop `stopped`, so its caller does not \ + retry: {last}" + ); + } else { + signal_supervisor(&run, libc::SIGTERM); + // The control thread polls the signal pipe, so the stop is + // flagged well within this. + std::thread::sleep(Duration::from_millis(500)); + run.release(); + } + assert!( + poll_until(HANDOFF_STEP_WAIT, || probes_gone(&run.n.socket_path)), + "run {label}: N finishes: {}", + run.events() + ); + let next = run + .state() + .supervisor + .expect("the state names a supervisor"); + assert_ne!( + next.launch_id, run.n.launch_id, + "run {label}: the state names N+1" + ); + assert!( + poll_until(HANDOFF_STEP_WAIT, || probes_gone(&next.socket_path)), + "run {label}: N+1 was stopped through the forward: {}", + run.events() + ); + let state = run.state(); + assert!( + state.stopped, + "run {label}: N+1 recorded its stop (stop_reason {:?}): {}", + state.stop_reason, + run.events() + ); + assert_eq!( + state.stop_reason.as_deref(), + Some(SUPERVISOR_STOP_REASON), + "run {label}: N+1's own stop reason" + ); + assert!( + poll_until(Duration::from_secs(5), || !group_alive(next.agent_pgid)), + "run {label}: no agent of N+1 survives" + ); + let forwarded = events_named(&run.root, "stop_forwarded"); + assert_eq!(forwarded.len(), 1, "run {label}: {}", run.events()); + assert_eq!(forwarded[0]["confirmed"], true, "{}", forwarded[0]); + assert_eq!(forwarded[0]["launch_id"], next.launch_id.as_str()); + } + + /// The control for the forward test: the same `before_record_save` hold + /// and no stop leaves N+1 running and the run not stopped. + #[test] + fn without_a_stop_after_safe_point_3_the_next_supervisor_runs_on() { + const NAME: &str = + "pipeline_launch::tests::without_a_stop_after_safe_point_3_the_next_supervisor_runs_on"; + if !devflow_core::test_support::in_child_test(NAME) { + run_forward_child(NAME, "before_record_save", "control"); + return; + } + let _guard = env_lock(); + let run = HandoffRun::launch(PhaseId::new(2313)); + run.go(); + run.wait_held(); + run.release(); + assert!( + poll_until(HANDOFF_STEP_WAIT, || probes_gone(&run.n.socket_path)), + "control: N finishes: {}", + run.events() + ); + let state = run.state(); + let next = state + .supervisor + .clone() + .expect("control: the state names N+1"); + assert_ne!(next.launch_id, run.n.launch_id); + assert!( + !state.stopped, + "control: not stopped ({:?})", + state.stop_reason + ); + assert!(probes_alive(&next.socket_path), "control: N+1 runs on"); + assert!( + events_named(&run.root, "stop_forwarded").is_empty(), + "control: nothing was forwarded" + ); + let outcome = + devflow_core::supervisor::request_shutdown(&next.socket_path, SUPERVISOR_STOP_WAIT); + assert!( + matches!( + outcome, + Ok(devflow_core::supervisor::ShutdownOutcome::Stopped { .. }) + ), + "control: N+1 stops through its socket: {outcome:?}" + ); + } + + const ORDER_RUN_NOT_FOUND: &str = "run_monitor not found in the non-test code"; + const ORDER_NO_TAKE: &str = "run_monitor never calls take_successor"; + const ORDER_NO_HOLD: &str = "no before_handoff_mark hold after take_successor"; + const ORDER_NO_MARK: &str = "no handed-off marking after the before_handoff_mark hold"; + const ORDER_NO_CHECK: &str = "no stop_requested() check after the handed-off marking"; + + /// R5-I1 as a pure scan over `source`'s non-test, comment-stripped code: + /// in `run_monitor`, in this order, the first `take_successor(` line, a + /// later `test_hold("before_handoff_mark")`, a later + /// `OwnerPhase::HandedOff` and a later `stop_requested()`. Names the + /// missing function or the first step it cannot find. + fn handoff_order_violation(source: &str) -> Option { + let lines = non_test_code_lines(source); + let Some(body) = item_lines(&lines, "pub(crate) fn run_monitor(", "}") else { + return Some(ORDER_RUN_NOT_FOUND.to_string()); + }; + let steps = [ + ("take_successor(", ORDER_NO_TAKE), + ("test_hold(\"before_handoff_mark\")", ORDER_NO_HOLD), + ("OwnerPhase::HandedOff", ORDER_NO_MARK), + ("stop_requested()", ORDER_NO_CHECK), + ]; + let mut from = 0; + for (needle, missing) in steps { + match body[from..].iter().position(|line| line.contains(needle)) { + Some(at) => from += at + 1, + None => return Some(missing.to_string()), + } + } + None + } + + /// R5-I1 (permanent source scan, R6-G4): `run_monitor` checks for a stop + /// right after its handed-off marking, with the `before_handoff_mark` + /// hold just before that marking. Every planted source carries a decoy + /// marking before `take_successor(` and a decoy `stop_requested()` check + /// in another function. + #[test] + fn run_monitor_checks_for_a_stop_right_after_the_handed_off_marking() { + assert_eq!( + handoff_order_violation(include_str!("pipeline_launch.rs")), + None, + "pipeline_launch.rs" + ); + let take = " let next = take_successor();\n"; + let hold = " test_hold(\"before_handoff_mark\");\n"; + let mark = " token.enter(OwnerPhase::HandedOff);\n"; + let check = " if token.stop_requested() {}\n"; + let planted = |steps: &[&str]| { + format!( + "pub(crate) fn run_monitor(\n decoy.enter(OwnerPhase::HandedOff);\n{}}}\nfn other() {{\n if token.stop_requested() {{}}\n}}\n", + steps.concat() + ) + }; + let violation = |steps: &[&str]| handoff_order_violation(&planted(steps)); + let missing = |text: &str| Some(text.to_string()); + + assert_eq!( + violation(&[take, hold, mark, check]), + None, + "the planned layout" + ); + assert_eq!( + violation(&[take, check, hold, mark, check]), + None, + "the hint's check-plus-re-check layout" + ); + assert_eq!( + violation(&[take, check, mark]), + missing(ORDER_NO_HOLD), + "the pre-revision layout" + ); + assert_eq!( + violation(&[take, hold, check, mark]), + missing(ORDER_NO_CHECK), + "a hold whose only check precedes the marking (run (d)'s RED state)" + ); + assert_eq!( + violation(&[take, mark, hold, check]), + missing(ORDER_NO_MARK), + "a hold after the marking" + ); + assert_eq!( + violation(&[take, hold, mark, " // if token.stop_requested() {}\n"]), + missing(ORDER_NO_CHECK), + "a commented-out post-marking check" + ); + assert_eq!( + violation(&[take, hold, check]), + missing(ORDER_NO_MARK), + "no marking" + ); + assert_eq!( + handoff_order_violation(&format!("fn other() {{\n{take}{hold}{mark}{check}}}\n")), + missing(ORDER_RUN_NOT_FOUND), + "no run_monitor" + ); + } } From d0057104074b89d4224d42815f0fd8e7ae0112b6 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 10:05:47 -0400 Subject: [PATCH 078/112] feat(50-23): forward a stop that reaches N after safe point 3 to N+1 - run_monitor holds at before_handoff_mark (test only), marks itself handed off, then checks for a flagged stop (R5-I1 order) - forward_stop_to_successor shuts N+1 down through its socket; a confirmed stop takes a safe point so a parked caller hears stopped, an unconfirmed one names devflow stop --phase N --launch - both outcomes are recorded as a stop_forwarded event Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_launch.rs | 70 ++++++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index 4bfc3939..cb1ea23e 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -1246,8 +1246,20 @@ pub(crate) fn run_monitor(invocation: &MonitorInvocation, advance: bool) -> Resu let advanced = advance_for_stage(project_root, Some(phase), stage); // The advance has returned and released the per-phase lock. Only // a run whose advance published a successor is handed off. - if take_successor().is_some() { - run.supervisor.token().enter(OwnerPhase::HandedOff); + if let Some(successor) = take_successor() { + let token = run.supervisor.token(); + #[cfg(test)] + test_hold("before_handoff_mark"); + token.enter(OwnerPhase::HandedOff); + // RV-04 / R5-I1 (50-23): a stop that reached this supervisor + // after safe point 3 was flagged (it stayed `Advancing`) and + // must not be dropped. The one owner mutex orders every stop + // request before the marking above (flagged, so seen here) or + // after it (answered `handed_off`, acting on nothing), so one + // check right after the marking drops no flagged stop. + if token.stop_requested() { + forward_stop_to_successor(&token, project_root, phase, &successor); + } } drop(stop_guard); advanced @@ -1268,6 +1280,60 @@ pub(crate) fn run_monitor(invocation: &MonitorInvocation, advance: bool) -> Resu result } +/// Forward a stop that reached this supervisor after safe point 3 to the +/// successor its advance recorded (50-23, RV-04), through the successor's +/// socket, bounded by [`SUPERVISOR_STOP_WAIT`]. Writes no state: the +/// successor is the recorded supervisor and records its own stop. +/// +/// A confirmed `Stopped` takes a safe point on `token`, which returns +/// `StopHere`, so `finish` answers a parked shutdown caller `stopped` (the +/// stop did happen, through the successor) and that caller does not also +/// retry against the successor. Anything else leaves `finish` answering +/// `handed_off`, so a socket caller retries against the state's record, and +/// names the `devflow stop --phase N --launch ` repair. Both outcomes +/// are recorded as a `stop_forwarded` event, because this process's stderr +/// is `/dev/null` in production. The stop's source is not consulted: the +/// token does not keep it (T-50-75). +fn forward_stop_to_successor( + token: &StopToken, + project_root: &Path, + phase: PhaseId, + successor: &SupervisorRecord, +) { + let launch_id = &successor.launch_id; + let outcome = + devflow_core::supervisor::request_shutdown(&successor.socket_path, SUPERVISOR_STOP_WAIT); + let confirmed = matches!( + outcome, + Ok(devflow_core::supervisor::ShutdownOutcome::Stopped { .. }) + ); + let repair = format!("devflow stop --phase {phase} --launch {launch_id}"); + let detail = match &outcome { + Ok(outcome) => format!("{outcome:?}"), + Err(err) => err.to_string(), + }; + if confirmed { + let _ = token.safe_point(); + eprintln!("forwarded a stop received after safe point 3 to launch {launch_id}: stopped"); + } else { + eprintln!( + "a stop reached this supervisor after safe point 3 and forwarding it to launch \ + {launch_id} did not confirm a stop ({detail}); run `{repair}`" + ); + } + events::emit( + project_root, + phase, + "stop_forwarded", + serde_json::json!({ + "launch_id": launch_id, + "confirmed": confirmed, + "outcome": detail, + "repair": (!confirmed).then_some(repair), + }), + ); +} + /// The [`monitor::StageSpec`] for `invocation`: the stream shape reads its /// prompt file, and each `--keep-env` name takes its value from this /// process's environment. From 8545c1151f16984463f688db6ce61a6ba9947d7c Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 10:31:25 -0400 Subject: [PATCH 079/112] test(50-15): add failing tests for the launch guard over a corrupt state - V-23: start and resume over a live supervisor whose state will not load refuse, naming `devflow stop --phase N --launch `; the printed line ends the run (D-09 repair); control: no live supervisor proceeds - V-37 (RV-05): state and live sidecar both corrupt refuses, naming the socket, the sidecar problem and bare `devflow stop --phase N`, which ends the run; control: another root's socket does not refuse - unlistable socket dir fails closed (50-06 threat flag), missing dir control; STALE decision mapping (T-50-40) - monitor-arm fixtures now launch a real supervisor; golden and pinned fragments rewritten for the socket-naming refusal - RED scaffold: guard takes the cache dir; stale_group_may_live stub Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 122 ++- crates/devflow-cli/tests/start_lock_e2e.rs | 937 +++++++++++++++++---- 2 files changed, 883 insertions(+), 176 deletions(-) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 1d133987..2692a623 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -3184,7 +3184,12 @@ fn default_logs_phase(project_root: &Path) -> Result { /// Out of scope: the lock-free window between agent exit and `advance` taking /// the lock stays backlog 999.136. This check says nothing about a gate waiter. fn refuse_start_over_a_live_run(project_root: &Path, phase: PhaseId) -> Result<(), CliError> { - refuse_launch_over_a_live_run(project_root, phase, LaunchVerb::Start) + refuse_launch_over_a_live_run( + project_root, + phase, + LaunchVerb::Start, + registry::cache_dir().as_deref(), + ) } /// Criterion 2, D-01 and D-02's `resume` half, promoted by the operator on @@ -3212,7 +3217,12 @@ pub(crate) fn refuse_resume_over_a_live_run( project_root: &Path, phase: PhaseId, ) -> Result<(), CliError> { - refuse_launch_over_a_live_run(project_root, phase, LaunchVerb::Resume) + refuse_launch_over_a_live_run( + project_root, + phase, + LaunchVerb::Resume, + registry::cache_dir().as_deref(), + ) } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -3225,7 +3235,9 @@ fn refuse_launch_over_a_live_run( project_root: &Path, phase: PhaseId, verb: LaunchVerb, + cache_dir: Option<&Path>, ) -> Result<(), CliError> { + let _ = cache_dir; // RED scaffold: GREEN sweeps it. let loaded = workflow::load_state(project_root, phase); if loaded.is_err() && !workflow::state_path(project_root, phase).exists() { return Ok(()); @@ -3245,6 +3257,12 @@ fn refuse_launch_over_a_live_run( Err(CliError::Message(live_run_refusal(phase, verb, &live))) } +/// RED scaffold for the STALE classification; GREEN implements it. +#[cfg_attr(not(test), expect(dead_code, reason = "RED scaffold"))] +fn stale_group_may_live(_decision: &supervisor::backstop::BackstopDecision) -> bool { + true +} + /// The refusal text for [`refuse_start_over_a_live_run`]. It fires while /// `start` holds the lock, so no other process holds it: `stop` would signal /// nothing here (see `stop_via_lock`). @@ -10741,4 +10759,104 @@ mod tests { "nothing is recorded" ); } + + /// 50-15: the start/resume guard over a state file that will not load + /// (D-09). These drive the guard in-process against a tempdir cache, so + /// nothing reads the operator's real `~/.cache/devflow`. + mod launch_guard { + use super::*; + use supervisor::backstop::BackstopDecision; + + /// A project root whose phase state file exists and will not load. + fn root_with_an_unloadable_state(phase: PhaseId) -> tempfile::TempDir { + let root = tempfile::tempdir().unwrap(); + let state = workflow::state_path(root.path(), phase); + std::fs::create_dir_all(state.parent().unwrap()).unwrap(); + std::fs::write(&state, b"{ this does not parse").unwrap(); + assert!(workflow::load_state(root.path(), phase).is_err()); + root + } + + /// The 50-06 threat flag: `sweep_in` answers a socket directory it + /// cannot list with no entries, which must not read as "nothing + /// live" here. The guard fails closed and names the directory. + #[test] + fn the_launch_guard_refuses_an_unlistable_socket_dir_over_an_unloadable_state() { + use std::os::unix::fs::PermissionsExt as _; + let phase = PhaseId::new(1501); + let root = root_with_an_unloadable_state(phase); + let cache = tempfile::tempdir().unwrap(); + let sock = registry::socket_dir_in(cache.path()); + std::fs::create_dir_all(&sock).unwrap(); + std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o000)).unwrap(); + let listable = std::fs::read_dir(&sock).is_ok(); + let result = refuse_launch_over_a_live_run( + root.path(), + phase, + LaunchVerb::Start, + Some(cache.path()), + ); + std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o700)).unwrap(); + assert!( + !listable, + "precondition: a 0o000 directory cannot be listed" + ); + let Err(CliError::Message(text)) = result else { + panic!("an unlistable socket directory must refuse: {result:?}"); + }; + for fragment in [ + sock.display().to_string(), + "cannot be listed".to_string(), + "refusing to start; nothing was written".to_string(), + ] { + assert!(text.contains(&fragment), "missing {fragment:?}: {text}"); + } + } + + /// Control for the test above: a socket directory that does not + /// exist means no launch has published a socket, so nothing is live. + #[test] + fn the_launch_guard_proceeds_over_an_unloadable_state_when_the_socket_dir_is_missing() { + let phase = PhaseId::new(1502); + let root = root_with_an_unloadable_state(phase); + let cache = tempfile::tempdir().unwrap(); + assert!(!registry::socket_dir_in(cache.path()).exists()); + let result = refuse_launch_over_a_live_run( + root.path(), + phase, + LaunchVerb::Resume, + Some(cache.path()), + ); + assert!( + result.is_ok(), + "a missing socket dir is not a refusal: {result:?}" + ); + } + + /// T-50-40: only the three backstop decisions that prove a STALE + /// socket's agent group gone let a launch through; every refusal, + /// an unprovable identity and a live leader count as live. + #[test] + fn only_a_provably_gone_group_lets_a_stale_socket_through_the_launch_guard() { + for gone in [ + BackstopDecision::PreviousBoot, + BackstopDecision::EmptyGroup, + BackstopDecision::LeaderRecycled, + ] { + assert!( + !stale_group_may_live(&gone), + "{gone:?} proves the group gone" + ); + } + for live in [ + BackstopDecision::LeaderPresentSignal, + BackstopDecision::RefuseLeaderGone(Vec::new()), + BackstopDecision::RefuseMismatch("agent_pgid".to_string()), + BackstopDecision::RefuseUnsafeTarget, + BackstopDecision::RefuseIdentityUnavailable, + ] { + assert!(stale_group_may_live(&live), "{live:?} must fail closed"); + } + } + } } diff --git a/crates/devflow-cli/tests/start_lock_e2e.rs b/crates/devflow-cli/tests/start_lock_e2e.rs index 082000bf..345f9cb7 100644 --- a/crates/devflow-cli/tests/start_lock_e2e.rs +++ b/crates/devflow-cli/tests/start_lock_e2e.rs @@ -16,6 +16,7 @@ use devflow_core::mode::Mode; use devflow_core::phase_id::PhaseId; use devflow_core::stage::Stage; use devflow_core::state::{AgentKind, State}; +use devflow_core::supervisor::{self, Liveness, ShutdownOutcome}; use std::fs; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Output}; @@ -272,25 +273,130 @@ impl Drop for LiveProcess { } } +/// A fake `claude` that answers `--version` and otherwise sleeps until it is +/// ended, so a launched run stays live for the whole test. +const SLEEPING_CLAUDE: &str = "if [ \"$1\" = \"--version\" ]; then echo \"2.1.220 (Claude Code)\"; exit 0; fi\nexec sleep 120"; + +/// How long [`LiveRun`]'s drop waits for its supervisor's final shutdown line. +const LIVE_RUN_SHUTDOWN_WAIT: Duration = Duration::from_secs(15); + +/// A run a real `devflow start` launched (capture shape, supervise mode) with +/// a sleeping fake agent: its supervisor answers ALIVE on `socket`, and +/// `sidecar` is what it published, read before any test corrupts it. +/// +/// Dropping it ends the run through that socket, then SIGKILLs the agent +/// group and the supervisor if each is still the process the sidecar names +/// (`agent::is_same_process`), so a run whose state or sidecar a test +/// corrupted is still reaped. Bind a `LaunchedRunReaper` before +/// [`LiveRun::launch`] as the backstop for a panic inside it. +struct LiveRun { + socket: PathBuf, + sidecar: supervisor::Sidecar, + _fake_bin: common::FakeBin, +} + +impl LiveRun { + fn launch(home: &common::HermeticHome, root: &Path, phase: PhaseId) -> Self { + let fake_bin = common::fake_bin_dir(SLEEPING_CLAUDE); + let output = child(home, root, phase, &fake_bin, "supervise") + .env("DEVFLOW_GATE_TIMEOUT_SECS", GATE_TIMEOUT_SECS) + .output() + .expect("run devflow start for the live-run fixture"); + assert!( + output.status.success(), + "the live-run fixture's start failed\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let record = devflow_core::workflow::load_state(root, phase) + .expect("the fixture's start saved its state") + .supervisor + .expect("the fixture's start recorded state.supervisor"); + let sidecar = supervisor::read_sidecar(&record.socket_path.with_extension("json")) + .expect("read the fixture's sidecar"); + let run = Self { + socket: record.socket_path, + sidecar, + _fake_bin: fake_bin, + }; + assert_eq!(run.sidecar.launch_id, record.launch_id, "sidecar launch id"); + assert!( + matches!(supervisor::probe(&run.socket), Ok(Liveness::Alive(_))), + "the fixture's supervisor must answer ALIVE" + ); + let pid_file = devflow_core::agent_result::agent_pid_path(root, phase); + common::wait_for( + || pid_file.exists(), + Duration::from_secs(10), + "the fixture's agent pid file", + ); + run + } + + fn launch_id(&self) -> &str { + &self.sidecar.launch_id + } + + fn sidecar_path(&self) -> PathBuf { + self.socket.with_extension("json") + } + + /// Remove the agent pid file, so the guard's pid half sees nothing and + /// only the supervisor socket can name this run: 999.139's shape, where + /// the pid half let `start --force` launch a second agent. + fn hide_agent_pid_file(&self, root: &Path, phase: PhaseId) { + fs::remove_file(devflow_core::agent_result::agent_pid_path(root, phase)) + .expect("remove the fixture's agent pid file"); + } +} + +impl Drop for LiveRun { + fn drop(&mut self) { + // No `expect` here: a panic inside drop during unwinding aborts. + let _ = supervisor::request_shutdown(&self.socket, LIVE_RUN_SHUTDOWN_WAIT); + let group = (self.sidecar.agent_pgid, self.sidecar.agent_start_time); + if devflow_core::agent::agent_running(group.0) + && devflow_core::agent::is_same_process(group.0, group.1) + { + common::signal_group(group.0, libc::SIGKILL); + } + let monitor = ( + self.sidecar.supervisor_pid, + self.sidecar.supervisor_start_time, + ); + if devflow_core::agent::agent_running(monitor.0) + && devflow_core::agent::is_same_process(monitor.0, monitor.1) + && let Ok(pid) = libc::pid_t::try_from(monitor.0) + { + // SAFETY: one process, identity-checked against its start time. + unsafe { libc::kill(pid, libc::SIGKILL) }; + } + } +} + /// The planted Code gate files, returned so a control can assert they are gone. struct PlantedGate { request: PathBuf, response: PathBuf, } -fn plant_leftover_run( - root: &Path, - phase: PhaseId, - monitor_pid: Option, - agent_pid: Option, -) -> PlantedGate { +/// A leftover Code-stage state with no supervisor record, an optional agent +/// pid file, and a planted Code gate. The launch guard reads no +/// `monitor_pid` (50-15), so none is planted; a test that needs a live +/// supervisor launches one with [`LiveRun::launch`]. +fn plant_leftover_run(root: &Path, phase: PhaseId, agent_pid: Option) -> PlantedGate { let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); state.stage = Stage::Code; - state.monitor_pid = monitor_pid; devflow_core::workflow::save_state(&state).expect("plant leftover state"); if let Some(pid) = agent_pid { write_agent_pid(root, phase, pid); } + plant_code_gate(root, phase) +} + +/// A Code gate request and an aborting response, as a leftover run leaves +/// them. +fn plant_code_gate(root: &Path, phase: PhaseId) -> PlantedGate { let request = Gates::write_gate(root, phase, Stage::Code, "planted leftover Code gate") .expect("plant Code gate request"); let response = Gates::response_path(root, phase, Stage::Code); @@ -368,15 +474,75 @@ fn events_contain_workflow_started(root: &Path) -> bool { .contains("\"workflow_started\"") } -/// Drives `start` against a planted live run and asserts the refusal plus -/// the absence of every side effect. `refusal` is the first assertion's -/// message and names the arm. +/// Sentences the live-run refusals carried for the deleted shell monitor, or +/// that 50-14's `stop` made false. Each must be gone, not merely joined by +/// new text, from every arm of both the start and the resume refusal. +fn removed_refusal_sentences(phase: PhaseId) -> Vec { + vec![ + "would only mark the state stopped".to_string(), + "it signals nothing and does not end this run".to_string(), + "monitor pid".to_string(), + "an `sh -c` script".to_string(), + format!("`.devflow/phase-{}-`", phase.padded()), + "send SIGTERM to the confirmed monitor first".to_string(), + "signalling the agent first lets the monitor launch the next stage".to_string(), + "may be running `devflow advance` as a foreground child".to_string(), + "`ps -ww -o pid=,args= --ppid `".to_string(), + "once it takes the lock after this refusal".to_string(), + ] +} + +/// The fragments a refusal must carry for a live supervisor socket that +/// `stop --launch` can address: its launch id and liveness, its socket, and +/// the exact repair line (50-14's `--launch` form works whether or not the +/// state loads). +fn supervisor_arm_fragments(phase: PhaseId, run: &LiveRun) -> Vec { + let id = run.launch_id(); + vec![ + format!("supervisor launch {id} ALIVE"), + format!("socket {}", run.socket.display()), + format!("run `devflow stop --phase {phase} --launch {id}` first"), + "`devflow stop` ends a supervisor through its socket, whether or not the state file loads" + .to_string(), + "on a STALE socket it signals the agent group only after proving that group is this launch's" + .to_string(), + ] +} + +/// The fragments a refusal must carry for a live agent its pid file names. +/// Every sentence that keeps the operator from signalling the wrong process +/// or launching a second agent is pinned, not only the repair commands +/// (T-48-19-06, T-48-19-07). +fn agent_arm_fragments(phase: PhaseId, pid: u32) -> Vec { + vec![ + format!("agent pid {pid} alive"), + "`devflow stop` never signals a pid from the agent pid file".to_string(), + "Check each pid before signalling it".to_string(), + // Identity, not the bare executable name `ps -p` prints (48-REVIEW + // CR-01); `-ww` keeps a long `args` from being cut. + "`ps -p` shows only the executable name".to_string(), + format!("agent pid {pid}: ps -ww -o pid=,lstart=,args= -p {pid}"), + "`__monitor`".to_string(), + format!("`--phase {phase}`"), + "`ps -o ppid= -p `".to_string(), + "an agent whose supervisor has exited cannot be tied to this phase that way".to_string(), + "so do not signal it on the pid alone".to_string(), + "Do neither while this phase's processes are live".to_string(), + "a later `start` would launch a second agent beside it".to_string(), + ] +} + +/// Drives `start` against a live run and asserts the refusal plus the +/// absence of every side effect. `present` and `absent` are the arm's own +/// fragments; the verb's common fragments and every removed sentence are +/// checked here. `refusal` is the first assertion's message and names the +/// arm. fn assert_start_refuses_live_run( home: &common::HermeticHome, root: &Path, phase: PhaseId, extra: &[&str], - live_pid: u32, + (present, absent): (&[String], &[String]), refusal: &str, ) { let fake_bin = common::fake_bin_dir("exit 0"); @@ -395,6 +561,8 @@ fn assert_start_refuses_live_run( snapshot_paths(&before) ); } + let branches_before = phase_branches(root, phase); + let started_before = workflow_started_count(root); let output = run_start(home, root, phase, &fake_bin, extra); let stdout = String::from_utf8_lossy(&output.stdout); @@ -404,57 +572,29 @@ fn assert_start_refuses_live_run( "{refusal}\nstdout: {stdout}\nstderr: {stderr}" ); - for fragment in [ + let common = [ phase.to_string(), - live_pid.to_string(), "refusing to start; nothing was written".to_string(), - format!("devflow stop --phase {phase}"), - "would only mark the state stopped".to_string(), - // stop is not a no-op once this run's own advance holds the lock - // (48-REVIEW WR-02). - "once it takes the lock after this refusal".to_string(), - // Identity, not the bare executable name `ps -p` prints (48-REVIEW - // CR-01). `-ww`: at a terminal `args` is cut at the screen width, - // which drops the Legacy script's phase paths. - format!("ps -ww -o pid=,lstart=,args= -p {live_pid}"), - "`ps -p` shows only the executable name".to_string(), - format!("`--phase {phase}`"), - "`__monitor`".to_string(), - format!("`.devflow/phase-{}-`", phase.padded()), - "`ps -o ppid= -p `".to_string(), - // A Legacy monitor in its advance tail defers SIGTERM behind a - // foreground `devflow advance` child (48-REVIEW WR-02). - "may be running `devflow advance` as a foreground child".to_string(), - "`ps -ww -o pid=,args= --ppid `".to_string(), - "signal that child, not only the monitor".to_string(), - // A live recycled pid refuses start again; only recover --clean - // then start gets past it (48-REVIEW WR-01). - "If the named processes have exited, run `devflow start` again".to_string(), - format!("run `devflow recover --clean --phase {phase}` first, then `devflow start`"), - // Every sentence that keeps the operator from signalling the wrong - // process or launching a second agent is pinned, not only the - // repair commands (T-48-19-06, T-48-19-07): a text that drops these - // but keeps the commands must fail. - "Check each pid before signalling it".to_string(), - "so do not signal it on the pid alone".to_string(), - "send SIGTERM to the confirmed monitor first".to_string(), - "signalling the agent first lets the monitor launch the next stage".to_string(), - "If a named pid is live but is not this phase's process (a recycled pid), `start` refuses again" + "Once every supervisor and process named above is gone, run `devflow start` again." .to_string(), - "Do neither while this phase's processes are live".to_string(), - "a later `start` would launch a second agent beside it".to_string(), - ] { + ]; + for fragment in common.iter().chain(present) { assert!( - stderr.contains(&fragment), + stderr.contains(fragment.as_str()), "refusal must contain {fragment:?}\nstderr: {stderr}" ); } // The superseded guidance must be gone, not merely joined by new text. - for stale in [ - format!("ps -p {live_pid}"), - "or after `ps -p` shows they are no longer this phase's processes".to_string(), - "(stop does end a run that holds the lock, for example one parked at a gate.)".to_string(), - ] { + for stale in removed_refusal_sentences(phase) + .into_iter() + .chain([ + "ps -p ".to_string(), + "or after `ps -p` shows they are no longer this phase's processes".to_string(), + "(stop does end a run that holds the lock, for example one parked at a gate.)" + .to_string(), + ]) + .chain(absent.iter().cloned()) + { assert!( !stderr.contains(&stale), "refusal must not contain superseded guidance {stale:?}\nstderr: {stderr}" @@ -475,6 +615,20 @@ fn assert_start_refuses_live_run( "a refused start must not leave {}", lock.display() ); + assert_eq!( + phase_branches(root, phase), + branches_before, + "a refused start must not create or change the phase branch" + ); + assert_eq!( + workflow_started_count(root), + started_before, + "a refused start must not emit workflow_started" + ); +} + +/// `git branch --list feature/phase-`'s output. +fn phase_branches(root: &Path, phase: PhaseId) -> String { let branches = devflow_core::test_support::git_command(root) .args([ "branch", @@ -484,15 +638,15 @@ fn assert_start_refuses_live_run( .output() .expect("list phase branch"); assert!(branches.status.success()); - assert!( - String::from_utf8_lossy(&branches.stdout).trim().is_empty(), - "a refused start must not create the phase branch: {}", - String::from_utf8_lossy(&branches.stdout) - ); - assert!( - !events_contain_workflow_started(root), - "a refused start must not emit workflow_started" - ); + String::from_utf8_lossy(&branches.stdout).into_owned() +} + +/// How many `workflow_started` events the project's event log holds. +fn workflow_started_count(root: &Path) -> usize { + fs::read_to_string(devflow_core::events::events_path(root)) + .unwrap_or_default() + .matches("\"workflow_started\"") + .count() } #[test] @@ -502,16 +656,27 @@ fn start_refuses_a_phase_whose_recorded_monitor_is_alive() { let root = repo.path(); let phase = PhaseId::new(95); init_repo(root, phase); - let monitor = LiveProcess::spawn(); - plant_leftover_run(root, phase, Some(monitor.pid()), None); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); + let run = LiveRun::launch(&home, root, phase); + // Only the supervisor can refuse: the agent pid file is gone. + run.hide_agent_pid_file(root, phase); + plant_code_gate(root, phase); + assert_eq!(workflow_started_count(root), 1, "the live run's own start"); assert_start_refuses_live_run( &home, root, phase, &[], - monitor.pid(), - "start must refuse a phase whose recorded monitor is alive", + ( + &supervisor_arm_fragments(phase, &run), + &["agent pid".to_string()], + ), + "start must refuse a phase whose recorded supervisor is alive", + ); + assert!( + matches!(supervisor::probe(&run.socket), Ok(Liveness::Alive(_))), + "a refused start must leave the live supervisor running" ); } @@ -523,14 +688,28 @@ fn start_refuses_a_phase_whose_recorded_agent_is_alive() { let phase = PhaseId::new(96); init_repo(root, phase); let agent = LiveProcess::spawn(); - plant_leftover_run(root, phase, None, Some(agent.pid())); + plant_leftover_run(root, phase, Some(agent.pid())); + let mut present = agent_arm_fragments(phase, agent.pid()); + present.extend([ + // A live recycled pid refuses start again; only recover --clean + // then start gets past it (48-REVIEW WR-01). + "If a named pid is live but is not this phase's process (a recycled pid), `start` refuses again" + .to_string(), + format!("run `devflow recover --clean --phase {phase}` first, then `devflow start`"), + ]); assert_start_refuses_live_run( &home, root, phase, &["--force"], - agent.pid(), + ( + &present, + &[ + "supervisor launch".to_string(), + format!("devflow stop --phase {phase} --launch"), + ], + ), "start must refuse a phase whose recorded agent is alive", ); } @@ -542,7 +721,7 @@ fn start_replaces_a_leftover_state_whose_recorded_processes_are_dead() { let root = repo.path(); let phase = PhaseId::new(97); init_repo(root, phase); - let planted = plant_leftover_run(root, phase, Some(NEVER_LIVE_PID), Some(NEVER_LIVE_PID)); + let planted = plant_leftover_run(root, phase, Some(NEVER_LIVE_PID)); let fake_bin = common::fake_bin_dir("exit 0"); let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); @@ -604,13 +783,30 @@ fn start_proceeds_when_no_state_exists_even_if_the_agent_pid_file_names_a_live_p // 48-20 (criterion 2, D-01, D-02): `resume` must not overwrite a live run. // --------------------------------------------------------------------------- -const START_REFUSAL_GOLDEN_PHASE_94: &str = "phase 94: a run is still live (monitor pid {pid} alive) — refusing to start; nothing was written\n\ - No process holds the phase lock right now, so `devflow stop --phase 94` would only mark the state stopped: it signals nothing and does not end this run. stop ends a run through the process that holds the lock: one parked at a gate, or this run's own `devflow advance` once it takes the lock after this refusal.\n\ - Check each pid before signalling it. `ps -p` shows only the executable name, which cannot tell phases apart; `-ww` keeps a long command line from being cut at the terminal width:\n\ - \x20\x20monitor pid {pid}: ps -ww -o pid=,lstart=,args= -p {pid}\n\ - This phase's monitor is either a `__monitor` process with this project's path and `--phase 94` in its args, or an `sh -c` script that names `.devflow/phase-94-` files and ends in `advance` with `--phase 94`. This phase's agent has that monitor as its parent (`ps -o ppid= -p `); an agent whose monitor has exited cannot be tied to this phase that way, so do not signal it on the pid alone.\n\ - To end this run now, send SIGTERM to the confirmed monitor first — signalling the agent first lets the monitor launch the next stage. If the agent has already exited, an `sh` monitor may be running `devflow advance` as a foreground child: the monitor defers SIGTERM until that child returns, and the child may take the lock as soon as this start exits. Find it with `ps -ww -o pid=,args= --ppid ` and signal that child, not only the monitor.\n\ - If the named processes have exited, run `devflow start` again. If a named pid is live but is not this phase's process (a recycled pid), `start` refuses again: run `devflow recover --clean --phase 94` first, then `devflow start`. Do neither while this phase's processes are live: `recover --clean` clears state even while an agent runs, and a later `start` would launch a second agent beside it."; +/// The whole start refusal for a live run at phase 94 whose state loads: its +/// supervisor answers ALIVE and its agent pid file names its live agent. +/// `{launch}`, `{socket}` and `{pid}` are that run's. +const START_REFUSAL_GOLDEN_PHASE_94: &str = concat!( + "phase 94: a run is still live (supervisor launch {launch} ALIVE, agent pid {pid} alive) — ", + "refusing to start; nothing was written\n", + "`devflow stop` ends a supervisor through its socket, whether or not the state file loads; ", + "on a STALE socket it signals the agent group only after proving that group is this ", + "launch's. Run each line below first:\n", + " launch {launch} (ALIVE, socket {socket}): run `devflow stop --phase 94 --launch {launch}` first\n", + "The agent pid file names a live process, and `devflow stop` never signals a pid from the ", + "agent pid file. Check each pid before signalling it. `ps -p` shows only the executable name, ", + "which cannot tell phases apart; `-ww` keeps a long command line from being cut at the ", + "terminal width:\n", + " agent pid {pid}: ps -ww -o pid=,lstart=,args= -p {pid}\n", + "This phase's agent has its supervisor, a `__monitor` process with this project's path and ", + "`--phase 94` in its args, as its parent (`ps -o ppid= -p `); an agent whose supervisor ", + "has exited cannot be tied to this phase that way, so do not signal it on the pid alone. ", + "If a named pid is live but is not this phase's process (a recycled pid), `start` refuses ", + "again: run `devflow recover --clean --phase 94` first, then `devflow start`. Do neither ", + "while this phase's processes are live: `recover --clean` clears state even while an agent ", + "runs, and a later `start` would launch a second agent beside it.\n", + "Once every supervisor and process named above is gone, run `devflow start` again.", +); fn events_contain(root: &Path, event: &str) -> bool { fs::read_to_string(devflow_core::events::events_path(root)) @@ -618,13 +814,10 @@ fn events_contain(root: &Path, event: &str) -> bool { .contains(&format!("\"{event}\"")) } -fn plant_resumable_run( - root: &Path, - phase: PhaseId, - monitor_pid: Option, - agent_pid: Option, - stopped: bool, -) { +/// A resumable Code-stage state with no supervisor record and an optional +/// agent pid file. No `monitor_pid` is planted: the launch guard reads none +/// (50-15). +fn plant_resumable_run(root: &Path, phase: PhaseId, agent_pid: Option, stopped: bool) { let mut state = State::new( phase, AgentKind::Claude, @@ -632,7 +825,6 @@ fn plant_resumable_run( root.to_path_buf(), ); state.stage = Stage::Code; - state.monitor_pid = monitor_pid; state.legacy_claude_launch = true; state.stopped = stopped; state.stop_reason = @@ -669,10 +861,13 @@ fn resume_command( } struct RefusalCase { - role: &'static str, - live_pid: u32, required: Vec, - never_named: Vec, + /// The arm's own fragments. + present: Vec, + /// Text this arm must not contain. + absent: Vec, + /// A live agent pid the refusal must leave running. + spared_agent: Option, refusal: &'static str, } @@ -692,6 +887,7 @@ fn assert_resume_refuses_live_run( snapshot_paths(&before) ); } + let launched_before = stage_launched_count(root); let output = resume_command(home, root, phase, fake_bin) .output() @@ -705,55 +901,31 @@ fn assert_resume_refuses_live_run( ); let padded = phase.padded(); - for fragment in [ + let common = [ + "refusing to resume; nothing was written".to_string(), format!( - "a run is still live ({} pid {} alive) — refusing to resume; nothing was written", - case.role, case.live_pid + "Once every supervisor and process named above is gone, run `devflow resume --phase {phase}` again." ), - format!("`devflow stop --phase {phase}` would only mark the state stopped: it signals nothing and does not end this run"), - "once it takes the lock after this refusal".to_string(), - "A state already marked stopped does not mean these processes have exited".to_string(), - "`devflow status` and `devflow doctor` name `devflow resume` for a phase whose monitor is dead even while its agent is still alive".to_string(), - "resume refuses until every named process has exited".to_string(), - "Check each pid before signalling it".to_string(), - "`ps -p` shows only the executable name".to_string(), - format!("{} pid {}: ps -ww -o pid=,lstart=,args= -p {}", case.role, case.live_pid, case.live_pid), - "`__monitor`".to_string(), - format!("`--phase {phase}`"), - format!("`.devflow/phase-{padded}-`"), - "`ps -o ppid= -p `".to_string(), - "so do not signal it on the pid alone".to_string(), - "send SIGTERM to the confirmed monitor first".to_string(), - "signalling the agent first lets the monitor launch the next stage".to_string(), - "may be running `devflow advance` as a foreground child".to_string(), - "the child may take the lock as soon as this resume exits".to_string(), - "`ps -ww -o pid=,args= --ppid `".to_string(), - "signal that child, not only the monitor".to_string(), - format!("If the named processes have exited, run `devflow resume --phase {phase}` again"), - "If a named pid is live but is not this phase's process (a recycled pid), `resume` refuses again".to_string(), - "no DevFlow verb clears a recorded pid while keeping the phase's work".to_string(), - format!("`devflow recover --clean --phase {phase}` removes the phase state, after which `resume` has nothing to resume, and `devflow start` refuses while the phase's worktree or branch exists"), - "`devflow start --force` recreates them from the base and discards the phase's work".to_string(), - "Do neither while this phase's processes are live".to_string(), - "a later `start` would launch a second agent beside it".to_string(), - ] { + ]; + for fragment in common.iter().chain(&case.present) { assert!( - stderr.contains(&fragment), + stderr.contains(fragment.as_str()), "refusal must contain {fragment:?}\nstderr: {stderr}" ); } - for stale in [ - "refusing to start".to_string(), - "run `devflow start` again".to_string(), - "as soon as this start exits".to_string(), - format!("ps -p {}", case.live_pid), - ] - .into_iter() - .chain(case.never_named.iter().cloned()) + for stale in removed_refusal_sentences(phase) + .into_iter() + .chain([ + "refusing to start".to_string(), + "run `devflow start` again".to_string(), + "as soon as this start exits".to_string(), + "ps -p ".to_string(), + ]) + .chain(case.absent.iter().cloned()) { assert!( !stderr.contains(&stale), - "refusal must not contain start-only or dead-pid text {stale:?}\nstderr: {stderr}" + "refusal must not contain start-only or superseded text {stale:?}\nstderr: {stderr}" ); } @@ -766,15 +938,38 @@ fn assert_resume_refuses_live_run( !root.join(format!(".devflow/lock-{padded}")).exists(), "a refused resume must not leave a phase lock" ); - assert!( - !events_contain(root, "stage_launched"), + assert_eq!( + stage_launched_count(root), + launched_before, "a refused resume must not emit stage_launched" ); - assert!( - devflow_core::agent::agent_running(case.live_pid), - "a refused resume must not signal the recorded {}", - case.role - ); + if let Some(pid) = case.spared_agent { + assert!( + devflow_core::agent::agent_running(pid), + "a refused resume must not signal the recorded agent {pid}" + ); + } +} + +/// How many `stage_launched` events the project's event log holds. +fn stage_launched_count(root: &Path) -> usize { + fs::read_to_string(devflow_core::events::events_path(root)) + .unwrap_or_default() + .matches("\"stage_launched\"") + .count() +} + +/// The resume agent arm's fragments beyond [`agent_arm_fragments`]. +fn resume_agent_fragments(phase: PhaseId) -> Vec { + vec![ + "A state already marked stopped does not mean these processes have exited".to_string(), + "`devflow status` names `devflow resume` for a phase whose supervisor is GONE even while its agent is still alive".to_string(), + "resume refuses until every named process has exited".to_string(), + "If a named pid is live but is not this phase's process (a recycled pid), `resume` refuses again".to_string(), + "no DevFlow verb clears a recorded pid while keeping the phase's work".to_string(), + format!("`devflow recover --clean --phase {phase}` removes the phase state, after which `resume` has nothing to resume, and `devflow start` refuses while the phase's worktree or branch exists"), + "`devflow start --force` recreates them from the base and discards the phase's work".to_string(), + ] } #[test] @@ -784,16 +979,11 @@ fn resume_refuses_a_phase_whose_recorded_monitor_is_alive() { let root = repo.path(); let phase = PhaseId::new(85); init_repo(root, phase); - let monitor = LiveProcess::spawn(); let fake_bin = common::fake_bin_dir("exec sleep 60"); - let _reaper = common::LaunchedRunReaper::new(root, phase, vec![monitor.pid()]); - plant_resumable_run( - root, - phase, - Some(monitor.pid()), - Some(NEVER_LIVE_PID), - false, - ); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); + let run = LiveRun::launch(&home, root, phase); + // Only the supervisor can refuse: the agent pid file is gone. + run.hide_agent_pid_file(root, phase); assert_resume_refuses_live_run( &home, @@ -801,16 +991,17 @@ fn resume_refuses_a_phase_whose_recorded_monitor_is_alive() { phase, &fake_bin, &RefusalCase { - role: "monitor", - live_pid: monitor.pid(), - required: vec![ - devflow_core::workflow::state_path(root, phase), - devflow_core::agent_result::agent_pid_path(root, phase), - ], - never_named: vec![format!("agent pid {NEVER_LIVE_PID}")], - refusal: "resume must refuse a phase whose recorded monitor is alive", + required: vec![devflow_core::workflow::state_path(root, phase)], + present: supervisor_arm_fragments(phase, &run), + absent: vec!["agent pid".to_string()], + spared_agent: None, + refusal: "resume must refuse a phase whose recorded supervisor is alive", }, ); + assert!( + matches!(supervisor::probe(&run.socket), Ok(Liveness::Alive(_))), + "a refused resume must leave the live supervisor running" + ); } #[test] @@ -823,15 +1014,20 @@ fn resume_refuses_a_stopped_phase_whose_recorded_agent_is_alive() { let agent = LiveProcess::spawn(); let fake_bin = common::fake_bin_dir("exec sleep 60"); let _reaper = common::LaunchedRunReaper::new(root, phase, vec![agent.pid()]); - plant_resumable_run(root, phase, Some(NEVER_LIVE_PID), Some(agent.pid()), true); + plant_resumable_run(root, phase, Some(agent.pid()), true); + let mut present = agent_arm_fragments(phase, agent.pid()); + present.extend(resume_agent_fragments(phase)); let case = RefusalCase { - role: "agent", - live_pid: agent.pid(), required: vec![ devflow_core::workflow::state_path(root, phase), devflow_core::agent_result::agent_pid_path(root, phase), ], - never_named: vec![format!("monitor pid {NEVER_LIVE_PID}")], + present, + absent: vec![ + "supervisor launch".to_string(), + format!("devflow stop --phase {phase} --launch"), + ], + spared_agent: Some(agent.pid()), refusal: "resume must refuse a phase whose recorded agent is alive", }; @@ -854,13 +1050,7 @@ fn resume_relaunches_a_phase_whose_recorded_processes_are_dead() { init_repo(root, phase); let fake_bin = common::fake_bin_dir("exec sleep 60"); let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); - plant_resumable_run( - root, - phase, - Some(NEVER_LIVE_PID), - Some(NEVER_LIVE_PID), - stopped, - ); + plant_resumable_run(root, phase, Some(NEVER_LIVE_PID), stopped); let output = resume_command(&home, root, phase, &fake_bin) .output() @@ -873,7 +1063,10 @@ fn resume_relaunches_a_phase_whose_recorded_processes_are_dead() { ); let state = devflow_core::workflow::load_state(root, phase).expect("load relaunched state"); assert!(!state.stopped, "resume must clear the stop mark"); - assert_ne!(state.monitor_pid, Some(NEVER_LIVE_PID)); + assert!( + state.supervisor.is_some(), + "the relaunch must record its supervisor: {state:?}" + ); assert!( events_contain(root, "stage_launched"), "a resumed dead leftover must emit stage_launched" @@ -922,35 +1115,431 @@ fn start_refusal_text_stays_byte_identical_for_a_live_monitor() { let root = repo.path(); let phase = PhaseId::new(94); init_repo(root, phase); - let monitor = LiveProcess::spawn(); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); + let run = LiveRun::launch(&home, root, phase); + let agent = run.sidecar.agent_pid; let fake_bin = common::fake_bin_dir("exit 0"); - plant_leftover_run(root, phase, Some(monitor.pid()), None); let output = run_start(&home, root, phase, &fake_bin, &[]); let stderr = String::from_utf8_lossy(&output.stderr); assert!( !output.status.success(), - "start must still refuse a phase whose recorded monitor is alive\nstderr: {stderr}" + "start must still refuse a phase whose recorded supervisor is alive\nstderr: {stderr}" ); let expected = format!( "error: {}\n", - START_REFUSAL_GOLDEN_PHASE_94.replace("{pid}", &monitor.pid().to_string()) + START_REFUSAL_GOLDEN_PHASE_94 + .replace("{launch}", run.launch_id()) + .replace("{socket}", &run.socket.display().to_string()) + .replace("{pid}", &agent.to_string()) ); assert!( stderr.contains(&expected), - "the start refusal must stay byte-identical to 48-19's text\nstderr: {stderr}" + "the start refusal must match the golden byte for byte\nexpected: {expected}\nstderr: {stderr}" ); assert!( !stderr.contains(&expected.replacen("refusing to start", "refusing to resume", 1)), "the golden comparison must be sensitive to the launch verb\nstderr: {stderr}" ); + for placeholder in ["{launch}", "{socket}", "{pid}"] { + assert!( + START_REFUSAL_GOLDEN_PHASE_94.contains(placeholder), + "the golden must carry {placeholder}, or the substitution proves nothing" + ); + } + assert!(START_REFUSAL_GOLDEN_PHASE_94.ends_with( + "Once every supervisor and process named above is gone, run `devflow start` again." + )); +} + +// --------------------------------------------------------------------------- +// 50-15 (D-09, R-12, 999.139): a state file that will not load must not hide +// a live run from `start` or `resume`. Each fixture is a real launched run +// whose agent pid file is removed, so only its supervisor socket can refuse. +// --------------------------------------------------------------------------- + +/// Bytes no `State` deserializes. +const CORRUPT_JSON: &[u8] = b"{ this does not parse"; + +/// Overwrite `phase`'s state file with invalid JSON; returns the bytes. +fn corrupt_state(root: &Path, phase: PhaseId) -> Vec { + let path = devflow_core::workflow::state_path(root, phase); + fs::write(&path, CORRUPT_JSON).expect("corrupt the state file"); assert!( - START_REFUSAL_GOLDEN_PHASE_94.starts_with( - "phase 94: a run is still live (monitor pid {pid} alive) — refusing to start; nothing was written" - ) + devflow_core::workflow::load_state(root, phase).is_err(), + "the corrupted state must not load" ); + CORRUPT_JSON.to_vec() +} + +/// Every `.sock` file under `home`'s socket directory, sorted. +fn sockets_under(home: &common::HermeticHome) -> Vec { + let mut sockets: Vec = fs::read_dir(home.cache_dir().join("sock")) + .map(|entries| { + entries + .map(|entry| entry.expect("read sock dir entry").path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "sock")) + .collect() + }) + .unwrap_or_default(); + sockets.sort(); + sockets +} + +/// `devflow start --force` for `phase`, as the 48-19 refusal tests run it. +fn force_start(home: &common::HermeticHome, root: &Path, phase: PhaseId) -> Output { + let fake_bin = common::fake_bin_dir("exit 0"); + run_start(home, root, phase, &fake_bin, &["--force"]) +} + +/// Run `launch` (a `start` or `resume` over a corrupt state) and assert it is +/// refused with no side effect: no new socket, the corrupt state bytes +/// unchanged, no `workflow_started` or `stage_launched` appended. Returns +/// its stderr. +fn assert_refused_over_a_corrupt_state( + home: &common::HermeticHome, + root: &Path, + phase: PhaseId, + launch: impl FnOnce() -> Output, +) -> String { + let state_file = devflow_core::workflow::state_path(root, phase); + let state_before = fs::read(&state_file).expect("read the corrupt state"); + let sockets_before = sockets_under(home); + let started_before = workflow_started_count(root); + let launched_before = stage_launched_count(root); + + let output = launch(); + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); assert!( - START_REFUSAL_GOLDEN_PHASE_94 - .ends_with("a later `start` would launch a second agent beside it.") + !output.status.success(), + "a launch over a live run whose state will not load must refuse\nstdout: {}\nstderr: {stderr}", + String::from_utf8_lossy(&output.stdout) + ); + assert_eq!( + sockets_under(home), + sockets_before, + "a refused launch must publish no second socket\nstderr: {stderr}" + ); + assert_eq!( + fs::read(&state_file).expect("read the state after the refusal"), + state_before, + "a refused launch must leave the corrupt state bytes unchanged" + ); + assert_eq!( + workflow_started_count(root), + started_before, + "a refused launch must not emit workflow_started" + ); + assert_eq!( + stage_launched_count(root), + launched_before, + "a refused launch must not emit stage_launched" + ); + stderr +} + +/// The backticked `devflow stop --phase …` command in `stderr`, split into +/// its words without the leading `devflow`. +fn printed_stop_command(stderr: &str, phase: PhaseId) -> Vec { + let head = format!("`devflow stop --phase {phase}"); + let at = stderr + .find(&head) + .unwrap_or_else(|| panic!("no {head}… command in the refusal\nstderr: {stderr}")); + let command = &stderr[at + 1..]; + let end = command + .find('`') + .expect("the printed command closes its backtick"); + command[..end] + .split_whitespace() + .skip(1) + .map(str::to_string) + .collect() +} + +/// Run a printed `devflow stop` command exactly as printed, from `root`. +fn run_printed_stop(home: &common::HermeticHome, root: &Path, words: &[String]) -> Output { + common::hermetic_devflow(home) + .args(words) + .current_dir(root) + .output() + .expect("run the printed devflow stop") +} + +/// Whether process group `pgid` has no member left (`kill(-pgid, 0)` fails +/// with ESRCH). +fn group_is_gone(pgid: u32) -> bool { + let pgid = libc::pid_t::try_from(pgid).expect("pgid fits pid_t"); + // SAFETY: signal 0 only checks for existence; nothing is delivered. + let status = unsafe { libc::kill(-pgid, 0) }; + status == -1 && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) +} + +#[test] +fn start_refuses_over_a_live_supervisor_when_the_state_will_not_load() { + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path(); + let phase = PhaseId::new(70); + init_repo(root, phase); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); + let run = LiveRun::launch(&home, root, phase); + run.hide_agent_pid_file(root, phase); + corrupt_state(root, phase); + + let stderr = + assert_refused_over_a_corrupt_state(&home, root, phase, || force_start(&home, root, phase)); + let id = run.launch_id(); + for fragment in [ + "refusing to start; nothing was written".to_string(), + format!("supervisor launch {id} ALIVE"), + format!("run `devflow stop --phase {phase} --launch {id}` first"), + ] { + assert!( + stderr.contains(&fragment), + "refusal must contain {fragment:?}\nstderr: {stderr}" + ); + } + assert!( + !stderr.contains("agent pid"), + "only the socket may refuse here (the pid file is gone)\nstderr: {stderr}" + ); + assert!( + matches!(supervisor::probe(&run.socket), Ok(Liveness::Alive(_))), + "a refused start must leave the live supervisor running" + ); +} + +#[test] +fn the_corrupt_state_refusal_names_a_stop_that_ends_the_run() { + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path(); + let phase = PhaseId::new(71); + init_repo(root, phase); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); + let run = LiveRun::launch(&home, root, phase); + run.hide_agent_pid_file(root, phase); + let corrupt = corrupt_state(root, phase); + let stderr = + assert_refused_over_a_corrupt_state(&home, root, phase, || force_start(&home, root, phase)); + let words = printed_stop_command(&stderr, phase); + assert_eq!( + words, + [ + "stop", + "--phase", + &phase.to_string(), + "--launch", + run.launch_id() + ], + "the refusal must print the `--launch` form\nstderr: {stderr}" + ); + + let stop = run_printed_stop(&home, root, &words); + assert!( + stop.status.success(), + "the printed stop must end the run\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&stop.stdout), + String::from_utf8_lossy(&stop.stderr) + ); + common::wait_for( + || matches!(supervisor::probe(&run.socket), Ok(Liveness::Gone)), + Duration::from_secs(5), + "the stopped supervisor's socket to probe Gone", + ); + common::wait_for( + || group_is_gone(run.sidecar.agent_pgid), + Duration::from_secs(5), + "the stopped run's agent group to be gone", + ); + assert_eq!( + fs::read(devflow_core::workflow::state_path(root, phase)).unwrap(), + corrupt, + "stop must leave the corrupt state bytes unchanged" + ); + + let again = force_start(&home, root, phase); + let again_stderr = String::from_utf8_lossy(&again.stderr); + assert!( + !again_stderr.contains("a run is still live"), + "once the run is stopped, start must not refuse over it\nstderr: {again_stderr}" ); } + +#[test] +fn resume_refuses_over_a_live_supervisor_when_the_state_will_not_load() { + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path(); + let phase = PhaseId::new(72); + init_repo(root, phase); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); + let run = LiveRun::launch(&home, root, phase); + run.hide_agent_pid_file(root, phase); + corrupt_state(root, phase); + let fake_bin = common::fake_bin_dir("exec sleep 60"); + + let stderr = assert_refused_over_a_corrupt_state(&home, root, phase, || { + resume_command(&home, root, phase, &fake_bin) + .output() + .expect("run devflow resume") + }); + let id = run.launch_id(); + for fragment in [ + "refusing to resume; nothing was written".to_string(), + format!("supervisor launch {id} ALIVE"), + format!("run `devflow stop --phase {phase} --launch {id}` first"), + ] { + assert!( + stderr.contains(&fragment), + "refusal must contain {fragment:?}\nstderr: {stderr}" + ); + } + assert!( + matches!(supervisor::probe(&run.socket), Ok(Liveness::Alive(_))), + "a refused resume must leave the live supervisor running" + ); +} + +#[test] +fn start_proceeds_over_a_corrupt_state_with_no_live_supervisor() { + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path(); + let phase = PhaseId::new(73); + init_repo(root, phase); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); + let run = LiveRun::launch(&home, root, phase); + run.hide_agent_pid_file(root, phase); + match supervisor::request_shutdown(&run.socket, LIVE_RUN_SHUTDOWN_WAIT) + .expect("shut the fixture's run down") + { + ShutdownOutcome::Stopped { .. } => {} + other => panic!("the fixture's shutdown must stop the run, got {other:?}"), + } + common::wait_for( + || !run.socket.exists() && !run.sidecar_path().exists(), + Duration::from_secs(5), + "the shut-down run's socket and sidecar to be removed", + ); + corrupt_state(root, phase); + let started_before = workflow_started_count(root); + + let output = force_start(&home, root, phase); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "with no live supervisor, start over a corrupt state proceeds as today\nstdout: {}\nstderr: {stderr}", + String::from_utf8_lossy(&output.stdout) + ); + assert!( + !stderr.contains("a run is still live"), + "no refusal over a dead run\nstderr: {stderr}" + ); + assert_eq!( + workflow_started_count(root), + started_before + 1, + "the start over the corrupt state must emit workflow_started" + ); +} + +#[test] +fn start_refuses_when_the_state_and_the_live_sidecar_both_will_not_load() { + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path(); + let phase = PhaseId::new(74); + init_repo(root, phase); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); + let run = LiveRun::launch(&home, root, phase); + run.hide_agent_pid_file(root, phase); + let (pgid, started) = (run.sidecar.agent_pgid, run.sidecar.agent_start_time); + corrupt_state(root, phase); + fs::write(run.sidecar_path(), CORRUPT_JSON).expect("corrupt the live sidecar"); + assert!(supervisor::read_sidecar(&run.sidecar_path()).is_err()); + + let stderr = + assert_refused_over_a_corrupt_state(&home, root, phase, || force_start(&home, root, phase)); + for fragment in [ + "refusing to start; nothing was written".to_string(), + format!("socket {} (ALIVE", run.socket.display()), + format!("launch {}", run.launch_id()), + "does not parse".to_string(), + format!("run `devflow stop --phase {phase}` first"), + "one that does not answer cannot be validated without its sidecar".to_string(), + ] { + assert!( + stderr.contains(&fragment), + "refusal must contain {fragment:?}\nstderr: {stderr}" + ); + } + assert!( + !stderr.contains("--launch"), + "a socket with no usable sidecar is not addressed by `--launch`\nstderr: {stderr}" + ); + let words = printed_stop_command(&stderr, phase); + assert_eq!(words, ["stop", "--phase", &phase.to_string()]); + + assert!( + devflow_core::agent::is_same_process(pgid, started), + "the agent group's leader must still be the launched agent before the stop" + ); + let stop = run_printed_stop(&home, root, &words); + assert!( + stop.status.success(), + "the printed stop must end the run\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&stop.stdout), + String::from_utf8_lossy(&stop.stderr) + ); + common::wait_for( + || matches!(supervisor::probe(&run.socket), Ok(Liveness::Gone)), + Duration::from_secs(5), + "the stopped supervisor's socket to probe Gone", + ); + common::wait_for( + || group_is_gone(pgid), + Duration::from_secs(5), + "the stopped run's agent group to be gone", + ); +} + +#[test] +fn a_live_socket_named_for_another_root_does_not_block_a_corrupt_state_start() { + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path(); + let other_repo = tempfile::tempdir().unwrap(); + let other = other_repo.path(); + let phase = PhaseId::new(75); + init_repo(root, phase); + init_repo(other, phase); + let _reaper = common::LaunchedRunReaper::new(root, phase, vec![]); + let _other_reaper = common::LaunchedRunReaper::new(other, phase, vec![]); + let run = LiveRun::launch(&home, root, phase); + run.hide_agent_pid_file(root, phase); + corrupt_state(root, phase); + fs::write(run.sidecar_path(), CORRUPT_JSON).expect("corrupt the live sidecar"); + fs::create_dir_all(other.join(".devflow")).unwrap(); + corrupt_state(other, phase); + + let output = force_start(&home, other, phase); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !stderr.contains("devflow stop --phase"), + "a socket named for another root must not refuse this root's start\nstderr: {stderr}" + ); + assert!( + !stderr.contains("a run is still live"), + "no live-run refusal for another root's socket\nstderr: {stderr}" + ); + assert!( + matches!(supervisor::probe(&run.socket), Ok(Liveness::Alive(_))), + "the other root's start must leave this root's supervisor running" + ); + match supervisor::request_shutdown(&run.socket, LIVE_RUN_SHUTDOWN_WAIT) + .expect("shut this root's run down") + { + ShutdownOutcome::Stopped { .. } => {} + other => panic!("the shutdown must stop the run, got {other:?}"), + } +} From f5989de12a02b3dc058649bf7c3e0573de6025da Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 10:36:08 -0400 Subject: [PATCH 080/112] feat(50-15): find a live run through its socket when the state will not load - refuse_launch_over_a_live_run reads no monitor_pid: a loaded state's supervisor record is probed (STALE classified read-only through backstop::assess, a probe error refuses); an unloadable state sweeps this root and phase with sweep_for_in (D-09, R-12, 999.139) - a swept socket with no usable sidecar refuses unless GONE and names bare `devflow stop --phase N` (RV-05); every other socket names its `devflow stop --phase N --launch ` line (50-14) - an unlistable socket directory refuses, naming it (50-06 threat flag) - the refusal drops the deleted shell-monitor sentences and the claim that stop signals nothing; the agent pid half is unchanged (D-12) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 432 ++++++++++++++++++++--------- 1 file changed, 300 insertions(+), 132 deletions(-) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 2692a623..0b490b3d 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -3165,21 +3165,16 @@ fn default_logs_phase(project_root: &Path) -> Result { }) } -/// Refuse `start` while the phase's recorded monitor or agent is alive -/// (criterion 2, D-01: one writer per phase). `start` calls this under the -/// per-phase lock and before any gate cleanup, git mutation or `save_state`, -/// so a refusal writes nothing. `--force` does not bypass it. +/// Refuse `start` while the phase's run is live (criterion 2, D-01: one +/// writer per phase). `start` calls this under the per-phase lock and before +/// any gate cleanup, git mutation or `save_state`, so a refusal writes +/// nothing. `--force` does not bypass it. See +/// [`refuse_launch_over_a_live_run`] for what counts as live. /// -/// Liveness is pid liveness through the helpers `status` uses -/// (`agent_pid_from_file`, `agent::agent_running`), not identity: neither pid -/// records a start time. A recycled pid gives a fail-closed false refusal, -/// accepted by the operator with Option A (2026-09-22). -/// -/// No-state carve-out (operator-approved 2026-09-22): without a state file this -/// returns Ok even if the agent pid file names a live process, because -/// `recover --clean` leaves that file behind and a recycled pid would otherwise -/// block `start` with no DevFlow repair. A state file that does not load is -/// checked on the agent pid file only. +/// No-state carve-out (operator-approved 2026-09-22, unchanged by D-12): +/// without a state file this returns Ok even if the agent pid file names a +/// live process, because `recover --clean` leaves that file behind and a +/// recycled pid would otherwise block `start` with no DevFlow repair. /// /// Out of scope: the lock-free window between agent exit and `advance` taking /// the lock stays backlog 999.136. This check says nothing about a gate waiter. @@ -3196,23 +3191,14 @@ fn refuse_start_over_a_live_run(project_root: &Path, phase: PhaseId) -> Result<( /// 2026-09-23 as backlog 999.140: `pipeline_launch::resume` calls this just /// after it has acquired its phase lock and before it loads or writes state. /// The `devflow resume` CLI arm is resume's only production caller; no monitor -/// or advance path calls resume, so a recorded pid is never that process. An -/// agent that runs `devflow resume` for its own phase is refused correctly. -/// -/// Liveness is shared with `start` and uses the same pid-only Option A that -/// the operator accepted for start. A recycled pid can therefore refuse a -/// resume; that distinct resume cost is the known limit recorded in 48-20, -/// not an operator-approved permanent limit. Without a state file, resume -/// still reports its own missing-state error. For an unloadable state, this -/// checks only the agent pid file (999.139, backlog), and resume then cannot -/// launch because its own `load_state` fails immediately afterwards. +/// or advance path calls resume. An agent that runs `devflow resume` for its +/// own phase is refused correctly. /// -/// Production hints remain unchanged in this plan (backlog 999.142): status -/// and doctor for a dead monitor with a live agent, doctor's dead-agent and -/// gate-pending findings, cleanup's worktree messages, and `no_waiter_repair` -/// through the gate verbs, gate sweep, stop and doctor (48-20 H1-H10). The -/// agent-exit-to-advance window remains 999.136, and this check says nothing -/// about a gate waiter. +/// Liveness is shared with `start` ([`refuse_launch_over_a_live_run`]). +/// Without a state file, resume still reports its own missing-state error. +/// For a state that will not load, the guard finds the run through its +/// sidecar (R-12, 999.139), and resume then cannot launch anyway because its +/// own `load_state` fails immediately afterwards. pub(crate) fn refuse_resume_over_a_live_run( project_root: &Path, phase: PhaseId, @@ -3231,129 +3217,311 @@ enum LaunchVerb { Resume, } +impl LaunchVerb { + fn label(self) -> &'static str { + match self { + LaunchVerb::Start => "start", + LaunchVerb::Resume => "resume", + } + } + + /// The command a refusal tells the operator to run again. + fn rerun(self, phase: PhaseId) -> String { + match self { + LaunchVerb::Start => "devflow start".to_string(), + LaunchVerb::Resume => format!("devflow resume --phase {phase}"), + } + } +} + +/// The shared start/resume guard (D-09, R-12, 999.139). Two halves, either of +/// which refuses: +/// +/// - **Supervisor.** The state loads → probe its `supervisor` record: ALIVE +/// is live; STALE is live unless [`supervisor::backstop::assess`] proves +/// its agent group gone ([`stale_group_may_live`]); a probe error is live +/// (fail closed, T-50-40); GONE is not. The state exists but will not load +/// → [`supervisor::sweep_for_in`] for this root and phase: an entry whose +/// sidecar parsed is classified the same way (with no record); an entry +/// with no usable sidecar, which 50-06 attributes by its file name or ping +/// reply, is live unless it probes GONE (RV-05: nothing recorded can prove +/// its group dead). A socket directory that exists but cannot be listed +/// refuses (the 50-06 threat flag: the sweep reads it as empty). With no +/// cache dir, no sweep runs, as for `stop` and `recover`. +/// - **Agent pid file** (unchanged, D-12): a live pid there is live. Pid +/// liveness, not identity: a recycled pid gives a fail-closed false +/// refusal (Option A, 2026-09-22). +/// +/// No `monitor_pid` is read. A loaded state with no record over a live +/// supervisor (a launcher that died before saving it, T-50-90/T-50-91) is +/// accepted until Phase 50.1. fn refuse_launch_over_a_live_run( project_root: &Path, phase: PhaseId, verb: LaunchVerb, cache_dir: Option<&Path>, ) -> Result<(), CliError> { - let _ = cache_dir; // RED scaffold: GREEN sweeps it. - let loaded = workflow::load_state(project_root, phase); - if loaded.is_err() && !workflow::state_path(project_root, phase).exists() { - return Ok(()); - } - let monitor = loaded - .ok() - .and_then(|state| state.monitor_pid) - .filter(|&pid| agent::agent_running(pid)); + let state_file = workflow::state_path(project_root, phase); + let sockets: Vec = match workflow::load_state(project_root, phase) { + Ok(state) => state + .supervisor + .and_then(|record| recorded_live_socket(&record)) + .into_iter() + .collect(), + Err(_) if !state_file.exists() => return Ok(()), + Err(_) => match cache_dir { + None => Vec::new(), + Some(cache_dir) => { + if let Some(why) = socket_dir_unlistable(cache_dir) { + let dir = registry::socket_dir_in(cache_dir); + return Err(unlistable_socket_dir_refusal( + phase, + verb, + (&state_file, &dir), + &why, + )); + } + supervisor::sweep_for_in(cache_dir, project_root, phase) + .iter() + .filter_map(swept_live_socket) + .collect() + } + }, + }; let agent = agent_pid_from_file(project_root, phase).filter(|&pid| agent::agent_running(pid)); - let live: Vec<(&str, u32)> = [("monitor", monitor), ("agent", agent)] - .into_iter() - .filter_map(|(role, pid)| pid.map(|pid| (role, pid))) - .collect(); - if live.is_empty() { + if sockets.is_empty() && agent.is_none() { return Ok(()); } - Err(CliError::Message(live_run_refusal(phase, verb, &live))) + Err(CliError::Message(live_run_refusal( + phase, verb, &sockets, agent, + ))) } -/// RED scaffold for the STALE classification; GREEN implements it. -#[cfg_attr(not(test), expect(dead_code, reason = "RED scaffold"))] -fn stale_group_may_live(_decision: &supervisor::backstop::BackstopDecision) -> bool { - true +/// A supervisor socket the launch guard counts as possibly live. +struct LiveSocket { + socket: PathBuf, + /// The record's or sidecar's launch id; for a socket with no usable + /// sidecar, its file name's, else its ping reply's, else `unknown`. + launch_id: String, + /// `ALIVE`, `STALE`, or why the socket could not be probed. + liveness: String, + /// Why the sidecar cannot be used, for a socket with none that parses: + /// `stop --launch` cannot validate it, so the refusal names bare `stop`. + sidecar_problem: Option, +} + +impl LiveSocket { + /// The refusal line that names this socket's repair. + fn repair_line(&self, phase: PhaseId) -> String { + let (id, liveness, socket) = (&self.launch_id, &self.liveness, self.socket.display()); + match &self.sidecar_problem { + None => format!( + "launch {id} ({liveness}, socket {socket}): run \ + `devflow stop --phase {phase} --launch {id}` first" + ), + Some(problem) => format!( + "socket {socket} ({liveness}, launch {id}, its sidecar {problem}): run \ + `devflow stop --phase {phase}` first — it shuts a supervisor that answers down \ + through its socket; one that does not answer cannot be validated without its \ + sidecar, so end its agent yourself and remove `{socket}` and `{}`", + self.socket.with_extension("json").display() + ), + } + } } -/// The refusal text for [`refuse_start_over_a_live_run`]. It fires while -/// `start` holds the lock, so no other process holds it: `stop` would signal -/// nothing here (see `stop_via_lock`). -fn live_run_refusal(phase: PhaseId, verb: LaunchVerb, live: &[(&str, u32)]) -> String { - match verb { - LaunchVerb::Start => start_live_run_refusal(phase, live), - LaunchVerb::Resume => resume_live_run_refusal(phase, live), +/// The loaded state's record as a [`LiveSocket`], or `None` when its socket +/// is GONE or STALE over a group the backstop proves gone. +fn recorded_live_socket(record: &SupervisorRecord) -> Option { + let live = |liveness: &str, sidecar_problem: Option| LiveSocket { + socket: record.socket_path.clone(), + launch_id: record.launch_id.clone(), + liveness: liveness.to_string(), + sidecar_problem, + }; + match supervisor::probe(&record.socket_path) { + Ok(supervisor::Liveness::Gone) => None, + Ok(supervisor::Liveness::Alive(_)) => Some(live("ALIVE", None)), + Err(error) => Some(live(&probe_failure(&error), None)), + Ok(supervisor::Liveness::Stale) => { + match supervisor::read_sidecar(&record.socket_path.with_extension("json")) { + Ok(sidecar) => { + let decision = supervisor::backstop::assess(&sidecar, Some(record)); + stale_group_may_live(&decision).then(|| live("STALE", None)) + } + Err(supervisor::SidecarError::Io(error)) + if error.kind() == std::io::ErrorKind::NotFound => + { + Some(live("STALE", Some("is missing".to_string()))) + } + Err(error) => Some(live("STALE", Some(format!("is unusable ({error})")))), + } + } } } -fn start_live_run_refusal(phase: PhaseId, live: &[(&str, u32)]) -> String { - let named = live - .iter() - .map(|(role, pid)| format!("{role} pid {pid}")) - .collect::>() - .join(", "); - let checks = live - .iter() - .map(|(role, pid)| format!("\n {role} pid {pid}: ps -ww -o pid=,lstart=,args= -p {pid}")) - .collect::(); - let padded = phase.padded(); - // Identity details below match the argv the monitors are spawned with: - // `pipe_owning_monitor_args` and the Legacy script plus `advance_tail` in - // devflow-core's monitor.rs (48-REVIEW CR-01). A Legacy monitor whose - // agent has exited runs `devflow advance` as a foreground child and - // defers SIGTERM until it returns (48-REVIEW WR-02). - format!( - "phase {phase}: a run is still live ({named} alive) — refusing to start; nothing was written\n\ - No process holds the phase lock right now, so `devflow stop --phase {phase}` would only mark the \ - state stopped: it signals nothing and does not end this run. stop ends a run through the process \ - that holds the lock: one parked at a gate, or this run's own `devflow advance` once it takes the \ - lock after this refusal.\n\ - Check each pid before signalling it. `ps -p` shows only the executable name, which cannot tell \ - phases apart; `-ww` keeps a long command line from being cut at the terminal width:{checks}\n\ - This phase's monitor is either a `__monitor` process with this project's path and `--phase {phase}` \ - in its args, or an `sh -c` script that names `.devflow/phase-{padded}-` files and ends in `advance` \ - with `--phase {phase}`. This phase's agent has that monitor as its parent (`ps -o ppid= -p `); \ - an agent whose monitor has exited cannot be tied to this phase that way, so do not signal it on the \ - pid alone.\n\ - To end this run now, send SIGTERM to the confirmed monitor first — signalling the agent first lets \ - the monitor launch the next stage. If the agent has already exited, an `sh` monitor may be running \ - `devflow advance` as a foreground child: the monitor defers SIGTERM until that child returns, and \ - the child may take the lock as soon as this start exits. Find it with \ - `ps -ww -o pid=,args= --ppid ` and signal that child, not only the monitor.\n\ - If the named processes have exited, run `devflow start` again. If a named pid is live but is not \ - this phase's process (a recycled pid), `start` refuses again: run \ - `devflow recover --clean --phase {phase}` first, then `devflow start`. Do neither while this \ - phase's processes are live: `recover --clean` clears state even while an agent runs, and a later \ - `start` would launch a second agent beside it." +/// A sweep entry for this root and phase as a [`LiveSocket`], or `None` +/// when it is GONE, or STALE with a parsed sidecar whose group the backstop +/// proves gone. An entry with no usable sidecar is live unless GONE (RV-05). +fn swept_live_socket(entry: &supervisor::SweepEntry) -> Option { + let liveness = match &entry.liveness { + Ok(supervisor::Liveness::Gone) => return None, + Ok(supervisor::Liveness::Alive(_)) => "ALIVE".to_string(), + Ok(supervisor::Liveness::Stale) => "STALE".to_string(), + Err(error) => probe_failure(error), + }; + let (launch_id, sidecar_problem) = match &entry.sidecar { + Some(sidecar) => { + let stale = matches!(entry.liveness, Ok(supervisor::Liveness::Stale)); + if stale && !stale_group_may_live(&supervisor::backstop::assess(sidecar, None)) { + return None; + } + (sidecar.launch_id.clone(), None) + } + None => { + let pinged = match &entry.liveness { + Ok(supervisor::Liveness::Alive(reply)) => Some(reply.launch_id.clone()), + _ => None, + }; + let problem = entry.sidecar_error.as_ref().map_or_else( + || "is missing".to_string(), + |error| format!("is unusable ({error})"), + ); + let id = entry.launch_id_in_name.clone().or(pinged); + (id.unwrap_or_else(|| "unknown".to_string()), Some(problem)) + } + }; + Some(LiveSocket { + socket: entry.socket_path.clone(), + launch_id, + liveness, + sidecar_problem, + }) +} + +/// A probe error as the liveness a refusal names. +fn probe_failure(error: &supervisor::ProbeError) -> String { + format!("probe failed ({}: {error})", probe_error_kind(error)) +} + +/// Whether a STALE socket's agent group may still run. Only the three +/// backstop decisions that prove the group gone (a previous boot, an empty +/// group, a recycled leader) say no; every refusal, an unprovable identity +/// (MAC-2), a live leader and any decision added later fail closed (T-50-40). +fn stale_group_may_live(decision: &supervisor::backstop::BackstopDecision) -> bool { + use supervisor::backstop::BackstopDecision; + !matches!( + decision, + BackstopDecision::PreviousBoot + | BackstopDecision::EmptyGroup + | BackstopDecision::LeaderRecycled ) } -fn resume_live_run_refusal(phase: PhaseId, live: &[(&str, u32)]) -> String { - let named = live - .iter() - .map(|(role, pid)| format!("{role} pid {pid}")) - .collect::>() - .join(", "); - let checks = live +/// The refusal for a state that will not load over a socket directory that +/// cannot be listed: a live run may be named only by its socket there, so +/// the launch fails closed (50-06 threat flag). +fn unlistable_socket_dir_refusal( + phase: PhaseId, + verb: LaunchVerb, + (state_file, dir): (&Path, &Path), + why: &str, +) -> CliError { + let label = verb.label(); + CliError::Message(format!( + "phase {phase}: the state file {} does not load, and the supervisor socket directory {} \ + cannot be listed ({why}) — refusing to {label}; nothing was written\n\ + A live run of this phase may be named only by its socket there, so {label} fails \ + closed. Make {} listable by this user (DevFlow creates it mode 0700), then run `{}` again.", + state_file.display(), + dir.display(), + dir.display(), + verb.rerun(phase) + )) +} + +/// The refusal text for [`refuse_launch_over_a_live_run`]. It fires while the +/// launcher holds the lock: each live socket is named with the `stop` line +/// that ends it (50-14), and a live agent pid with the identity checks the +/// operator must make before signalling it. +fn live_run_refusal( + phase: PhaseId, + verb: LaunchVerb, + sockets: &[LiveSocket], + agent: Option, +) -> String { + let mut named: Vec = sockets .iter() - .map(|(role, pid)| format!("\n {role} pid {pid}: ps -ww -o pid=,lstart=,args= -p {pid}")) - .collect::(); - let padded = phase.padded(); + .map(|live| format!("supervisor launch {} {}", live.launch_id, live.liveness)) + .collect(); + named.extend(agent.map(|pid| format!("agent pid {pid} alive"))); + let mut text = format!( + "phase {phase}: a run is still live ({}) — refusing to {}; nothing was written", + named.join(", "), + verb.label() + ); + if !sockets.is_empty() { + text.push_str( + "\n`devflow stop` ends a supervisor through its socket, whether or not the state \ + file loads; on a STALE socket it signals the agent group only after proving that \ + group is this launch's. Run each line below first:", + ); + for live in sockets { + text.push_str("\n "); + text.push_str(&live.repair_line(phase)); + } + } + if let Some(pid) = agent { + text.push('\n'); + text.push_str(&agent_refusal_guidance(phase, verb, pid)); + } + text.push_str(&format!( + "\nOnce every supervisor and process named above is gone, run `{}` again.", + verb.rerun(phase) + )); + text +} + +/// The live-run refusal's paragraph for a live agent its pid file names: +/// `stop` never signals that pid, so the operator confirms its identity +/// before signalling it, and a recycled pid has its own repair per verb. +fn agent_refusal_guidance(phase: PhaseId, verb: LaunchVerb, pid: u32) -> String { + let stopped_note = match verb { + LaunchVerb::Start => "", + LaunchVerb::Resume => { + " A state already marked stopped does not mean these processes have exited: \ + `devflow status` names `devflow resume` for a phase whose supervisor is GONE even \ + while its agent is still alive, and resume refuses until every named process has \ + exited." + } + }; + let recycled = match verb { + LaunchVerb::Start => format!( + "If a named pid is live but is not this phase's process (a recycled pid), `start` \ + refuses again: run `devflow recover --clean --phase {phase}` first, then \ + `devflow start`." + ), + LaunchVerb::Resume => format!( + "If a named pid is live but is not this phase's process (a recycled pid), `resume` \ + refuses again, and no DevFlow verb clears a recorded pid while keeping the phase's \ + work: `devflow recover --clean --phase {phase}` removes the phase state, after \ + which `resume` has nothing to resume, and `devflow start` refuses while the \ + phase's worktree or branch exists; `devflow start --force` recreates them from \ + the base and discards the phase's work." + ), + }; format!( - "phase {phase}: a run is still live ({named} alive) — refusing to resume; nothing was written\n\ - No process holds the phase lock right now, so `devflow stop --phase {phase}` would only mark the \ - state stopped: it signals nothing and does not end this run. stop ends a run through the process \ - that holds the lock: one parked at a gate, or this run's own `devflow advance` once it takes the \ - lock after this refusal.\n\ - A state already marked stopped does not mean these processes have exited. `devflow status` and \ - `devflow doctor` name `devflow resume` for a phase whose monitor is dead even while its agent is \ - still alive; resume refuses until every named process has exited.\n\ - Check each pid before signalling it. `ps -p` shows only the executable name, which cannot tell \ - phases apart; `-ww` keeps a long command line from being cut at the terminal width:{checks}\n\ - This phase's monitor is either a `__monitor` process with this project's path and `--phase {phase}` \ - in its args, or an `sh -c` script that names `.devflow/phase-{padded}-` files and ends in `advance` \ - with `--phase {phase}`. This phase's agent has that monitor as its parent (`ps -o ppid= -p `); \ - an agent whose monitor has exited cannot be tied to this phase that way, so do not signal it on the \ - pid alone.\n\ - To end this run now, send SIGTERM to the confirmed monitor first — signalling the agent first lets \ - the monitor launch the next stage. If the agent has already exited, an `sh` monitor may be running \ - `devflow advance` as a foreground child: the monitor defers SIGTERM until that child returns, and \ - the child may take the lock as soon as this resume exits. Find it with \ - `ps -ww -o pid=,args= --ppid ` and signal that child, not only the monitor.\n\ - If the named processes have exited, run `devflow resume --phase {phase}` again. If a named pid is \ - live but is not this phase's process (a recycled pid), `resume` refuses again, and no DevFlow verb \ - clears a recorded pid while keeping the phase's work: `devflow recover --clean --phase {phase}` \ - removes the phase state, after which `resume` has nothing to resume, and `devflow start` refuses \ - while the phase's worktree or branch exists; `devflow start --force` recreates them from the base \ - and discards the phase's work. Do neither while this phase's processes are live: `recover --clean` \ - clears state even while an agent runs, and a later `start` would launch a second agent beside it." + "The agent pid file names a live process, and `devflow stop` never signals a pid from \ + the agent pid file.{stopped_note} Check each pid before signalling it. `ps -p` shows \ + only the executable name, which cannot tell phases apart; `-ww` keeps a long command \ + line from being cut at the terminal width:\n \ + agent pid {pid}: ps -ww -o pid=,lstart=,args= -p {pid}\n\ + This phase's agent has its supervisor, a `__monitor` process with this project's path \ + and `--phase {phase}` in its args, as its parent (`ps -o ppid= -p `); an agent \ + whose supervisor has exited cannot be tied to this phase that way, so do not signal it \ + on the pid alone. {recycled} Do neither while this phase's processes are live: \ + `recover --clean` clears state even while an agent runs, and a later `start` would \ + launch a second agent beside it." ) } From 9bd371407ca9b7c66ffc6dd38ff2b5fb56c88b09 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 10:38:11 -0400 Subject: [PATCH 081/112] test(50-15): prove two racing starts yield one live supervisor (V-12) - two `devflow start` children for one root and phase, spawned back to back: exactly one exits 0, the loser is refused at the per-phase lock or at the live-run guard, one socket probes ALIVE and the sidecars name one agent pgid - control: two starts for different phases (separate roots) yield two ALIVE sockets and two pgids, so the counting can see more than one Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/tests/supervisor_e2e.rs | 182 +++++++++++++++++++++ 1 file changed, 182 insertions(+) diff --git a/crates/devflow-cli/tests/supervisor_e2e.rs b/crates/devflow-cli/tests/supervisor_e2e.rs index 50d818e9..b2a63b79 100644 --- a/crates/devflow-cli/tests/supervisor_e2e.rs +++ b/crates/devflow-cli/tests/supervisor_e2e.rs @@ -666,3 +666,185 @@ fn census_never_sees_a_live_supervisor(supervisor_pid: u32) { ); drop(wrapper); } + +// --------------------------------------------------------------------------- +// 50-15 (R-1, V-12): two `devflow start` launches racing for one phase yield +// exactly one live supervisor and one agent group. +// --------------------------------------------------------------------------- + +/// Bounds any gate a launched run parks at, as the 48-19 refusal tests do. +const RACE_GATE_TIMEOUT_SECS: &str = "15"; + +/// A spawned `devflow start` for `phase` with the arguments the 48-19 +/// live-run refusal tests use (supervise mode, `--no-worktree`, the capture +/// shape), so a loser fails only at the per-phase lock or at the live-run +/// guard, never at an earlier refusal. +fn spawn_start( + home: &common::HermeticHome, + root: &Path, + phase: PhaseId, + fake_bin: &common::FakeBin, +) -> std::process::Child { + let existing = std::env::var_os("PATH").unwrap_or_default(); + let path = format!("{}:{}", fake_bin.path.display(), existing.to_string_lossy()); + common::hermetic_devflow(home) + .args([ + "start", + "--phase", + &phase.to_string(), + "--agent", + "claude", + "--mode", + "supervise", + "--no-worktree", + "--legacy-claude-launch", + ]) + .arg(root) + .current_dir(root) + .env("PATH", path) + .env("DEVFLOW_GATE_TIMEOUT_SECS", RACE_GATE_TIMEOUT_SECS) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn devflow start") +} + +/// What a sweep of `home`'s socket directory sees: the ALIVE sockets, and the +/// distinct agent pgids its parsed sidecars name. +fn live_supervisors(home: &common::HermeticHome) -> (Vec, Vec) { + let entries = supervisor::sweep_in(&home.cache_dir()); + let alive = entries + .iter() + .filter(|entry| matches!(entry.liveness, Ok(Liveness::Alive(_)))) + .map(|entry| entry.socket_path.clone()) + .collect(); + let mut pgids: Vec = entries + .iter() + .filter_map(|entry| entry.sidecar.as_ref().map(|sidecar| sidecar.agent_pgid)) + .collect(); + pgids.sort_unstable(); + pgids.dedup(); + (alive, pgids) +} + +#[test] +fn two_concurrent_starts_yield_exactly_one_live_supervisor() { + let home = common::HermeticHome::new(); + let repo = tempfile::tempdir().unwrap(); + let root = repo.path().canonicalize().unwrap(); + let phase = PhaseId::new(63); + init_repo(&root, phase); + let fake_bin = common::fake_bin_dir(WAITING_CLAUDE); + let _reaper = common::LaunchedRunReaper::new(&root, phase, vec![]); + + let first = spawn_start(&home, &root, phase, &fake_bin); + let second = spawn_start(&home, &root, phase, &fake_bin); + let outputs = [ + first.wait_with_output().expect("wait for the first start"), + second + .wait_with_output() + .expect("wait for the second start"), + ]; + + let report = |output: &Output| { + format!( + "status {:?}\nstdout: {}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) + }; + let winners = outputs + .iter() + .filter(|output| output.status.success()) + .count(); + assert_eq!( + winners, + 1, + "exactly one racing start must succeed\nfirst: {}\nsecond: {}", + report(&outputs[0]), + report(&outputs[1]) + ); + let loser = outputs + .iter() + .find(|output| !output.status.success()) + .expect("one start lost"); + let stderr = String::from_utf8_lossy(&loser.stderr); + let lock_refusal = stderr.contains("holds the per-phase lock"); + let live_refusal = stderr.contains(&format!("devflow stop --phase {phase}")) + && stderr.contains("a run is still live"); + assert!( + lock_refusal || live_refusal, + "the loser must be refused at the lock or at the live-run guard, nothing else\n{}", + report(loser) + ); + println!( + "V-12 loser refusal: {}", + if lock_refusal { + "per-phase lock" + } else { + "live-supervisor guard" + } + ); + + let (alive, pgids) = live_supervisors(&home); + assert_eq!( + alive.len(), + 1, + "exactly one socket may probe ALIVE: {alive:?}\nloser: {}", + report(loser) + ); + assert_eq!( + pgids.len(), + 1, + "the sidecars must name exactly one agent group: {pgids:?}" + ); + assert!( + !group_members(pgids[0]).is_empty(), + "the one agent group {} must be live", + pgids[0] + ); +} + +#[test] +fn two_starts_for_different_phases_yield_two_live_supervisors() { + let home = common::HermeticHome::new(); + let repos = [tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()]; + let roots: Vec = repos + .iter() + .map(|repo| repo.path().canonicalize().unwrap()) + .collect(); + let phases = [PhaseId::new(64), PhaseId::new(65)]; + for (root, phase) in roots.iter().zip(phases) { + init_repo(root, phase); + } + let fake_bin = common::fake_bin_dir(WAITING_CLAUDE); + let _reapers: Vec = roots + .iter() + .zip(phases) + .map(|(root, phase)| common::LaunchedRunReaper::new(root, phase, vec![])) + .collect(); + + let first = spawn_start(&home, &roots[0], phases[0], &fake_bin); + let second = spawn_start(&home, &roots[1], phases[1], &fake_bin); + for (child, phase) in [first, second].into_iter().zip(phases) { + let output = child.wait_with_output().expect("wait for start"); + assert!( + output.status.success(), + "phase {phase}: a start with no rival for its phase must succeed\nstderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + let (alive, pgids) = live_supervisors(&home); + assert_eq!( + alive.len(), + 2, + "the counting must see both live supervisors: {alive:?}" + ); + assert_eq!( + pgids.len(), + 2, + "the sidecars must name two distinct agent groups: {pgids:?}" + ); +} From e97d5fced91a69c9e748defdd33cd7b1ed67ca1b Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 13:52:43 -0400 Subject: [PATCH 082/112] refactor(50-22): record only the supervisor on a launch - spawn_agent_and_record no longer clears a monitor pid; publish_record writes state.supervisor only; refuse_launch clears only the record - stage_launched carries supervisor_pid, socket_path, launch_id, launch; the start line names the supervisor pid - the --until stop clears only the supervisor record - tests assert state.supervisor; persistence, early-failure, until-stop and whole-state handoff tests renamed off monitor_pid Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/pipeline_gate.rs | 26 ++-- crates/devflow-cli/src/pipeline_launch.rs | 181 ++++++++++------------ 2 files changed, 90 insertions(+), 117 deletions(-) diff --git a/crates/devflow-cli/src/pipeline_gate.rs b/crates/devflow-cli/src/pipeline_gate.rs index 736fc019..f6129ea7 100644 --- a/crates/devflow-cli/src/pipeline_gate.rs +++ b/crates/devflow-cli/src/pipeline_gate.rs @@ -68,13 +68,12 @@ pub(crate) fn transition( if state.stop_until == Some(from) { state.stopped = true; state.stop_reason = Some(format!("stopped after {from} completed (--until {from})")); - // T-20-03b / doctor gap (D-09): clear monitor_pid/gate_pending so - // neither check_dead_agent nor check_dead_monitor misreports this - // intentional stop as a crashed agent or dead monitor. The supervisor - // record goes with monitor_pid (50-13): the supervisor running this - // advance finishes once it returns, so the record would otherwise - // name a socket that is gone. - state.monitor_pid = None; + // T-20-03b / doctor gap (D-09): clear the supervisor record and + // gate_pending so neither check_dead_agent nor check_dead_monitor + // misreports this intentional stop as a crashed agent or dead + // monitor (50-13): the supervisor running this advance finishes once + // it returns, so the record would otherwise name a socket that is + // gone. state.supervisor = None; state.gate_pending = false; workflow::save_state(state)?; @@ -915,19 +914,17 @@ mod tests { assert!(devflow_core::ship_evidence::is_stopped_at(&evidence)); } - /// 50-13: the `--until` stop clears the supervisor record together with - /// `monitor_pid`, because the supervisor running this advance finishes - /// once the advance returns. Both are set first, so a stop that clears - /// neither (or only the pid) fails here. + /// 50-13: the `--until` stop clears the supervisor record, because the + /// supervisor running this advance finishes once the advance returns. + /// The record is set first, so a stop that leaves it standing fails here. #[test] - fn until_stop_clears_the_supervisor_record_with_the_monitor_pid() { + fn until_stop_clears_the_supervisor_record() { let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(1313); let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); state.stage = Stage::Plan; state.stop_until = Some(Stage::Plan); - state.monitor_pid = Some(4242); state.supervisor = Some(devflow_core::state::SupervisorRecord::new( root.join("until.sock"), "until00000000000".to_string(), @@ -941,10 +938,9 @@ mod tests { let reloaded = workflow::load_state(root, phase).unwrap(); assert!(reloaded.stopped, "the --until stop marks the run stopped"); - assert_eq!(reloaded.monitor_pid, None); assert_eq!( reloaded.supervisor, None, - "the supervisor record is cleared with monitor_pid" + "the --until stop clears the supervisor record" ); } diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index cb1ea23e..d9a3a63e 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -575,15 +575,14 @@ where /// Abort a launch the canary refused, leaving no half-launched stage behind. /// -/// Clearing `monitor_pid` and `supervisor` applies WR-04's rationale (see +/// Clearing `supervisor` applies WR-04's rationale (see /// [`record_failed_launch`]) to an aborted launch: `transition()` has already /// advanced `state.stage` and saved it by the time this runs, so a refusal that -/// left the PREVIOUS stage's monitor pid or supervisor record standing would -/// read as a run gone while active and send the operator to `devflow resume`, -/// after the wrong problem entirely. With neither, the run reads not launched, -/// and the right remedy is the message below. +/// left the PREVIOUS stage's supervisor record standing would read as a run +/// gone while active and send the operator to `devflow resume`, after the +/// wrong problem entirely. With no record, the run reads not launched, and the +/// right remedy is the message below. fn refuse_launch(state: &mut State, message: String) -> Result<(), CliError> { - state.monitor_pid = None; state.supervisor = None; workflow::save_state(state)?; Err(CliError::Message(message)) @@ -1573,9 +1572,8 @@ fn record_failed_launch(state: &mut State) -> Result<(), CliError> { Ok(()) } -/// The tail of [`launch_stage_inner`]: clear the stale monitor pid, validate -/// the agent binary, archive the prior capture, spawn the monitor, and -/// record the launch. Extracted (28-03, Task 2) so +/// The tail of [`launch_stage_inner`]: validate the agent binary, archive +/// the prior capture, spawn the monitor, and record the launch. Extracted (28-03, Task 2) so /// [`relaunch_checkpoint_session`] can share this EXACT tail for a /// checkpoint auto-decide resume — a resume is a continuation of the same /// stage's agent run, not a fresh stage entry, so it must not duplicate this @@ -1593,20 +1591,18 @@ fn spawn_agent_and_record( archived_stage: Option, launch: monitor::MonitorLaunch, ) -> Result<(), CliError> { - // WR-04 (18-fix): clear the prior stage's monitor pid up front, before - // any fallible step below (`ensure_agent_binary`) can return early via - // `?`. Without this, a failed relaunch left `state.stage` already - // advanced (by `transition()`, before this function was ever called) - // alongside a stale `monitor_pid` still naming the PREVIOUS stage's - // (now-dead) monitor — `liveness()` then misreports `Stuck → devflow - // resume`, even when the real remedy is unrelated. The real pid is set - // again below once `monitor::spawn_monitor` actually succeeds. - state.monitor_pid = None; + // Persist the caller's launch bookkeeping (`launch_stage_inner`'s + // `checkpoint_resumes` reset and Validate dispatch stamp) before + // anything is spawned, as this save always has. It once also cleared + // the previous stage's monitor pid (WR-04, 18-fix); that field is gone + // from the CLI (50-22), and the record keeps its own rule below. workflow::save_state(state)?; - // 50-23 (T-50-62): every error before the publish save ends the launch - // with no supervisor record (see `record_failed_launch`); the success - // path below never clears the record before the save that replaces it. + // WR-04 (18-fix, 50-23 T-50-62): every error before the publish save + // ends the launch with no supervisor record (see `record_failed_launch`), + // so a failed relaunch never leaves `state.stage` advanced beside a + // record naming the previous stage's supervisor; the success path below + // never clears the record before the save that replaces it. let SpawnedLaunch { published, ctx, @@ -1628,11 +1624,9 @@ fn spawn_agent_and_record( let socket_path = published.record.socket_path.display().to_string(); let launch_id = published.record.launch_id.clone(); // `transition()` calls `workflow::save_state` BEFORE `launch_stage`, so a - // pid recorded only in memory here is lost unless it is written again + // record held only in memory here is lost unless it is written again // (18b). The ping-confirmed supervisor record replaces any previous one - // in this same save; it is never cleared first (WR-04's pre-clear above - // applies to `monitor_pid` only). `monitor_pid` names the supervisor - // until 50-22 removes it. + // in this same save; it is never cleared first. // // The handoff (50-13, R-1): this save is the publish. The previous // supervisor, when this launch is its advance, stays alive and @@ -1669,7 +1663,7 @@ fn spawn_agent_and_record( Err(err) => warn!(phase = %state.phase, "could not consume cron instructions: {err}"), } // 23b: register this (project_root, phase) in the machine-global - // registry on the same code path that just recorded monitor_pid, so a + // registry on the same code path that just recorded the supervisor, so a // running phase cannot be missing from `devflow gate list --all-roots`. // Best-effort observability — never a reason to fail a launch. let _ = devflow_core::registry::register_in(&ctx.cache_dir, &state.project_root, state.phase); @@ -1680,14 +1674,14 @@ fn spawn_agent_and_record( serde_json::json!({ "stage": state.stage.to_string(), "agent": state.agent.to_string(), - "monitor_pid": pid, + "supervisor_pid": pid, "socket_path": socket_path, "launch_id": launch_id, "launch": launch_label, }), ); println!( - "stage {} → launched {} (monitor pid {pid}, {launch_label} supervisor at {socket_path})", + "stage {} → launched {} (supervisor pid {pid}, {launch_label} supervisor at {socket_path})", state.stage, agents::driver_for(state.agent).name() ); @@ -1740,7 +1734,6 @@ fn publish_record>( mut save: impl FnMut(&State) -> Result<(), E>, ) -> Result { let record = published.record; - state.monitor_pid = Some(published.supervisor_pid); state.supervisor = Some(record.clone()); if let Err(err) = save(state) { return Err(shut_down_unrecorded(state, &record, err.into())); @@ -2960,23 +2953,24 @@ mod tests { } /// 18b: after `launch_stage` spawns a monitor, the persisted state file - /// for that phase carries the monitor's pid — `transition()` saves state - /// BEFORE calling `launch_stage`, so the pid must be saved again inside - /// `launch_stage` or it is lost. + /// for that phase carries the published supervisor record — `transition()` + /// saves state BEFORE calling `launch_stage`, so the record must be saved + /// again inside `launch_stage` or it is lost. /// /// **Premise moved off `STREAM_JSON_STAGES` membership deliberately /// (34-06); do not "simplify" the opt-out away.** This test's subject is - /// pid persistence, not which launch path runs, so it previously relied on + /// record persistence, not which launch path runs, so it previously relied on /// its stage being ABSENT from `STREAM_JSON_STAGES` — an incidental /// premise that 34-05's widening destroys, at which point `canary_gate` /// invokes the real `ClaudeCanaryLauncher` and the launch fails on a - /// delivery refusal that has nothing to do with pid persistence. Pinning + /// delivery refusal that has nothing to do with record persistence. Pinning /// the legacy path via the opt-out makes the premise explicit and stable /// under any contents of the constant, exactly as /// `canary_gate_only_applies_to_the_stream_launch_path` does. #[test] - fn launch_stage_persists_monitor_pid_for_reload() { - const NAME: &str = "pipeline_launch::tests::launch_stage_persists_monitor_pid_for_reload"; + fn launch_stage_persists_the_supervisor_record_for_reload() { + const NAME: &str = + "pipeline_launch::tests::launch_stage_persists_the_supervisor_record_for_reload"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_dir_with_agent_stub("claude"); let output = @@ -2995,7 +2989,7 @@ mod tests { // legacy opt-out set below makes this a launch shape whose Code stage // cannot bound the chain flag's lifetime, and // `preflight_unattended_launch_check` refuses exactly that combination - // in `Mode::Auto`. This test's subject — pid persistence — is + // in `Mode::Auto`. This test's subject — record persistence — is // mode-independent, so supervise keeps the premise the 34-06 note // below establishes without asking the preflight to permit a launch // D-07 exists to refuse. @@ -3017,23 +3011,23 @@ mod tests { // reap it before `dir` drops below. Bound here — this test's LAST // `&mut state` use — the guard outranks every panicking checkpoint // that follows: `result.unwrap()`, the `assert!` on - // `state.monitor_pid.is_some()`, `workflow::load_state(...).unwrap()`, - // and the `assert_eq!` on the reloaded pid. Binding here also covers + // `state.supervisor.is_some()`, `workflow::load_state(...).unwrap()`, + // and the `assert_eq!` on the reloaded record. Binding here also covers // the narrower case of a launch that spawned the monitor and then // failed a later `?` inside `launch_stage_inner` — `result` would be - // `Err` but the pid would nonetheless be live (G-25-2, 25-17). + // `Err` but the supervisor would nonetheless be live (G-25-2, 25-17). let _reap_guard = ReapMonitorOnDrop::after_launch(&state); result.unwrap(); assert!( - state.monitor_pid.is_some(), - "launch_stage must record the monitor pid on the in-memory state" + state.supervisor.is_some(), + "launch_stage must record the published supervisor on the in-memory state" ); let reloaded = workflow::load_state(root, phase).unwrap(); assert_eq!( - reloaded.monitor_pid, state.monitor_pid, - "the monitor pid recorded by launch_stage must be persisted to disk, \ + reloaded.supervisor, state.supervisor, + "the supervisor record published by launch_stage must be persisted to disk, \ since transition() saves state before launch_stage runs" ); } @@ -3148,7 +3142,7 @@ mod tests { root.to_path_buf(), ); // The capture shape, for the same reason as - // `launch_stage_persists_monitor_pid_for_reload`: no delivery canary. + // `launch_stage_persists_the_supervisor_record_for_reload`: no delivery canary. state.legacy_claude_launch = true; workflow::save_state(&state).unwrap(); @@ -3626,8 +3620,8 @@ mod tests { the next time it advances past Plan" ); assert!( - reloaded.monitor_pid.is_some(), - "resume() must have spawned a monitor whose pid is recorded in state — if this \ + reloaded.supervisor.is_some(), + "resume() must have spawned a monitor whose supervisor is recorded in state — if this \ fails, the reap guard above is silently reaping nothing and this test has \ stopped covering the launch path it was written to cover" ); @@ -3707,7 +3701,7 @@ mod tests { "resume must relaunch the saved stage" ); assert!( - reloaded.monitor_pid.is_some(), + reloaded.supervisor.is_some(), "resume must have spawned a monitor for the saved stage" ); assert_eq!( @@ -3819,7 +3813,7 @@ mod tests { let phase = PhaseId::new(67); // `Mode::Supervise` for the same 35.1-03 reason as - // `launch_stage_persists_monitor_pid_for_reload` above: the legacy + // `launch_stage_persists_the_supervisor_record_for_reload` above: the legacy // opt-out set below is refused in `Mode::Auto` by // `preflight_unattended_launch_check`, and this test's subject — the // unfired `stop_until` cap surviving a resume — is mode-independent. @@ -3865,8 +3859,8 @@ mod tests { "an unfired cap must not itself flip stopped to true — resume only relaunches" ); assert!( - reloaded.monitor_pid.is_some(), - "resume() must still have spawned a monitor whose pid is recorded in state" + reloaded.supervisor.is_some(), + "resume() must still have spawned a monitor whose supervisor is recorded in state" ); } @@ -3930,8 +3924,8 @@ mod tests { assert!(!reloaded.stopped); assert_eq!(reloaded.stop_reason, None); assert!( - reloaded.monitor_pid.is_some(), - "resume() must still relaunch and record a monitor pid with no cap present" + reloaded.supervisor.is_some(), + "resume() must still relaunch and record a supervisor with no cap present" ); } @@ -3974,7 +3968,7 @@ mod tests { let reloaded = workflow::load_state(root, phase).unwrap(); assert_eq!(reloaded.agent, AgentKind::Codex); - assert!(reloaded.monitor_pid.is_some()); + assert!(reloaded.supervisor.is_some()); assert!(!devflow_core::ship::cron_instructions_path(root, phase).exists()); let events = std::fs::read_to_string(devflow_core::events::events_path(root)).unwrap(); let handoff = events @@ -4161,8 +4155,9 @@ mod tests { /// Whole-state comparison keeps this handoff check current when State gains fields. #[test] - fn resume_with_agent_preserves_every_state_field_except_agent_and_monitor_pid() { - const NAME: &str = "pipeline_launch::tests::resume_with_agent_preserves_every_state_field_except_agent_and_monitor_pid"; + fn resume_with_agent_preserves_every_state_field_except_agent() { + const NAME: &str = + "pipeline_launch::tests::resume_with_agent_preserves_every_state_field_except_agent"; if !devflow_core::test_support::in_child_test(NAME) { let path_dir = agent_free_dir_with_agent_stub("codex"); let output = @@ -4243,7 +4238,6 @@ mod tests { for value in [&mut before, &mut after] { value.as_object_mut().unwrap().remove("agent"); - value.as_object_mut().unwrap().remove("monitor_pid"); } assert_eq!(before, after); } @@ -4289,7 +4283,7 @@ mod tests { let reloaded = workflow::load_state(root, phase).unwrap(); assert_eq!(reloaded.agent, AgentKind::Codex); assert!(!reloaded.stopped); - assert!(reloaded.monitor_pid.is_some()); + assert!(reloaded.supervisor.is_some()); assert!(!devflow_core::ship::cron_instructions_path(root, phase).exists()); } @@ -4327,7 +4321,7 @@ mod tests { "a failed resume must leave the phase marked stopped, not a false-active zombie" ); assert!(reloaded.stop_reason.is_some_and(|r| r.contains("claude"))); - assert!(reloaded.monitor_pid.is_none()); + assert!(reloaded.supervisor.is_none()); return; } @@ -4425,20 +4419,24 @@ mod tests { }); } - /// WR-04 (18-fix): an early failure in `launch_stage_inner` — before - /// `monitor::spawn_monitor` ever runs — must not leave a stale - /// `monitor_pid` behind. Pre-fix, `state.monitor_pid` still named the - /// PREVIOUS stage's (now-dead) monitor after `ensure_agent_binary` - /// returned early via `?`, and `liveness()`/`doctor` then misreported - /// `Stuck → devflow resume` — the wrong remedy for what's actually an - /// agent-binary/staleness failure. PATH is neutralized to a `git`-only - /// directory under `ENV_MUTEX`, mirroring `transition_resets_infra_failures`, - /// so `ensure_agent_binary("claude")` reliably fails without touching a - /// real agent CLI and without racing other PATH-mutating tests. + /// WR-04 (18-fix, 50-23): an early failure in `launch_stage_inner` — + /// before `monitor::spawn_monitor` ever runs — must not leave the + /// PREVIOUS stage's supervisor record behind. Pre-fix, the state still + /// named the previous stage's (now-dead) monitor after the launch + /// returned early via `?`, and `status`/`doctor` then misreported it as + /// gone while active, naming `devflow resume` — the wrong remedy for + /// what is actually a launch failure. + /// + /// Which early failure this reaches (checked by hand mutants, 50-22): the + /// child's PATH holds only `git` and an `exit 0` stub `claude`, so for an + /// `Auto` Code launch the delivery canary reads `Absent` and the launch + /// ends in [`refuse_launch`], before `ensure_agent_binary` runs. The + /// other early-failure path, [`record_failed_launch`], is pinned by + /// `a_failed_monitor_spawn_clears_the_supervisor_record`. #[test] - fn launch_stage_inner_clears_monitor_pid_on_early_failure() { - const NAME: &str = - "pipeline_launch::tests::launch_stage_inner_clears_monitor_pid_on_early_failure"; + fn launch_stage_inner_clears_the_supervisor_record_on_early_failure() { + const NAME: &str = "pipeline_launch::tests::\ + launch_stage_inner_clears_the_supervisor_record_on_early_failure"; enter_agent_free_child!(NAME, "claude"); let _guard = env_lock(); @@ -4447,11 +4445,9 @@ mod tests { let phase = PhaseId::new(93); let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); state.stage = Stage::Code; - // A stale pid from a prior stage's now-dead monitor — this is what - // must be cleared, not carried forward into the new stage. - state.monitor_pid = Some(999_999); - // 50-23: the prior stage's supervisor record, which must be cleared - // too, or `status` reads GONE-while-active and names `devflow resume`. + // 50-23: the prior stage's supervisor record, which must be cleared, + // not carried forward into the new stage, or `status` reads + // GONE-while-active and names `devflow resume`. state.supervisor = Some(fabricated_prior_record(root)); workflow::save_state(&state).unwrap(); @@ -4459,22 +4455,13 @@ mod tests { assert!( result.is_err(), - "ensure_agent_binary must fail against the neutralized, agent-free PATH" - ); - assert_eq!( - state.monitor_pid, None, - "an early launch failure must clear the stale monitor_pid in-memory, not carry it \ - forward from the previous stage" + "the launch must fail early: the stub `claude` cannot confirm the canary" ); assert_eq!( state.supervisor, None, "an early launch failure must clear the prior supervisor record in memory" ); let reloaded = workflow::load_state(root, phase).unwrap(); - assert_eq!( - reloaded.monitor_pid, None, - "the monitor_pid clear must be persisted to state.json, not just in-memory" - ); assert_eq!( reloaded.supervisor, None, "the supervisor record clear must be persisted to state.json" @@ -5140,9 +5127,8 @@ mod tests { let phase = PhaseId::new(122); let mut state = canary_state(root, phase); - // The previous stage's monitor pid, as `transition()` would leave it. - state.monitor_pid = Some(4_294_967_000); - // 50-23: and the previous stage's supervisor record. + // The previous stage's supervisor record, as `transition()` would + // leave it (50-23). state.supervisor = Some(fabricated_prior_record(root)); workflow::save_state(&state).unwrap(); @@ -5165,21 +5151,12 @@ mod tests { got: {message}" ); - assert!( - state.monitor_pid.is_none(), - "a refused launch must not leave the previous stage's monitor pid standing — \ - liveness() would report Stuck and point at `devflow resume`, which cannot help" - ); assert_eq!( state.supervisor, None, "a refused launch must not leave the previous stage's supervisor record standing: \ it reads GONE-while-active and sends the operator to `devflow resume`" ); let reloaded = workflow::load_state(root, phase).unwrap(); - assert!( - reloaded.monitor_pid.is_none(), - "the cleared pid must be persisted, not only cleared in memory" - ); assert_eq!( reloaded.supervisor, None, "the cleared supervisor record must be persisted, not only cleared in memory" @@ -5365,8 +5342,8 @@ mod tests { ); assert_eq!(state.canary, Some(canary::CanaryOutcome::Absent)); assert!( - state.monitor_pid.is_none(), - "a refused launch must record no monitor pid" + state.supervisor.is_none(), + "a refused launch must record no supervisor" ); assert_eq!( stage_launched_count(root, phase), @@ -6854,7 +6831,7 @@ mod tests { assert!( reloaded_for_reap .as_ref() - .is_some_and(|reloaded| reloaded.monitor_pid.is_some()), + .is_some_and(|reloaded| reloaded.supervisor.is_some()), "the checkpoint must relaunch the session, not only record the event" ); let gate_fired = events_of_kind(root, "gate_fired"); From e3b361375c0a1d0704e1a9a11a3682c022563899 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 14:02:34 -0400 Subject: [PATCH 083/112] refactor(50-22): drop the CLI's last monitor pid reads - registry_reachable_pids keeps lock holders only; the /proc census cannot match a 3.x supervisor (50-12) - doctor, preflight and e2e tests assert state.supervisor; the stop unit test is covered by stop_e2e's planted-pid test - phase7_cli's reap guard takes the supervisor pid from its socket - supervisor_e2e checks the socket against the sidecar's pid Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 157 ++++++++++----------- crates/devflow-cli/src/preflight.rs | 10 +- crates/devflow-cli/tests/phase7_cli.rs | 77 ++++++---- crates/devflow-cli/tests/stop_e2e.rs | 10 +- crates/devflow-cli/tests/supervisor_e2e.rs | 14 +- 5 files changed, 141 insertions(+), 127 deletions(-) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 0b490b3d..220700b0 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -3252,9 +3252,8 @@ impl LaunchVerb { /// liveness, not identity: a recycled pid gives a fail-closed false /// refusal (Option A, 2026-09-22). /// -/// No `monitor_pid` is read. A loaded state with no record over a live -/// supervisor (a launcher that died before saving it, T-50-90/T-50-91) is -/// accepted until Phase 50.1. +/// A loaded state with no record over a live supervisor (a launcher that +/// died before saving it, T-50-90/T-50-91) is accepted until Phase 50.1. fn refuse_launch_over_a_live_run( project_root: &Path, phase: PhaseId, @@ -5370,12 +5369,19 @@ fn stray_layer_label(layer: agent::StrayLayer) -> &'static str { } } -/// The set of pids that a live registry entry still reaches: every -/// [`devflow_core::state::State::monitor_pid`] recorded under any of -/// `roots` (via [`workflow::list_states`]) plus every lock holder pid for -/// those same roots' phases (via [`lock::holder_identity`]). A pid in this -/// set is by definition not a stray (CR-01, 25-15-PLAN.md) — a live -/// registry entry, lock file, or state file still names it. +/// The set of pids that a live registry entry still reaches: every lock +/// holder pid for the phases of `roots` (listed via +/// [`workflow::list_states`], read via [`lock::holder_identity`]). A pid in +/// this set is by definition not a stray (CR-01, 25-15-PLAN.md) — a live +/// lock file still names it. +/// +/// Lock holders only (50-22). A 3.x supervisor needs no entry: the `/proc` +/// census this set filters ([`agent::discover_stray_devflow_processes`]) +/// matches only the 2.x monitor-wrapper and `devflow advance` argv shapes +/// and cannot match a supervisor's (50-12, pinned by +/// `agent::tests::the_stray_census_matches_only_the_2x_process_shapes` and +/// by `census_never_sees_a_live_supervisor` in the supervisor e2e), so a +/// supervisor's pid can never reach a reap through this filter. /// /// Deliberately excludes the per-phase agent pid file: the `/proc` census /// this set filters ([`agent::discover_stray_devflow_processes`]) can only @@ -5390,8 +5396,8 @@ fn stray_layer_label(layer: agent::StrayLayer) -> &'static str { /// read-only path). /// /// RESIDUAL (T-25-15-08, not eliminated): this is a point-in-time read. A -/// root registered, or a `monitor_pid` written, between this read and a -/// caller's later signal is not covered by it. The untouched +/// root registered, or a lock taken, between this read and a caller's later +/// signal is not covered by it. The untouched /// `agent::is_same_process` identity re-confirmation and /// [`agent::STRAY_MIN_AGE`] age floor downstream remain the last line of /// defence in that window. @@ -5406,9 +5412,6 @@ pub(crate) fn registry_reachable_pids(roots: &[PathBuf]) -> HashSet { continue; } for state in workflow::list_states(root) { - if let Some(pid) = state.monitor_pid { - reachable.insert(pid); - } if let Some((pid, _start_time)) = lock::holder_identity(root, state.phase) { reachable.insert(pid); } @@ -6837,10 +6840,10 @@ mod tests { assert!(!agent::agent_running(u32::MAX)); } - /// 18b: persisting `monitor_pid` for one phase must not disturb a - /// concurrently-active sibling phase's `monitor_pid` (concurrency edge). + /// 18b: persisting the supervisor record for one phase must not disturb + /// a concurrently-active sibling phase's record (concurrency edge). #[test] - fn monitor_pid_persisted_for_one_phase_does_not_disturb_a_sibling() { + fn supervisor_record_persisted_for_one_phase_does_not_disturb_a_sibling() { let dir = tempfile::tempdir().unwrap(); let root = dir.path(); @@ -6850,7 +6853,14 @@ mod tests { Mode::Auto, root.to_path_buf(), ); - phase7.monitor_pid = Some(111); + let record7 = devflow_core::state::SupervisorRecord::new( + root.join("seven.sock"), + "seven00000000000".to_string(), + 111, + 1, + String::new(), + ); + phase7.supervisor = Some(record7.clone()); workflow::save_state(&phase7).unwrap(); let mut phase8 = State::new( @@ -6859,28 +6869,41 @@ mod tests { Mode::Auto, root.to_path_buf(), ); - phase8.monitor_pid = Some(222); + let record8 = devflow_core::state::SupervisorRecord::new( + root.join("eight.sock"), + "eight00000000000".to_string(), + 222, + 2, + String::new(), + ); + phase8.supervisor = Some(record8.clone()); workflow::save_state(&phase8).unwrap(); let reloaded7 = workflow::load_state(root, PhaseId::new(7)).unwrap(); let reloaded8 = workflow::load_state(root, PhaseId::new(8)).unwrap(); - assert_eq!(reloaded7.monitor_pid, Some(111)); - assert_eq!(reloaded8.monitor_pid, Some(222)); + assert_eq!(reloaded7.supervisor, Some(record7)); + assert_eq!(reloaded8.supervisor, Some(record8)); } /// 18b (idempotency edge): running `devflow status` twice must produce - /// byte-identical `.devflow/` state — the new monitor liveness probe is + /// byte-identical `.devflow/` state — the supervisor liveness probe is /// purely a read, same as the existing agent liveness probe it sits - /// beside. Also exercises the `u32::MAX` boundary pid (precision edge, - /// via `agent::agent_running`'s existing hardening) so the probe can - /// only ever report `Stuck`, never a false `Healthy`. + /// beside. The planted record names a socket that does not exist and the + /// `u32::MAX` boundary agent group (precision edge), so the probe can + /// only read it GONE, never a false ALIVE. #[test] fn status_reading_monitor_liveness_writes_no_state_and_no_event() { let dir = tempfile::tempdir().unwrap(); let root = dir.path(); let phase = PhaseId::new(66); let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); - state.monitor_pid = Some(u32::MAX); + state.supervisor = Some(devflow_core::state::SupervisorRecord::new( + root.join("never-bound.sock"), + "statusidem000000".to_string(), + u32::MAX, + 1, + String::new(), + )); workflow::save_state(&state).unwrap(); let state_path = workflow::state_path(root, phase); @@ -7630,24 +7653,6 @@ mod tests { stop(root, phase).expect("stop against a stale lock must succeed, not error"); } - /// T-23-51: a live `monitor_pid` recorded in `State` — with no open gate - /// and no lock file — must never be treated as a signalling target. - /// `stop` only ever looks at the lock file. - #[test] - fn stop_never_treats_monitor_pid_as_a_signalling_target() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path(); - let phase = PhaseId::new(203); - let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); - state.monitor_pid = Some(std::process::id()); - workflow::save_state(&state).unwrap(); - - stop(root, phase).expect( - "stop must succeed — a recorded monitor_pid alone must never be signalled or \ - treated as a blocker", - ); - } - /// Task 3 behavior: no open gate, no lock file, and no persisted state /// at all — a phase that was never started — is a successful no-op. #[test] @@ -8313,7 +8318,7 @@ mod tests { root, PhaseId::new(7), "stage_launched", - serde_json::json!({"stage": "code", "agent": "claude", "monitor_pid": 1}), + serde_json::json!({"stage": "code", "agent": "claude", "supervisor_pid": 1}), ); // events::emit always stamps `ts` with the current time; rewrite it // to a fixed, known-past value so the assertion is deterministic @@ -9734,18 +9739,20 @@ mod tests { } /// Task 1's tracer (CR-01, 999.44/DEN-68): one discovery pass names - /// a live `monitor_pid`-recorded pid, a live lock-holder pid, and a - /// genuine orphan under no registered root at all. Only the orphan - /// may survive `unreachable_stray_candidates`'s filter. All three - /// pids are real, spawned processes crossed through + /// a live lock-holder pid and a genuine orphan under no registered root + /// at all. Only the orphan may survive `unreachable_stray_candidates`'s + /// filter. Both pids are real, spawned processes crossed through /// `wait_for_exec_visibility` — not synthesised numbers — so the /// census and the filter are both exercised for real, in one pass. + /// + /// A third, state-recorded fixture (a pid in the state's monitor-pid + /// field) was removed with that input (50-22): lock holders are the + /// only reachable source, because the census cannot match a 3.x + /// supervisor (`agent::tests::the_stray_census_matches_only_the_2x_process_shapes`). #[test] fn reachable_pids_are_excluded_from_both_the_findings_and_the_reap_candidates() { - let mut state_child = spawn_wrapper_shaped_fixture(); let mut lock_child = spawn_wrapper_shaped_fixture(); let mut orphan_child = spawn_wrapper_shaped_fixture(); - let state_pid = state_child.id(); let lock_pid = lock_child.id(); let orphan_pid = orphan_child.id(); @@ -9753,22 +9760,19 @@ mod tests { let project_root_guard = tempfile::tempdir().unwrap(); let project_root = project_root_guard.path().to_path_buf(); - // Phase 1: a real `State` naming `state_pid` as its monitor. - let mut state = State::new( + // Phase 1: a real `State` with no lock, the registered phase. + let state = State::new( PhaseId::new(1), AgentKind::Claude, Mode::Auto, project_root.clone(), ); - state.monitor_pid = Some(state_pid); workflow::save_state(&state).unwrap(); // Phase 2: a real `State` (so `list_states` enumerates the // phase at all — mirrors real operation, where a phase with an - // active lock also has a persisted state), deliberately with NO - // `monitor_pid`, so this phase's only path into the reachable - // set is the lock file below, not the monitor_pid path Phase 1 - // already exercises. + // active lock also has a persisted state), whose only path into + // the reachable set is the lock file below. let state_2 = State::new( PhaseId::new(2), AgentKind::Claude, @@ -9806,10 +9810,6 @@ mod tests { .collect(); let reachable = registry_reachable_pids(®istered_roots); - assert!( - reachable.contains(&state_pid), - "state_pid must be reachable via its recorded monitor_pid" - ); assert!( reachable.contains(&lock_pid), "lock_pid must be reachable via the lock file's holder_identity" @@ -9823,11 +9823,7 @@ mod tests { // Without this, the test could pass vacuously because the // census never saw the fixtures at all (T-25-15-11). let census = agent::discover_stray_devflow_processes(); - for (pid, label) in [ - (state_pid, "state"), - (lock_pid, "lock"), - (orphan_pid, "orphan"), - ] { + for (pid, label) in [(lock_pid, "lock"), (orphan_pid, "orphan")] { assert!( census.iter().any(|p| p.pid == pid), "the {label} fixture (pid {pid}) must be part of the real /proc census" @@ -9839,10 +9835,6 @@ mod tests { retained.iter().any(|p| p.pid == orphan_pid), "the orphan must survive the filter" ); - assert!( - !retained.iter().any(|p| p.pid == state_pid), - "the state-named pid must be filtered out" - ); assert!( !retained.iter().any(|p| p.pid == lock_pid), "the lock-held pid must be filtered out" @@ -9853,26 +9845,21 @@ mod tests { findings.iter().any(|f| f.pid == orphan_pid), "the orphan must produce a finding" ); - assert!( - !findings.iter().any(|f| f.pid == state_pid), - "the state-named pid must produce no finding" - ); assert!( !findings.iter().any(|f| f.pid == lock_pid), "the lock-held pid must produce no finding" ); - // Reap all three with a VERIFIED signal, never a bare one + // Reap both with a VERIFIED signal, never a bare one // (mirrors `reap_strays_e2e.rs:202-223`'s own teardown), plus a // final `wait()` on each to reclaim the zombie regardless. - for pid in [state_pid, lock_pid, orphan_pid] { + for pid in [lock_pid, orphan_pid] { agent::terminate_and_verify( pid, agent::TERMINATE_VERIFY_WAIT, agent::TERMINATE_VERIFY_POLL, ); } - let _ = state_child.wait(); let _ = lock_child.wait(); let _ = orphan_child.wait(); } @@ -9893,13 +9880,21 @@ mod tests { let mut child = spawn_wrapper_shaped_fixture(); let pid = child.id(); - let mut state = State::new(PhaseId::new(1), AgentKind::Claude, Mode::Auto, root.clone()); - state.monitor_pid = Some(pid); + let state = State::new(PhaseId::new(1), AgentKind::Claude, Mode::Auto, root.clone()); workflow::save_state(&state).unwrap(); + // The pid's path into the reachable set: a lock file naming it as + // holder, in the `.devflow/lock-{phase:02}` shape (pid, start time). + let start_time = agent::process_start_time(pid) + .expect("must read the lock fixture's own recorded start time"); + std::fs::write( + root.join(".devflow").join("lock-01"), + format!("{pid}\n{start_time}"), + ) + .unwrap(); assert!( registry_reachable_pids(std::slice::from_ref(&root)).contains(&pid), - "the pid must be reachable while its root and state file still exist" + "the pid must be reachable while its root, state file and lock file still exist" ); std::fs::remove_dir_all(&root).unwrap(); diff --git a/crates/devflow-cli/src/preflight.rs b/crates/devflow-cli/src/preflight.rs index b78cd29b..fab5dd14 100644 --- a/crates/devflow-cli/src/preflight.rs +++ b/crates/devflow-cli/src/preflight.rs @@ -2202,7 +2202,7 @@ mod tests { // from the constant; 34-05's widening puts it on the stream path, // where `canary_gate` invokes the real `ClaudeCanaryLauncher` and the // launch fails on a delivery refusal unrelated to launch counting. - // The `launches == 1` and `monitor_pid.is_some()` assertions below + // The `launches == 1` and `supervisor.is_some()` assertions below // are untouched, so a launch must still genuinely happen. state.legacy_claude_launch = true; workflow::save_state(&state).unwrap(); @@ -2245,7 +2245,7 @@ mod tests { exactly once, not {launches}" ); assert!( - state.monitor_pid.is_some(), + state.supervisor.is_some(), "this test is expected to drive a real monitor::spawn_monitor \ through run_preflight's Advance arm — None here means WR-05's \ premise was wrong and the guard above reaped nothing" @@ -2276,7 +2276,7 @@ mod tests { // (34-06); do not "simplify" this away. Same reasoning as the Advance // arm's sibling above: the subject is the launch COUNT through the // LoopBack arm, not which launch path is taken, and the `launches == 1` - // and `monitor_pid.is_some()` assertions below are untouched. + // and `supervisor.is_some()` assertions below are untouched. state.legacy_claude_launch = true; workflow::save_state(&state).unwrap(); @@ -2324,7 +2324,7 @@ mod tests { exactly once, not {launches}" ); assert!( - state.monitor_pid.is_some(), + state.supervisor.is_some(), "this test is expected to drive a real monitor::spawn_monitor \ through run_preflight's LoopBack arm — None here means WR-05's \ premise was wrong and the guard above reaped nothing" @@ -2436,7 +2436,7 @@ mod tests { "a human Advance must reset the retry counter" ); assert!( - state.monitor_pid.is_some(), + state.supervisor.is_some(), "this test's Advance arm is expected to drive a real \ monitor::spawn_monitor via launch_stage_inner — None here means \ this residual finding's premise was wrong and the guard above \ diff --git a/crates/devflow-cli/tests/phase7_cli.rs b/crates/devflow-cli/tests/phase7_cli.rs index 7001e392..547a6ff1 100644 --- a/crates/devflow-cli/tests/phase7_cli.rs +++ b/crates/devflow-cli/tests/phase7_cli.rs @@ -621,14 +621,14 @@ fn start_until_plan_halts_cleanly() { ); assert!(state.stopped, "stop marker must be set"); assert_eq!( - state.monitor_pid, None, - "the stop path must clear monitor_pid so no monitor is left behind" + state.supervisor, None, + "the stop path must clear the supervisor record so no monitor is left behind" ); assert!( state.stop_reason.is_some(), "a human-readable stop_reason must be recorded" ); - // The stop path cleared monitor_pid; the guard still binds (no-op) so + // The stop path cleared the supervisor record; the guard still binds (no-op) so // every monitor-spawning test in this file follows the same teardown // shape (41-02 Task 1 systematic pass). let _reap = MonitorReapGuard::after_launch(&state); @@ -1005,8 +1005,8 @@ fn start_codex_without_context_fails_preflight() { /// 20b instance 1 (D-06, review: Codex HIGH fail-closed-on-live-agent): /// `cleanup --force` must refuse to remove a worktree whose agent pid is -/// genuinely alive, even when the persisted `State` carries `monitor_pid = -/// None` — a classification `liveness()` reports as `Unknown`, NOT `Healthy`. +/// genuinely alive, even when the persisted `State` carries no supervisor +/// record — a run with no addressable monitor, NOT a healthy one. /// A guard that only refuses on `Healthy`/`BetweenStages` would still delete /// this worktree out from under a live agent. Against unmodified `cleanup` /// (no liveness check at all today) this test FAILS: cleanup removes the @@ -1032,7 +1032,7 @@ fn cleanup_force_refuses_on_live_agent_unknown_monitor() { fs::create_dir_all(pid_path.parent().unwrap()).unwrap(); fs::write(&pid_path, std::process::id().to_string()).unwrap(); - // Persist a State with monitor_pid = None (Unknown liveness) and + // Persist a State with no supervisor record (no addressable monitor) and // worktree_path pointing at the created worktree (the worktree->phase // join key, review: Codex MEDIUM). let mut state = devflow_core::state::State::new( @@ -1056,7 +1056,7 @@ fn cleanup_force_refuses_on_live_agent_unknown_monitor() { assert!( !output.status.success(), "cleanup --force must refuse to remove a live agent's worktree even \ - under Unknown liveness (monitor_pid = None)" + with no supervisor record" ); let combined = format!( "{}{}", @@ -1098,8 +1098,8 @@ fn cleanup_force_refuses_on_dead_monitor_live_agent() { fs::create_dir_all(pid_path.parent().unwrap()).unwrap(); fs::write(&pid_path, std::process::id().to_string()).unwrap(); - // Persist a State with a DEAD monitor pid (`liveness()` classifies this - // Stuck, since monitor_alive is false), plus worktree_path. + // Persist a State whose supervisor record names a socket that does not + // exist (a dead monitor: the probe reads GONE), plus worktree_path. let mut state = devflow_core::state::State::new( phase, devflow_core::state::AgentKind::Claude, @@ -1107,7 +1107,13 @@ fn cleanup_force_refuses_on_dead_monitor_live_agent() { root.to_path_buf(), ); state.worktree_path = Some(wt_path.clone()); - state.monitor_pid = Some(0x7FFF_FFFE); // essentially never a live pid + state.supervisor = Some(devflow_core::state::SupervisorRecord::new( + root.join("dead-monitor-never-bound.sock"), + "deadmonitor00000".to_string(), + 0x7FFF_FFFE, // essentially never a live process group + 1, + String::new(), + )); devflow_core::workflow::save_state(&state).unwrap(); let fake_bin = fake_bin_dir(&[]); @@ -1140,7 +1146,7 @@ fn cleanup_force_refuses_on_dead_monitor_live_agent() { } /// CR-02 (phase 20 review): a phase halted via `devflow start --until -/// ` clears `monitor_pid` and its agent has already exited by +/// ` clears the supervisor record and its agent has already exited by /// design — `Liveness::Unknown` with `agent_alive == false` sails straight /// through the live-agent refusal, so an ordinary `devflow cleanup` (no /// `--force`) must not delete the worktree of a phase the operator parked @@ -1163,7 +1169,7 @@ fn cleanup_keeps_worktree_for_until_stopped_phase_without_force() { devflow_core::worktree::add(root, &wt_path, &branch, &branch, false).unwrap(); // No agent pid file (the stage's agent has already exited normally) and - // monitor_pid = None (cleared by the --until stop path) — Unknown + // no supervisor record (cleared by the --until stop path) — Unknown // liveness, agent_alive == false. Only `state.stopped` distinguishes // this from a genuinely dead, safe-to-remove phase. let mut state = devflow_core::state::State::new( @@ -1272,8 +1278,9 @@ fn cleanup_is_idempotent_when_worktree_already_removed() { .join(format!("phase-{padded}", padded = phase.padded())); devflow_core::worktree::add(root, &wt_path, &branch, &branch, false).unwrap(); - // Dead agent, dead monitor (Stuck liveness) — a genuinely dead phase, - // safe for cleanup to proceed. + // Dead agent, dead monitor (a supervisor record whose socket does not + // exist, so the probe reads GONE) — a genuinely dead phase, safe for + // cleanup to proceed. let mut state = devflow_core::state::State::new( phase, devflow_core::state::AgentKind::Claude, @@ -1281,7 +1288,13 @@ fn cleanup_is_idempotent_when_worktree_already_removed() { root.to_path_buf(), ); state.worktree_path = Some(wt_path.clone()); - state.monitor_pid = Some(0x7FFF_FFFE); + state.supervisor = Some(devflow_core::state::SupervisorRecord::new( + root.join("dead-monitor-never-bound.sock"), + "deadmonitor00000".to_string(), + 0x7FFF_FFFE, + 1, + String::new(), + )); devflow_core::workflow::save_state(&state).unwrap(); let fake_bin = fake_bin_dir(&[]); @@ -1336,7 +1349,7 @@ fn pi_stub(launch: &str) -> String { /// Wait until a phase's chain settles in a state the tests can reap from: /// either the never-silent gate fired (`gate_pending`) or the machine /// stopped (`--until` / completion). Returns that state — whose -/// `monitor_pid` is the LAST monitor the chain spawned, the one a +/// `supervisor` record names the LAST monitor the chain spawned, the one a /// `MonitorReapGuard` must capture (a guard bound to an early state read /// reaps an already-exited pid and leaks the gate-waiting monitor, HYG-01). fn wait_for_settled(root: &Path, phase: PhaseId) -> devflow_core::state::State { @@ -1573,26 +1586,39 @@ enum StubMode { /// of the binary crate's `ReapMonitorOnDrop`, built on the PUBLIC /// `devflow_core::agent` surface (integration tests cannot reach /// `devflow-cli`'s test_support). TERM->KILL escalation with VERIFIED death, -/// keyed to `state.monitor_pid`, bound strictly after the final +/// keyed to the pid of the supervisor `state.supervisor` records (read from +/// its socket by [`live_supervisor_pid`]), bound strictly after the final /// `&mut State` use. 41-02 Task 1 turns this into the systematic pass. struct MonitorReapGuard { pid: Option, } +/// The pid of the supervisor `state` records, from its socket's ping reply. +/// The state carries no monitor pid (50-22); before that field went, it held +/// this same pid. `None` when the state has no record or its socket does not +/// answer as alive: nothing is left for a guard to reap. +fn live_supervisor_pid(state: &devflow_core::state::State) -> Option { + let record = state.supervisor.as_ref()?; + match devflow_core::supervisor::probe(&record.socket_path) { + Ok(devflow_core::supervisor::Liveness::Alive(reply)) => Some(reply.supervisor_pid), + _ => None, + } +} + impl MonitorReapGuard { fn after_launch(state: &devflow_core::state::State) -> Self { // Register the pid THIS guard is responsible for (the settled-state // monitor it will reap), and mark a guard in flight for the audit's // ordering barrier. `state` must be the SETTLED state - // (`wait_for_settled` / `wait_for_gate`), whose `monitor_pid` is the - // chain's LAST monitor — the one that would leak if not reaped. - if let Some(pid) = state.monitor_pid { + // (`wait_for_settled` / `wait_for_gate`), whose supervisor record + // names the chain's LAST monitor — the one that would leak if not + // reaped. + let pid = live_supervisor_pid(state); + if let Some(pid) = pid { registry().lock().unwrap().insert(pid); ACTIVE_GUARDS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); } - Self { - pid: state.monitor_pid, - } + Self { pid } } } @@ -1847,9 +1873,8 @@ fn unguarded_monitor_is_detected_by_the_registry() { let _reaper = common::LaunchedRunReaper::new(root, PhaseId::new(7), Vec::new()); let state = load_state(root, PhaseId::new(7)).expect("load state"); - let pid = state - .monitor_pid - .expect("a launched stage must record a monitor pid"); + let pid = live_supervisor_pid(&state) + .expect("a launched stage must record a supervisor that answers its socket"); assert!( devflow_core::agent::agent_running(pid), "premise: with no guard the monitor stays alive — nothing reaps it" diff --git a/crates/devflow-cli/tests/stop_e2e.rs b/crates/devflow-cli/tests/stop_e2e.rs index 784070e3..0327b6c5 100644 --- a/crates/devflow-cli/tests/stop_e2e.rs +++ b/crates/devflow-cli/tests/stop_e2e.rs @@ -169,10 +169,11 @@ fn stop_writes_no_state_while_the_lock_holder_survives_the_signal() { /// T-50-38 (renamed from `stop_never_signals_the_recorded_monitor_pid`): /// `stop` never signals a pid read from state; it addresses the socket. The -/// state plants a live test-owned process as `monitor_pid` and as the agent -/// group of a `supervisor` record whose socket file does not exist, so the -/// probe reads GONE: `stop` reports nothing running and the planted process -/// is still alive afterwards. +/// state plants a live test-owned process as the agent group of a +/// `supervisor` record whose socket file does not exist, so the probe reads +/// GONE: `stop` reports nothing running and the planted process is still +/// alive afterwards. Also covers T-23-51's unit test (removed in 50-22): a +/// pid recorded in state alone is never a signalling target or a blocker. #[test] fn stop_never_signals_a_recorded_pid_it_addresses_the_socket() { let home = common::HermeticHome::new(); @@ -185,7 +186,6 @@ fn stop_never_signals_a_recorded_pid_it_addresses_the_socket() { .expect("spawn the planted process"); let start = devflow_core::agent::process_start_time(planted.id()).expect("start time"); let mut state = State::new(phase, AgentKind::Claude, Mode::Auto, root.to_path_buf()); - state.monitor_pid = Some(planted.id()); state.supervisor = Some(SupervisorRecord::new( home.cache_dir().join("sock").join("missing.sock"), "launch-missing".to_string(), diff --git a/crates/devflow-cli/tests/supervisor_e2e.rs b/crates/devflow-cli/tests/supervisor_e2e.rs index b2a63b79..1a1f1493 100644 --- a/crates/devflow-cli/tests/supervisor_e2e.rs +++ b/crates/devflow-cli/tests/supervisor_e2e.rs @@ -189,11 +189,6 @@ fn start_publishes_for(shape: Shape, phase: PhaseId) { "{shape:?}: socket path is {} bytes", socket.as_os_str().len() ); - assert!( - state.monitor_pid.is_some(), - "{shape:?}: monitor_pid stays populated until 50-22" - ); - let reply = match supervisor::probe(&socket).expect("probe the recorded socket") { Liveness::Alive(reply) => reply, other => panic!("{shape:?}: the recorded socket must answer ALIVE, got {other:?}"), @@ -201,17 +196,16 @@ fn start_publishes_for(shape: Shape, phase: PhaseId) { assert_eq!(reply.launch, shape.label(), "{shape:?}: launch label"); assert_eq!(reply.launch_id, record.launch_id, "{shape:?}: launch id"); assert_eq!(reply.agent_pgid, record.agent_pgid, "{shape:?}: agent pgid"); - assert_eq!( - Some(reply.supervisor_pid), - state.monitor_pid, - "{shape:?}: monitor_pid names the supervisor" - ); let sidecar_path = socket.with_extension("json"); let sidecar = supervisor::read_sidecar(&sidecar_path).expect("read the sidecar"); assert_eq!(sidecar.project_root, root, "{shape:?}: sidecar root"); assert_eq!(sidecar.phase, phase, "{shape:?}: sidecar phase"); assert_eq!(sidecar.launch, shape.label(), "{shape:?}: sidecar launch"); + assert_eq!( + sidecar.supervisor_pid, reply.supervisor_pid, + "{shape:?}: the recorded socket answers for the supervisor its sidecar names" + ); match supervisor::request_shutdown(&socket, SHUTDOWN_WAIT).expect("shutdown the run") { ShutdownOutcome::Stopped { .. } => {} From b46ece121c87c1f9cc3eed4778e547733566065c Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 14:14:28 -0400 Subject: [PATCH 084/112] refactor(50-16)!: remove State's monitor pid field - Delete `State::monitor_pid`, its initialiser and the state.rs tests that pinned it; rewrite the round-trip test around `supervisor` and assert the removed key is absent from persisted JSON (R-9, D-20: no shim or alias) - Add doc_check::no_monitor_pid_field_json_key_or_help_mention_remains (T-50-43): declaration, quoted JSON key in either crate, help snapshot, with planted controls - semver-gate.sh check --through 50-16: SEMVER verdict=match BREAKING CHANGE: devflow_core::state::State no longer has a pub `monitor_pid` field; liveness is the supervisor socket record only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-core/src/agent.rs | 6 +- crates/devflow-core/src/doc_check.rs | 174 ++++++++++++++++++++++++++- crates/devflow-core/src/state.rs | 67 +++++------ 3 files changed, 205 insertions(+), 42 deletions(-) diff --git a/crates/devflow-core/src/agent.rs b/crates/devflow-core/src/agent.rs index eb60007e..7e2e2ab0 100644 --- a/crates/devflow-core/src/agent.rs +++ b/crates/devflow-core/src/agent.rs @@ -405,9 +405,9 @@ const ADVANCE_SUBCOMMAND: &str = "advance"; /// found through its socket instead — `doctor`'s supervisor findings, /// including the deleted-root class this census used to cover — and ended /// with `devflow stop --phase N --launch `. Do not add a `__monitor` -/// matcher: a live supervisor's pid is in no reachability safety set once -/// `monitor_pid` leaves it (50-22), so `--reap-strays` would treat every -/// live run as a stray. +/// matcher: a live supervisor's pid is in no reachability safety set, since +/// the state records no monitor pid (50-22, 50-16), so `--reap-strays` would +/// treat every live run as a stray. /// /// This is a pure, read-only survey: it never signals a process. Deciding /// whether to act on a result, and re-confirming identity immediately diff --git a/crates/devflow-core/src/doc_check.rs b/crates/devflow-core/src/doc_check.rs index bcacde05..786c66ac 100644 --- a/crates/devflow-core/src/doc_check.rs +++ b/crates/devflow-core/src/doc_check.rs @@ -78,6 +78,12 @@ fn read_files(paths: impl IntoIterator) -> String { } fn rust_source() -> String { + read_files(rust_file_paths()) +} + +/// Every `.rs` file under `/crates`, both crates, src and tests, +/// in sorted order. +fn rust_file_paths() -> Vec { fn visit(dir: &Path, files: &mut Vec) { let Ok(entries) = std::fs::read_dir(dir) else { return; @@ -95,7 +101,7 @@ fn rust_source() -> String { let mut files = Vec::new(); visit(&workspace_root().join("crates"), &mut files); files.sort(); - read_files(files) + files } fn extract_tokens(text: &str, prefix: &str, allowed: impl Fn(char) -> bool) -> Vec { @@ -499,3 +505,169 @@ fn devflow_core_dependency_pin_matches_workspace_version() { "workspace.dependencies.devflow-core's pinned version must match workspace.package.version" ); } + +/// The `State` field 50-16 removed (R-9), assembled from pieces so this file +/// never holds the name contiguously and the scan below cannot flag itself. +const REMOVED_STATE_FIELD: &str = concat!("monitor", "_pid"); + +/// Lines that survive comment stripping, numbered from 1: a line whose trimmed +/// start is `//` (doc comments included) is dropped. +fn uncommented_lines(text: &str) -> impl Iterator { + text.lines() + .enumerate() + .filter(|(_, line)| !line.trim_start().starts_with("//")) + .map(|(index, line)| (index + 1, line)) +} + +fn is_identifier_char(ch: char) -> bool { + ch.is_alphanumeric() || ch == '_' +} + +/// True when `line`'s trimmed start is `pub ` plus `field`, followed by a +/// non-identifier character (or nothing): the declaration, not a longer name. +fn declares_pub_field(line: &str, field: &str) -> bool { + line.trim_start() + .strip_prefix("pub ") + .and_then(|rest| rest.strip_prefix(field)) + .is_some_and(|after| { + after + .chars() + .next() + .is_none_or(|ch| !is_identifier_char(ch)) + }) +} + +/// True when `line` holds `field` wrapped in double quotes — a JSON or event +/// key — whether the closing quote is bare (`"k"`, raw strings, `json!`) or +/// escaped inside an ordinary string literal (`"{\"k\": 1}"`). +fn holds_quoted_key(line: &str, field: &str) -> bool { + let opened = ["\"", field].concat(); + line.match_indices(&opened).any(|(start, _)| { + let after = &line[start + opened.len()..]; + after.starts_with('"') || after.starts_with("\\\"") + }) +} + +/// Every place the removed field survives (T-50-43): (a) its `pub` +/// declaration on an uncommented line of state.rs, (b) its quoted key on an +/// uncommented line of any Rust file, (c) any mention in a snapshot file. +fn removed_field_findings( + state_source: &str, + rust_files: &[(PathBuf, String)], + snapshots: &[(PathBuf, String)], +) -> Vec { + let field = REMOVED_STATE_FIELD; + let mut findings = Vec::new(); + for (number, line) in uncommented_lines(state_source) { + if declares_pub_field(line, field) { + findings.push(format!("state.rs:{number}: declaration: {}", line.trim())); + } + } + for (path, text) in rust_files { + for (number, line) in uncommented_lines(text) { + if holds_quoted_key(line, field) { + findings.push(format!( + "{}:{number}: quoted key: {}", + path.display(), + line.trim() + )); + } + } + } + for (path, text) in snapshots { + if text.contains(field) { + findings.push(format!("{}: snapshot mention", path.display())); + } + } + findings +} + +fn read_labelled(paths: Vec) -> Vec<(PathBuf, String)> { + paths + .into_iter() + .map(|path| { + let text = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())); + (path, text) + }) + .collect() +} + +/// T-50-43: the socket is the only liveness source once `State`'s monitor +/// pid field is gone (R-9, 50-16). The compiler proves no Rust code reads or +/// writes the field; this scan covers what it cannot see — the declaration +/// itself, a JSON or event key of that name in either crate, and a mention in +/// the CLI's help snapshot. Planted controls prove each check decides. +#[test] +fn no_monitor_pid_field_json_key_or_help_mention_remains() { + let field = REMOVED_STATE_FIELD; + let none: &[(PathBuf, String)] = &[]; + let planted = |text: String| vec![(PathBuf::from("planted.rs"), text)]; + + // (a) declaration: flagged; a longer name and a commented one are not. + let declared = [" pub ", field, ": Option,"].concat(); + assert_eq!(removed_field_findings(&declared, none, none).len(), 1); + let longer = [" pub ", field, "_launch: Option,"].concat(); + assert!(removed_field_findings(&longer, none, none).is_empty()); + let commented = [" // pub ", field, ": Option,"].concat(); + let doc_commented = [" /// pub ", field, ": Option,"].concat(); + assert!(removed_field_findings(&commented, none, none).is_empty()); + assert!(removed_field_findings(&doc_commented, none, none).is_empty()); + + // (b) quoted key: flagged on a code line, bare or escaped. + let raw_key = [" let json = r#\"{\"", field, "\": 4242}\"#;"].concat(); + assert_eq!(removed_field_findings("", &planted(raw_key), none).len(), 1); + let escaped_key = [" let json = \"{\\\"", field, "\\\": 4242}\";"].concat(); + assert_eq!( + removed_field_findings("", &planted(escaped_key), none).len(), + 1 + ); + // ...and not on a comment line, as a local, in a helper's name, or as a + // prefix of a longer quoted string. + for clean in [ + [" // json.contains(\"", field, "\")"].concat(), + [" let ", field, " = 4242;"].concat(), + [" fn wait_for_", field, "_exit() {}"].concat(), + [" let message = \"", field, " must not appear\";"].concat(), + ] { + assert!( + removed_field_findings("", &planted(clean.clone()), none).is_empty(), + "planted clean line was flagged: {clean}" + ); + } + + // (c) snapshot mention: flagged. + let snapshot = vec![( + PathBuf::from("planted-help.txt"), + [" --", field, " \n"].concat(), + )]; + assert_eq!(removed_field_findings("", none, &snapshot).len(), 1); + + // The real scan: state.rs, every Rust file under crates/, every snapshot. + let root = workspace_root(); + let state_source = std::fs::read_to_string(root.join("crates/devflow-core/src/state.rs")) + .expect("state.rs must be readable"); + let rust_files = read_labelled(rust_file_paths()); + assert!(!rust_files.is_empty(), "the walker must find Rust sources"); + let snapshot_dir = root.join("crates/devflow-cli/tests/snapshots"); + let mut snapshot_paths: Vec = std::fs::read_dir(&snapshot_dir) + .unwrap_or_else(|error| panic!("{} must exist: {error}", snapshot_dir.display())) + .flatten() + .map(|entry| entry.path()) + .filter(|path| path.is_file()) + .collect(); + snapshot_paths.sort(); + assert!( + !snapshot_paths.is_empty(), + "{} must hold at least one snapshot", + snapshot_dir.display() + ); + let snapshots = read_labelled(snapshot_paths); + + let findings = removed_field_findings(&state_source, &rust_files, &snapshots); + assert!( + findings.is_empty(), + "the removed State field survives (T-50-43):\n{}", + findings.join("\n") + ); +} diff --git a/crates/devflow-core/src/state.rs b/crates/devflow-core/src/state.rs index e91a1a46..fffab24e 100644 --- a/crates/devflow-core/src/state.rs +++ b/crates/devflow-core/src/state.rs @@ -24,8 +24,9 @@ use crate::stage::Stage; /// state files written by older binaries still load. /// /// This exists because `State` accumulates a field roughly every phase that -/// adds a run-scoped concept (`worktree_path`, `monitor_pid`, `stop_until`, -/// `yes_ship`, and — in phase 28 — `session_id` and `checkpoint_resumes`). +/// adds a run-scoped concept (`worktree_path`, `stop_until`, `yes_ship`, in +/// phase 28 `session_id` and `checkpoint_resumes`, and in phase 50 +/// `supervisor`). /// Without `non_exhaustive`, each of those additions is a semver-breaking /// change for any consumer that used a struct literal, which would force a /// major bump for what is really an internal bookkeeping change. Paying that @@ -279,13 +280,6 @@ pub struct State { /// always live under the main `project_root`; only the agent's cwd changes. #[serde(default)] pub worktree_path: Option, - /// PID of the detached monitor process that owns the agent for the - /// current stage, recorded by `launch_stage` at spawn time. `None` means - /// no monitor has been spawned for this state yet, OR the state was - /// written by a binary predating this field — in both cases the - /// liveness probe reports Unknown, never Stuck. - #[serde(default)] - pub monitor_pid: Option, /// The published, ping-confirmed address of the supervisor that owns the /// current stage (D-04): its socket path, launch id, and the agent /// identity the stale-socket backstop validates. Written by the launcher, @@ -642,7 +636,6 @@ impl State { started_at: timestamp_now(), project_root, worktree_path: None, - monitor_pid: None, supervisor: None, session_id: None, checkpoint_resumes: 0, @@ -726,7 +719,7 @@ mod tests { assert_eq!(state.preflight_retries, 0); assert_eq!(state.phase_validate_failures, 0); assert!(!state.started_at.is_empty()); - assert_eq!(state.monitor_pid, None); + assert_eq!(state.supervisor, None); assert_eq!(state.stop_until, None); assert!(!state.stopped); assert_eq!(state.stop_reason, None); @@ -1226,26 +1219,41 @@ mod tests { assert_eq!(loaded_absent.preflight_retries, 0); } - /// `monitor_pid` round-trips through serde as an exact `u32` (18b). + /// The persisted JSON carries the `supervisor` record and no key for the + /// monitor pid field 50-16 removed (R-9): the socket record is the only + /// liveness source a state file holds. The removed key is built from + /// pieces so this file holds no quoted copy of it (T-50-43). #[test] - fn monitor_pid_round_trips_through_serde() { + fn persisted_state_carries_the_supervisor_and_no_monitor_pid_key() { let mut state = State::new( PhaseId::new(1), AgentKind::Claude, Mode::Auto, PathBuf::from("/repo"), ); - state.monitor_pid = Some(4242); + let record = SupervisorRecord::new( + PathBuf::from("/c/sock/0123456789abcdef-1-00000000000000ab.sock"), + "00000000000000ab".to_string(), + 4242, + 123_456, + "boot-id".to_string(), + ); + state.supervisor = Some(record.clone()); let json = serde_json::to_string(&state).unwrap(); assert!( - json.contains("monitor_pid"), - "monitor_pid must appear in persisted JSON" + json.contains("\"supervisor\""), + "the supervisor key must appear in persisted JSON" + ); + let removed_key = ["\"", concat!("monitor", "_pid"), "\""].concat(); + assert!( + !json.contains(&removed_key), + "the removed monitor pid key must not appear in persisted JSON: {json}" ); let loaded: State = serde_json::from_str(&json).unwrap(); assert_eq!( - loaded.monitor_pid, - Some(4242), - "monitor_pid must round-trip through serde" + loaded.supervisor, + Some(record), + "the supervisor record must round-trip through serde" ); } @@ -1290,25 +1298,8 @@ mod tests { assert_eq!(loaded.supervisor, None, "an absent key reads as None"); } - /// A serde-absent `monitor_pid` (state written by a pre-18b binary) must - /// deserialize to `None`, not `Some(0)` — a `Some(0)` default would let a - /// pre-18b state file render as a monitor at pid 0. - #[test] - fn monitor_pid_absent_from_json_defaults_to_none() { - let json = r#"{ - "stage": "code", - "phase": 1, - "agent": "claude", - "mode": "auto", - "started_at": "0", - "project_root": "/repo" - }"#; - let loaded: State = serde_json::from_str(json).unwrap(); - assert_eq!(loaded.monitor_pid, None); - } - /// `session_id` round-trips through serde as an exact `Option` - /// (D-04, 28-02) — mirrors the `monitor_pid` pair above. + /// (D-04, 28-02) — mirrors the `supervisor` round-trip above. #[test] fn session_id_round_trips_through_serde() { let mut state = State::new( @@ -1388,7 +1379,7 @@ mod tests { /// 23-09 Task 1: `yes_ship` round-trips through serde as an exact `bool` /// — its own key appears in the persisted JSON, and a fresh deserialize - /// recovers the value set, mirroring the `monitor_pid` pair above. + /// recovers the value set, mirroring the `supervisor` round-trip above. #[test] fn yes_ship_round_trips_through_serde() { let mut state = State::new( From 8b291c4b2efae556a58e705fa09ad988e96c4ee5 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 14:26:31 -0400 Subject: [PATCH 085/112] fix(50-16): skip the 0o000 socket-dir tests when running as root - The pinned container runs check.sh as uid 0, which lists a 0o000 directory, so doctor_reports_an_unlistable_socket_dir_instead_of_nothing (50-12) and the_launch_guard_refuses_an_unlistable_socket_dir_over_an_ unloadable_state (50-15) failed their own fixture precondition there - Skip under uid 0, the 999.113 pattern already used in ship.rs and supervisor.rs; on a non-root host both still run in full - scripts/check-in-container.sh: exit 0 (454 passed, 0 failed) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- crates/devflow-cli/src/commands.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 220700b0..23e2b2a5 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -8815,6 +8815,15 @@ mod tests { fn doctor_reports_an_unlistable_socket_dir_instead_of_nothing() { use std::os::unix::fs::PermissionsExt as _; let cache = tempfile::tempdir().unwrap(); + // Root lists a 0o000 directory (the pinned container runs as + // uid 0), so the fixture cannot produce its precondition there; + // skip rather than assert it (the 999.113 pattern, 50-16). + if std::os::unix::fs::MetadataExt::uid(&std::fs::metadata(cache.path()).unwrap()) == 0 { + eprintln!( + "skipping doctor_reports_an_unlistable_socket_dir_instead_of_nothing: running as root, which bypasses the 0o000 fixture" + ); + return; + } assert_eq!(socket_dir_unlistable(cache.path()), None, "absent"); assert_eq!(render_orphan_supervisors_text(&[], None), ""); let sock = registry::socket_dir_in(cache.path()); @@ -10949,6 +10958,15 @@ mod tests { let phase = PhaseId::new(1501); let root = root_with_an_unloadable_state(phase); let cache = tempfile::tempdir().unwrap(); + // Root lists a 0o000 directory (the pinned container runs as + // uid 0), so the fixture cannot produce its precondition there; + // skip rather than assert it (the 999.113 pattern, 50-16). + if std::os::unix::fs::MetadataExt::uid(&std::fs::metadata(cache.path()).unwrap()) == 0 { + eprintln!( + "skipping the_launch_guard_refuses_an_unlistable_socket_dir_over_an_unloadable_state: running as root, which bypasses the 0o000 fixture" + ); + return; + } let sock = registry::socket_dir_in(cache.path()); std::fs::create_dir_all(&sock).unwrap(); std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o000)).unwrap(); From e5cc48d01389ee448ff828438924399a3ec59c9a Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 14:37:25 -0400 Subject: [PATCH 086/112] docs(50-17): document run liveness and the socket-based stop arms - OPERATIONS.md: stop row rewritten (gate, then socket ALIVE/STALE/GONE, then lock holder; --launch), the __monitor paragraph for both launch shapes, a Run liveness section (status/doctor/cleanup/recover/start wording, orphan sweep, socket location, modes, 107-byte bound, signals) - remove the monitor-script, monitor_pid and stop-marker descriptions - doc_check: permanent scan pinning the liveness fragments, with planted controls for missing and comment-hidden fragments Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- OPERATIONS.md | 119 +++++++++++++++++++++++---- crates/devflow-core/src/doc_check.rs | 96 +++++++++++++++++++++ 2 files changed, 200 insertions(+), 15 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index 38e074ac..7e5679fb 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -30,7 +30,7 @@ Define → Plan → Code → Validate → Ship |---|---| | `devflow start --phase N --agent claude\|codex\|opencode --mode auto\|supervise [--force] [--no-worktree] [--dry-run] [--until define\|plan\|code\|validate]` | Begin a phase; a detached monitor owns the agent and auto-advances. `--until ` runs through `` then halts cleanly (no orphaned monitor, no `doctor` false positive) instead of continuing to Ship; `--until ship` is rejected (a no-op — Ship is already terminal). Refuses before scaffolding anything if phase N is not reachable from `develop` (see Preflight refusals below) | | `devflow resume --phase N` | Resume a rate-limited or infra-paused phase from its saved stage (loads `state-NN.json`, does not recreate the branch/worktree or reset to Define) — the command a primary-loop rate-limit auto-resume cron invokes. Also clears a `--until`-stopped phase's stop marker, so the phase advances past its old stop point instead of re-stopping immediately | -| `devflow status` | All active phases: stage, gate state, agent liveness, elapsed, last action | +| `devflow status` | All active phases: stage, gate state, supervisor liveness (ALIVE / STALE / GONE, with the repair; see [Run liveness](#run-liveness)), elapsed, last action | | `devflow logs [-f] [--phase N] [--stderr]` | Print/follow a phase's captured agent output | | `devflow history [N]` | Show chronological events with retained capture and review evidence for a phase | | `devflow gate list` | Gates awaiting a response | @@ -40,23 +40,112 @@ Define → Plan → Code → Validate → Ship | `devflow parallel --phases 7,8 [--agents claude,codex] [--mode M] [--force]` | Run phases concurrently, each in its own worktree + monitor | | `devflow list` | Feature branches with divergence from develop | | `devflow reference [--branch B] [--refresh]` | Static snapshot worktree at `.worktrees/reference/` | -| `devflow stop --phase N [PROJECT] [--root PATH]` | End a running phase cleanly (23c). Answers its open gate with a rejection if one is open — the target unwinds through its own abort path, no signal sent — otherwise signals the process recorded in `.devflow/lock-{phase:02}` (never `state.monitor_pid`, which the generated monitor script's trap only ever captures for the agent, not the trailing `advance`) after confirming it is alive and its `/proc//cmdline` identifies it as belonging to DevFlow. Idempotent; marks `state.stopped`/`state.stop_reason` so `cleanup`'s existing fail-closed refusal (unweakened by this command) recognizes the phase as no longer live — `stop` then `cleanup --force` compose in that order | +| `devflow stop --phase N [PROJECT] [--root PATH] [--launch ID]` | End a running phase cleanly (23c). In this order: **(1)** an open gate with a confirmed live waiter is answered with a rejection — the waiter unwinds through its own abort path, no signal sent. **(2)** Otherwise the run's supervisor socket is probed: the one `.devflow/state-NN.json` records, or, when that file is missing or does not load, the one live socket found through its sidecar for this project and phase. **ALIVE** → a `shutdown` request: the supervisor ends the agent's process group (exit code 143) and runs no advance. **STALE** → the guarded backstop: it signals (SIGTERM, then SIGKILL) only an agent group whose leader is still present with the recorded start time and still leads the group; a socket from a previous boot, an empty group or a recycled leader pid has its socket and sidecar removed with nothing signalled; a group whose leader is gone is refused and its members listed, with `ps -o pid,pgid,sid,lstart,args -g ` to inspect them (residual limit: nothing proves those members are the agent's, so `stop` never signals them — end them yourself). A probe that cannot answer is refused, nothing signalled. **(3)** Only when no live socket remains (**GONE**, or nothing recorded) is the process holding `.devflow/lock-{phase:02}` signalled, after confirming it is alive and its `/proc//cmdline` identifies it as belonging to DevFlow. No pid read from the state file is ever signalled. Idempotent; records `stopped`/`stop_reason` so `cleanup`'s fail-closed refusal recognizes the phase as no longer live — `stop` then `cleanup --force` compose in that order. When the state file does not load, `stop` still ends the run, says `stopped` was not recorded and leaves the file untouched; several live sockets with no usable state are refused, with one `devflow stop --phase N --launch ` line each. `--launch ` tears down only the socket with that launch id — the form `devflow doctor` prints for an orphan, usable from any directory — and answers no gate, signals no lock holder and writes no state | | `devflow cleanup [--force]` | Remove phase worktrees + their feature branches | | `devflow recover [--clean] [--phase N]` | Inspect state; `--clean` sweeps stale phases only; `--clean --phase N` clears one phase unconditionally | | `devflow test` | cargo test + clippy + fmt --check | -| `devflow doctor [--json]` | Environment audit (agents installed, versions, RUST_LOG) plus per-phase reconciliation; `--json` emits one object `{"environment": [...], "reconciliation": [...]}` | +| `devflow doctor [--json]` | Environment audit (agents installed, versions, RUST_LOG) plus per-phase reconciliation, including each phase's supervisor liveness and a read-only sweep for orphan supervisor sockets (see [Run liveness](#run-liveness)); `--json` emits one object with the keys `environment`, `reconciliation`, `planning_doc_staleness`, `stray_processes`, `orphan_supervisors` and `orphan_supervisor_sweep_error` | | `devflow release --check` | Read-only release-cut preflight: workspace self-pin, develop/main divergence (no `git fetch` — reads already-fetched refs), crates.io publish order, and changelog-version agreement (999.96). `--check` is required; a bare `devflow release` is rejected toward the deferred release-cut executor (merge/tag/sync/publish, DEN-50) | | `devflow ship --phase N [--force]` | **Dead-monitor recovery.** `devflow gate approve` only *writes* the Ship gate's response file — a live monitor polling for it is what actually advances the workflow. If that monitor died before consuming the response (e.g. the machine restarted mid-pipeline), the approval sits unconsumed forever. `devflow ship` reads the already-written Ship response directly and drives the phase through the same terminal path (`finish_workflow`) the live monitor would have — requires `state.stage == Stage::Ship` and an existing request+response pair with no prior ack. `--force` is accepted for explicit operator intent but never skips the stage check, the per-phase lock, the gate-existence check, or the ack check — it can never be used to skip Validate or jump ahead of a healthy pipeline. If the Ship response routes to a rejection that loops back to Code, `devflow ship` launches a **new, detached monitor agent** to drive the retry — the command prints this explicitly so it is never a silently long-running process. | | `devflow evidence --phase N [--json] [--require-shipped] [--root PATH]` | **Read-only structural oracle (23-06):** reports DevFlow's own append-only record of whether a phase actually shipped, instead of trusting an agent-authored attestation document. `shipped` is strictly true only after the terminal-only `workflow_shipped` event has been emitted (the last step inside `finish_workflow_with_gate_timeout`, once the entire post-Ship hook batch has succeeded) — a phase halted by `--until ` always reports `shipped: false`, even though it emits the older, ambiguous `workflow_finished` event too (surfaced separately as `workflow_finished_seen`/`finished_reason` for corroboration, never consulted by `shipped`). `--require-shipped` exits non-zero unless `shipped` is true, so it is declarable as a `verify::external_verify_commands` Layer 0 probe (opt-in per phase; a phase's PLAN must declare it and the operator must approve it via `DEVFLOW_TRUST_EXTERNAL_VERIFY`) — a failed declared probe outranks every agent-controlled signal. | -(`devflow advance` is internal — invoked by monitors with `--phase N`.) +(`devflow advance` is internal and takes `--phase N`. A supervisor does not +run it as a command; it advances in-process. `devflow status` names it as a +repair only for a gate-pending phase whose supervisor is GONE.) (`devflow monitor` is internal and registered as the hidden `__monitor` -subcommand — it is the pipe-owning monitor's own process body, re-exec'd -detached by `spawn_monitor` for a Claude `stream-json` launch. It supervises -the agent's stdin/stdout, then calls the same `advance` the shell monitor's -tail invokes. Never run it by hand: it expects to own a child's pipes and -writes the phase's capture, exit-code and agent-pid files as it goes.) +subcommand — it is the stage supervisor's own process body. Every stage +launch re-execs it detached, in its own process group, in one of two launch +shapes: **capture** (agent stdin `/dev/null`, stdout and stderr to the phase +capture files) for adapters other than Claude and Antigravity (such as `codex` +and `opencode`), the +`--legacy-claude-launch` opt-out and a checkpoint resume; or **stream** (the +supervisor owns the agent's stdin and stdout pipes and applies the idle +timeout) for a Claude or Antigravity `stream-json` launch. Both shapes run the +same Rust supervisor: it starts the agent in the agent's own process group, +records its pid and exit code, answers on a per-launch socket (see [Run +liveness](#run-liveness)), and when the agent exits runs the advance +in-process; the next stage's supervisor publishes its socket before this one +removes its own. Never run it by hand: it expects to own the agent and writes +the phase's capture, exit-code and agent-pid files as it goes.) + +## Run liveness + +Whether a phase's run is alive is answered from its supervisor socket, never +from a pid. Each launch records its socket's path in `.devflow/state-NN.json` +(the `supervisor` record); `devflow status`, `doctor`, `stop`, `cleanup`, +`recover --clean`, `start` and `resume` probe that recorded path (a connect +and one `ping`) and get one of three answers: + +| State | Meaning | Repair | +|---|---|---| +| **ALIVE** | The socket answered the ping. The reply names the current owner (agent running, advancing, gate wait, …) and the launch shape. A moment between stages is ALIVE, because the supervisor advances in-process | None needed. To end the run: `devflow stop --phase N` | +| **STALE** | The socket file is still there and owned by you, but nothing accepts the connect: the supervisor died without cleaning up, and its agent group may still be running | `devflow stop --phase N` (the backstop ends a validated agent group and removes the socket and sidecar), then `devflow resume --phase N` | +| **GONE** | No file at the recorded path: the supervisor has exited. Normal when the phase is stopped; when it is not, nothing will advance the phase | `devflow resume --phase N` (when a gate is pending, answer it and run `devflow advance --phase N`) | + +How each command shows them: + +- **`devflow status`** prints `supervisor: — ` for each + phase, and under it the repair: `devflow stop --phase N` then + `devflow resume --phase N` for STALE; `devflow resume --phase N` (plus + `devflow advance --phase N` when a gate is pending) for GONE while the phase + is not stopped. A phase with no supervisor recorded reads `not launched`. +- **`devflow doctor`** reports STALE, GONE-while-not-stopped and a failed probe + as problems, naming the phase and launch id but no path. Its repairs: + `devflow stop --phase N && devflow resume --phase N` for STALE, + `devflow resume --phase N` for GONE, `devflow status` for a failed probe. + ALIVE, a stopped GONE and an unlaunched phase are silent. `doctor --json` + gives each phase a `supervisor` object (`liveness`, `owner`, `launch`, + `launch_id`). +- **"probe failed".** A probe that cannot decide (the path is not a socket, + belongs to another user, cannot be stat'ed or connected to, or the peer + gives no valid ping reply) reads `UNKNOWN — the probe failed: …`. It is + never guessed to be ALIVE or GONE: `status` names the socket to inspect, + `doctor`'s repair is `devflow status`, `stop` refuses and signals nothing, + and `cleanup` keeps the worktree unless `--force`. +- **`devflow cleanup`** refuses an ALIVE run even with `--force` — end it + first with `devflow stop --phase N`, or wait for it (the refusal's own text + still says `devflow resume`, which does not end a live run). It keeps a + STALE run's worktree, or one whose probe failed, unless `--force`. +- **`devflow recover --clean --phase N`** refuses while the phase's supervisor + is ALIVE and deletes nothing. +- **`devflow start` / `devflow resume`** refuse while a run is live: an ALIVE + socket, a STALE one unless the backstop's checks prove its agent group gone, + or a probe that fails. When the state file exists but does not load, they + search the socket directory for this project and phase instead, and refuse + when that directory exists but cannot be listed. The refusal names a + `devflow stop` line for each live socket. + +**Orphan sockets.** `devflow doctor` also lists, read-only, under +`supervisor sockets no state claims:`, each socket of this project whose state +is missing, does not load, records no supervisor or names another launch; +each socket of this project with a missing or unparsable sidecar; and, +machine-wide, each socket whose project root no longer exists. Each is +repaired by `devflow stop --phase N --launch `. `doctor` changes nothing; +`doctor --json` reports them as `orphan_supervisors`, and a socket directory +it cannot list as `orphan_supervisor_sweep_error` rather than as no orphans. +A launch caught between publishing its socket and saving its record can be +listed this way for a moment. + +**Where the sockets live.** Under `registry::cache_dir()/sock`: the cache +directory is `DEVFLOW_CACHE_DIR` when set, then `$XDG_CACHE_HOME/devflow`, +then `~/.cache/devflow`. A launch with none of `DEVFLOW_CACHE_DIR`, +`XDG_CACHE_HOME` and `HOME` set fails (`cannot resolve DevFlow's cache +directory for this launch`) instead of running without a socket. Each socket +is named `--.sock`, with a `.json` sidecar of +the same stem recording the launch (project root, phase, agent process group, +start time, boot id). The `sock` directory is mode 0700 and refused if it is a +symlink or owned by another user; sockets and sidecars are 0600. A socket path +longer than Linux's 107-byte `sun_path` limit is refused by name (`socket path +is N bytes, longer than the 107-byte sun_path limit`) before anything is +bound; point `DEVFLOW_CACHE_DIR` at a shorter directory if the cache path is +deep. + +**Signals.** `SIGTERM` or `SIGINT` sent to a supervisor process does what +`devflow stop`'s ALIVE arm does: a running agent's group is ended, no next +stage is launched, and `stopped` is recorded. A supervisor parked at a gate +clears the gate and records `stopped` rather than aborting the phase, so the +phase stays resumable. ## Preflight refusals @@ -83,9 +172,10 @@ devflow gate reject 15 --note "abort: wrong direction" # ends the ph `--stage` is only needed when one phase somehow has several open gates. Under the hood this writes `.devflow/gates/NN-.response.json` -atomically; the blocked monitor polls it (exponential backoff, so pickup can -take up to ~60s), acks, and moves on. The CLI refuses to overwrite an -unconsumed response. +atomically; the supervisor parked at the gate checks for it about once a +second (a gate waited on outside a supervisor, such as `devflow ship`'s +foreground wait, backs off up to ~60s), acks, and moves on. The CLI refuses to +overwrite an unconsumed response. ## Notify hook (never miss a gate) @@ -134,7 +224,7 @@ never receives the variable. | `DEVFLOW_YES_SHIP` | false | Standing Ship gate pre-authorization (D-12, `28-CONTEXT.md`); overrides `devflow.toml`'s `yes_ship` key, which itself is OR-combined with `--yes-ship` at `devflow start` | | `DEVFLOW_ANTIGRAVITY_IDLE_TIMEOUT_SECS` | 120 | The per-agent idle timeout for the pipe-owning Antigravity monitor (phase 41, D-08): how long the monitor waits with NO output on the child's `stream-json` stream before recording an `idle_timeout` verdict and terminating it. Same 120s floor and clamp semantics as `DEVFLOW_CLAUDE_IDLE_TIMEOUT_SECS`. The default is a DECIDED starting point, not a silent inheritance of the Claude-measured floor — it is to be revisited after the first real Antigravity cadence measurement. Claude's variable is never read for Antigravity and vice versa | | `DEVFLOW_CLAUDE_IDLE_TIMEOUT_SECS` | 120 | How long the pipe-owning Claude monitor waits with NO output on the child's stream before recording an `idle_timeout` verdict and terminating it. Clamped to a 120s floor: a lower value is raised and the clamp is printed loudly at `devflow start`. The floor is 4x the CLI's measured `tool_progress` keepalive interval (a fixed 30.00s, measured 2026-08-03 over five workload-controlled trials on CLI 2.1.220), so it survives three consecutive missed keepalives; the earlier 30s floor sat exactly ON the keepalive interval and would have killed healthy stages running any tool call longer than ~30s, including `cargo test --workspace`. Every stream line resets the window, so a healthy long-running stage is never bounded by wall clock | -| `DEVFLOW_CLAUDE_LEGACY_LAUNCH` | false | D-11's escape hatch (31-04): force the pre-31 single-document Claude launch — prompt positionally in argv, `--output-format json`, the `sh` monitor — instead of the Phase 31 `stream-json` transport. OR-combined with `--legacy-claude-launch` at `devflow start` / `devflow resume` and persisted to `state-NN.json`, so a detached monitor honours it; a run authorized by this variable alone prints a notice naming it as the source. **Parsed as a bool: `=false` does NOT enable it, and an unparseable value warns and leaves the legacy path OFF.** Using it is announced on three channels (stdout, `phase-NN-monitor.log`, and a `claude_legacy_launch_forced` event in `events.jsonl`) because an escape hatch used routinely erodes what it protects. **What it gives up: the legacy path cannot deliver background-task notifications, so a multi-plan wave may orphan delegated work — that is 999.64, the defect Phase 31 exists to fix.** Once set for a run it is never cleared by `devflow resume`; edit `state-NN.json` to turn it off | +| `DEVFLOW_CLAUDE_LEGACY_LAUNCH` | false | D-11's escape hatch (31-04): force the pre-31 single-document Claude launch — prompt positionally in argv, `--output-format json`, the capture launch shape — instead of the Phase 31 `stream-json` transport. OR-combined with `--legacy-claude-launch` at `devflow start` / `devflow resume` and persisted to `state-NN.json`, so a detached monitor honours it; a run authorized by this variable alone prints a notice naming it as the source. **Parsed as a bool: `=false` does NOT enable it, and an unparseable value warns and leaves the legacy path OFF.** Using it is announced on three channels (stdout, `phase-NN-monitor.log`, and a `claude_legacy_launch_forced` event in `events.jsonl`) because an escape hatch used routinely erodes what it protects. **What it gives up: the legacy path cannot deliver background-task notifications, so a multi-plan wave may orphan delegated work — that is 999.64, the defect Phase 31 exists to fix.** Once set for a run it is never cleared by `devflow resume`; edit `state-NN.json` to turn it off | | `RUST_LOG` | `info` | Log verbosity (stderr) | | `DEVFLOW_LOG_FORMAT` | plain | `json` for machine-readable log lines | | `DEVFLOW_E2E_CHILD_TIMEOUT_SECS` | 90 | Test-only: bounds `gate_sweep_e2e.rs`'s patience with a spawned `devflow advance` child so CI cannot hang indefinitely; not read by any production code path | @@ -147,8 +237,7 @@ never receives the variable. | `lock-NN` | Per-phase advance lock (held across gate waits) | | `lock-project` | Coarse checkout lock (held seconds, around shared-git mutations) | | `phase-NN-stdout` / `phase-NN-stderr.log` | Agent capture files (what `logs` tails) | -| `phase-NN-exit` / `phase-NN-agent-pid` | Exit code + PID the monitor records | -| `phase-NN-monitor-stop` | Stop marker the `sh` monitor's TERM/INT trap writes before killing the agent, so an agent child that lost the TERM still stops before it starts; removed before each launch | +| `phase-NN-exit` / `phase-NN-agent-pid` | Exit code + PID the supervisor records | | `gates/NN-.json` (+ `.response.json`, `.ack.json`) | Gate request / answer / receipt | | `events.jsonl` | Append-only event log (schema v1, one JSON object per line, phase id on every line) — tail it from any tool | | `cron-instructions-NN.json` | Rate-limit resume record naming `devflow resume --phase N` for a paused run | diff --git a/crates/devflow-core/src/doc_check.rs b/crates/devflow-core/src/doc_check.rs index 786c66ac..138fbd36 100644 --- a/crates/devflow-core/src/doc_check.rs +++ b/crates/devflow-core/src/doc_check.rs @@ -671,3 +671,99 @@ fn no_monitor_pid_field_json_key_or_help_mention_remains() { findings.join("\n") ); } + +/// `text` with every `` span removed (an unterminated one runs to +/// the end of the text), so a fragment hidden in an HTML comment never counts +/// as documented. +fn strip_html_comments(text: &str) -> String { + let mut kept = String::with_capacity(text.len()); + let mut rest = text; + while let Some(open) = rest.find("") { + Some(close) => rest = &after_open[close + "-->".len()..], + None => return kept, + } + } + kept.push_str(rest); + kept +} + +/// The `fragments` that `text` does not contain, in the order given. +fn missing_fragments<'a>(text: &str, fragments: &[&'a str]) -> Vec<&'a str> { + fragments + .iter() + .copied() + .filter(|fragment| !text.contains(fragment)) + .collect() +} + +/// Fragments OPERATIONS.md must hold, compared as written (50-17). +const RUN_LIVENESS_FRAGMENTS: &[&str] = &[ + "GONE", + "STALE", + "ALIVE", + "devflow stop --phase N", + "devflow resume --phase N", + "ps -o pid,pgid,sid,lstart,args -g", + "DEVFLOW_CACHE_DIR", +]; + +/// Fragments OPERATIONS.md must hold, compared lowercased the way +/// `source_devflow_env_vars_and_subcommands_are_documented` compares (RV-06). +const INTERNAL_SUBCOMMAND_FRAGMENTS: &[&str] = &["devflow monitor", "devflow advance"]; + +/// 50-17 (R6-G4): OPERATIONS.md documents run liveness as the supervisor +/// socket answers it (GONE / STALE / ALIVE and the repair verbs), `devflow +/// stop`'s arms including the backstop's refusal and its inspection command, +/// the socket location's override, and both internal subcommands, outside +/// HTML comments. +#[test] +fn operations_md_documents_run_liveness_and_the_stop_arms() { + let every: String = RUN_LIVENESS_FRAGMENTS.join("\n"); + let all_but_stale = every.replace("STALE", ""); + assert_eq!( + missing_fragments(&all_but_stale, RUN_LIVENESS_FRAGMENTS), + vec!["STALE"], + "planted control: a text missing only STALE must report exactly STALE" + ); + let gone_only_in_a_comment = every.replace("GONE", ""); + assert!( + missing_fragments(&gone_only_in_a_comment, RUN_LIVENESS_FRAGMENTS).is_empty(), + "planted control: before stripping, the commented GONE still counts" + ); + assert_eq!( + missing_fragments( + &strip_html_comments(&gone_only_in_a_comment), + RUN_LIVENESS_FRAGMENTS + ), + vec!["GONE"], + "planted control: a GONE only inside an HTML comment must be missing after stripping" + ); + assert_eq!( + strip_html_comments("kept >Monitor: exit - Monitor->>Files: stdout, stderr, exit, PID state - Monitor->>CLI: advance --phase 16 - CLI->>Files: archive prior evidence + CLI->>Sup: re-exec as __monitor with phase identity + Sup->>Agent: launch stage prompt (own process group) + Sup-->>CLI: socket published, answers ping + CLI->>Files: record the socket in state-16.json + Agent-->>Sup: exit + Sup->>Files: stdout, stderr, exit, PID state + Sup->>Sup: advance in-process, archive prior evidence + Sup->>Next: launch the next stage + Next-->>Sup: socket published + Sup->>Files: record the next socket, then unlink its own ``` ## Completion Evidence From 5b6e21804afae6692c1e5f993818ecb941df3b30 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Sun, 18 Oct 2026 15:18:18 -0400 Subject: [PATCH 089/112] fix(50-09): point the backstop's group hint at pgrep -g, not ps -g On procps-ng, ps -g selects by session, so the leader-gone refusal's inspection command listed no member of a live process group whose id differs from its session id. The hint now runs ps -o pid,pgid,sid,lstart,args -p "$(pgrep -d, -g )". Verified on procps-ng 4.0.6 against a group with pgid != sid: the old command printed only a header, the new one listed the member. Test pins and the doc_check fragment were changed first and failed on the old text. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182ocSM2E4fb2zu52jxGLzi --- OPERATIONS.md | 2 +- crates/devflow-core/src/doc_check.rs | 2 +- crates/devflow-core/src/supervisor/backstop.rs | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index 51e7030a..3f27b4a3 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -40,7 +40,7 @@ Define → Plan → Code → Validate → Ship | `devflow parallel --phases 7,8 [--agents claude,codex] [--mode M] [--force]` | Run phases concurrently, each in its own worktree + monitor | | `devflow list` | Feature branches with divergence from develop | | `devflow reference [--branch B] [--refresh]` | Static snapshot worktree at `.worktrees/reference/` | -| `devflow stop --phase N [PROJECT] [--root PATH] [--launch ID]` | End a running phase cleanly (23c). In this order: **(1)** an open gate with a confirmed live waiter is answered with a rejection — the waiter unwinds through its own abort path, no signal sent. **(2)** Otherwise the run's supervisor socket is probed: the one `.devflow/state-NN.json` records, or, when that file is missing or does not load, the one live socket found through its sidecar for this project and phase. **ALIVE** → a `shutdown` request: the supervisor ends the agent's process group (exit code 143) and runs no advance. **STALE** → the guarded backstop: it signals (SIGTERM, then SIGKILL) only an agent group whose leader is still present with the recorded start time and still leads the group; a socket from a previous boot, an empty group or a recycled leader pid has its socket and sidecar removed with nothing signalled; a group whose leader is gone is refused and its members listed, with `ps -o pid,pgid,sid,lstart,args -g ` to inspect them (residual limit: nothing proves those members are the agent's, so `stop` never signals them — end them yourself). A probe that cannot answer is refused, nothing signalled. **(3)** Only when no live socket remains (**GONE**, or nothing recorded) is the process holding `.devflow/lock-{phase:02}` signalled, after confirming it is alive and its `/proc//cmdline` identifies it as belonging to DevFlow. No pid read from the state file is ever signalled. Idempotent; records `stopped`/`stop_reason` so `cleanup`'s fail-closed refusal recognizes the phase as no longer live — `stop` then `cleanup --force` compose in that order. When the state file does not load, `stop` still ends the run, says `stopped` was not recorded and leaves the file untouched; several live sockets with no usable state are refused, with one `devflow stop --phase N --launch ` line each. `--launch ` tears down only the socket with that launch id — the form `devflow doctor` prints for an orphan, usable from any directory — and answers no gate, signals no lock holder and writes no state | +| `devflow stop --phase N [PROJECT] [--root PATH] [--launch ID]` | End a running phase cleanly (23c). In this order: **(1)** an open gate with a confirmed live waiter is answered with a rejection — the waiter unwinds through its own abort path, no signal sent. **(2)** Otherwise the run's supervisor socket is probed: the one `.devflow/state-NN.json` records, or, when that file is missing or does not load, the one live socket found through its sidecar for this project and phase. **ALIVE** → a `shutdown` request: the supervisor ends the agent's process group (exit code 143) and runs no advance. **STALE** → the guarded backstop: it signals (SIGTERM, then SIGKILL) only an agent group whose leader is still present with the recorded start time and still leads the group; a socket from a previous boot, an empty group or a recycled leader pid has its socket and sidecar removed with nothing signalled; a group whose leader is gone is refused and its members listed, with `ps -o pid,pgid,sid,lstart,args -p "$(pgrep -d, -g )"` to inspect them (residual limit: nothing proves those members are the agent's, so `stop` never signals them — end them yourself). A probe that cannot answer is refused, nothing signalled. **(3)** Only when no live socket remains (**GONE**, or nothing recorded) is the process holding `.devflow/lock-{phase:02}` signalled, after confirming it is alive and its `/proc//cmdline` identifies it as belonging to DevFlow. No pid read from the state file is ever signalled. Idempotent; records `stopped`/`stop_reason` so `cleanup`'s fail-closed refusal recognizes the phase as no longer live — `stop` then `cleanup --force` compose in that order. When the state file does not load, `stop` still ends the run, says `stopped` was not recorded and leaves the file untouched; several live sockets with no usable state are refused, with one `devflow stop --phase N --launch ` line each. `--launch ` tears down only the socket with that launch id — the form `devflow doctor` prints for an orphan, usable from any directory — and answers no gate, signals no lock holder and writes no state | | `devflow cleanup [--force]` | Remove phase worktrees + their feature branches | | `devflow recover [--clean] [--phase N]` | Inspect state; `--clean` sweeps stale phases only; `--clean --phase N` clears one phase unconditionally | | `devflow test` | cargo test + clippy + fmt --check | diff --git a/crates/devflow-core/src/doc_check.rs b/crates/devflow-core/src/doc_check.rs index df5af8d0..9d0ca01a 100644 --- a/crates/devflow-core/src/doc_check.rs +++ b/crates/devflow-core/src/doc_check.rs @@ -706,7 +706,7 @@ const RUN_LIVENESS_FRAGMENTS: &[&str] = &[ "ALIVE", "devflow stop --phase N", "devflow resume --phase N", - "ps -o pid,pgid,sid,lstart,args -g", + "ps -o pid,pgid,sid,lstart,args -p \"$(pgrep -d, -g", "DEVFLOW_CACHE_DIR", ]; diff --git a/crates/devflow-core/src/supervisor/backstop.rs b/crates/devflow-core/src/supervisor/backstop.rs index ca5cfdf1..7c3a93f6 100644 --- a/crates/devflow-core/src/supervisor/backstop.rs +++ b/crates/devflow-core/src/supervisor/backstop.rs @@ -208,7 +208,7 @@ impl BackstopDecision { left the group. The leader is gone, so these may be the original agent's \ children or an unrelated group that reused the id; nothing was signalled, \ and {files} are kept. Inspect the group with: ps -o \ - pid,pgid,sid,lstart,args -g {pgid}", + pid,pgid,sid,lstart,args -p \"$(pgrep -d, -g {pgid})\"", members.len() ); for member in members { @@ -1161,7 +1161,10 @@ mod tests { let text = decision.describe(&socket, &sidecar); assert!( - text.contains(&format!("ps -o pid,pgid,sid,lstart,args -g {}", group.pgid)), + text.contains(&format!( + "ps -o pid,pgid,sid,lstart,args -p \"$(pgrep -d, -g {})\"", + group.pgid + )), "{text}" ); assert!( @@ -1362,7 +1365,7 @@ mod tests { let gone = BackstopDecision::RefuseLeaderGone(vec![member]).describe(&socket, &sidecar); assert!(!gone.contains("cannot prove process group"), "{gone}"); assert!( - gone.contains("ps -o pid,pgid,sid,lstart,args -g 4242"), + gone.contains(r#"ps -o pid,pgid,sid,lstart,args -p "$(pgrep -d, -g 4242)""#), "{gone}" ); } From 70014bfb1e73af5ca978ec02b72e01d9d51ae8f4 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Mon, 19 Oct 2026 07:33:48 -0400 Subject: [PATCH 090/112] test(50-02): wait out a run's unrecorded supervisors before the root drops - parallel_creates_two_worktrees_and_spawns_two_monitors left /tmp/.tmpXXXXXX/.devflow/.lock-07.coord (plus .gitignore) after every run: 5 of 5 alone, and once per full `cargo test --workspace` - Mechanism: the phase-7 reaper's shutdown lands while the Plan supervisor's advance is launching Code (safe point 3). Code is spawned but never recorded and waits on lock-07; Plan gives up after SUPERVISOR_STOP_WAIT (11 s) and answers `stopped`, the reaper returns, the TempDir is removed, then Code takes lock-07 to record its flagged stop and recreates /.devflow on the way - LaunchedRunReaper now also waits (bounded, 15 s) until no `devflow __monitor` naming its root and phase is alive, and ends any left after the wait (start-time checked, `spare` honoured) - Evidence: with the phase-7 reaper dropped first so every stop hits that window (about 13.4 s per run), 0 of 3 runs leaked; the same order with the old reaper leaked 2 of 3. Unmodified test: 5 of 5 before, 0 of 5 after Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HRWwk1oVv9oLFxjb9cSiCm --- crates/devflow-cli/tests/common/mod.rs | 86 +++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/crates/devflow-cli/tests/common/mod.rs b/crates/devflow-cli/tests/common/mod.rs index 6727f967..feec5f39 100644 --- a/crates/devflow-cli/tests/common/mod.rs +++ b/crates/devflow-cli/tests/common/mod.rs @@ -147,6 +147,10 @@ pub fn kill_and_reap(child: &mut Child) { /// How long [`LaunchedRunReaper`] waits for a supervisor's final shutdown line. const REAP_SHUTDOWN_WAIT: Duration = Duration::from_secs(15); +/// How long [`LaunchedRunReaper`] waits for the phase's supervisors to exit on +/// their own before it ends the ones still alive. +const REAP_EXIT_WAIT: Duration = Duration::from_secs(15); + /// Ends, on drop, the run recorded in the phase state, through its /// supervisor's socket (D-07): a `shutdown` request first; then, if the /// recorded agent group's leader is still the process the record names @@ -155,6 +159,15 @@ const REAP_SHUTDOWN_WAIT: Duration = Duration::from_secs(15); /// to the agent named in the agent-pid file. A pid in `spare` is never /// signalled. Bind it before (or right after) the spawn that launches a run, /// and keep the run's [`HermeticHome`] alive until this has dropped. +/// +/// It then waits, bounded by [`REAP_EXIT_WAIT`], until no `devflow +/// __monitor` whose argv names this root and phase is alive, and ends any +/// still alive after that wait. The state cannot name every one: a stop that +/// lands while an advance is launching the next stage leaves that next +/// supervisor spawned but never recorded, with the stop flagged. It records +/// its own stop under the per-phase lock a moment later, and taking that lock +/// recreates `/.devflow/` when the root's `TempDir` has already been +/// removed, leaving `/tmp/.tmpXXXXXX/.devflow/.lock-NN.coord` behind. pub struct LaunchedRunReaper { root: PathBuf, phase: PhaseId, @@ -169,10 +182,70 @@ impl LaunchedRunReaper { spare, } } -} -impl Drop for LaunchedRunReaper { - fn drop(&mut self) { + /// Every live `devflow __monitor` whose argv names this root (as given or + /// canonicalized) after `--project` and this phase after `--phase`, with + /// its start time: each supervisor this run started, recorded or not. + fn live_supervisors(&self) -> Vec<(u32, u64)> { + use std::os::unix::ffi::OsStrExt as _; + let canonical = fs::canonicalize(&self.root).unwrap_or_else(|_| self.root.clone()); + let roots = [ + self.root.as_os_str().as_bytes(), + canonical.as_os_str().as_bytes(), + ]; + let Ok(entries) = fs::read_dir("/proc") else { + return Vec::new(); + }; + entries + .flatten() + .filter_map(|entry| { + let pid = entry.file_name().to_str()?.parse::().ok()?; + let cmdline = fs::read(entry.path().join("cmdline")).ok()?; + let args: Vec<&[u8]> = cmdline.split(|byte| *byte == 0).collect(); + let value_after = |flag: &str| { + args.windows(2) + .find(|pair| pair[0] == flag.as_bytes()) + .map(|pair| pair[1]) + }; + let phase = value_after("--phase") + .and_then(|value| std::str::from_utf8(value).ok()) + .and_then(|value| value.parse::().ok()); + let ours = args.contains(&b"__monitor".as_slice()) + && value_after("--project").is_some_and(|value| roots.contains(&value)) + && phase == Some(self.phase); + if !ours { + return None; + } + let start = devflow_core::agent::process_start_time(pid)?; + devflow_core::agent::agent_running(pid).then_some((pid, start)) + }) + .collect() + } + + /// Wait for this phase's supervisors to exit, then end any left, so none + /// of them writes under the root once this returns. + fn wait_for_supervisors_to_exit(&self) { + let deadline = Instant::now() + REAP_EXIT_WAIT; + while Instant::now() < deadline { + if self.live_supervisors().is_empty() { + return; + } + std::thread::sleep(Duration::from_millis(50)); + } + for (pid, start) in self.live_supervisors() { + if !self.spare.contains(&pid) && devflow_core::agent::is_same_process(pid, start) { + let _ = devflow_core::agent::terminate_and_verify( + pid, + devflow_core::agent::TERMINATE_VERIFY_WAIT, + devflow_core::agent::TERMINATE_VERIFY_POLL, + ); + } + } + } + + /// The shutdown described on [`LaunchedRunReaper`]: the recorded + /// supervisor, else the agent in the agent-pid file. + fn end_recorded_run(&self) { let record = devflow_core::workflow::load_state(&self.root, self.phase) .ok() .and_then(|state| state.supervisor); @@ -206,6 +279,13 @@ impl Drop for LaunchedRunReaper { } } +impl Drop for LaunchedRunReaper { + fn drop(&mut self) { + self.end_recorded_run(); + self.wait_for_supervisors_to_exit(); + } +} + /// Send `signal` to process group `pgid`. Refuses pgid 0 or 1, a value that /// does not fit `pid_t`, and this process's own group. pub fn signal_group(pgid: u32, signal: libc::c_int) { From e3493ea8644ddeeb0d6de856c025167985541106 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Mon, 19 Oct 2026 07:37:35 -0400 Subject: [PATCH 091/112] test(50-16): run the unlistable socket-dir tests as root too - doctor_reports_an_unlistable_socket_dir_instead_of_nothing (50-12) and the_launch_guard_refuses_an_unlistable_socket_dir_over_an_ unloadable_state (50-15) skipped under uid 0 since cdb6efc: root's CAP_DAC_OVERRIDE lists a 0o000 directory, so the pinned container never ran their assertions (WINDOWS.md #19) - The reads now run on a scoped thread that clears CAP_DAC_OVERRIDE and CAP_DAC_READ_SEARCH from its own effective set (capset(2) is per-thread), so root is refused by the same 0o000 directory with EACCES and reaches the same arm; the uid-0 skips are removed - Both tests now also pin the error: permission denied - Pinned image as uid 0: both pass; with socket_dir_unlistable forced to None both fail at their assertions; with the capability drop disabled both fail their 0o000 precondition. Host (uid 1000): pass, and fail with either fail-closed check removed Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HRWwk1oVv9oLFxjb9cSiCm --- crates/devflow-cli/src/commands.rs | 118 ++++++++++++++++++++++------- 1 file changed, 91 insertions(+), 27 deletions(-) diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index 23e2b2a5..b03a21a9 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -8358,6 +8358,72 @@ mod tests { ); } + /// Run `body` on a new thread without `CAP_DAC_OVERRIDE` and + /// `CAP_DAC_READ_SEARCH` in its effective set (WINDOWS.md #19). Either + /// lets uid 0 list a mode-0o000 directory, so the pinned container (root) + /// could not produce an unlistable socket directory; without them root is + /// refused like any other owner. Capabilities are per-thread and + /// `capset(2)` changes only the calling thread's, so the harness keeps + /// its own, and they end with the thread. A non-root thread holds neither + /// capability, so nothing changes there. + fn without_dac_override(body: impl FnOnce() -> T + Send) -> T { + std::thread::scope(|scope| { + scope + .spawn(|| { + drop_dac_override_on_this_thread(); + body() + }) + .join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) + }) + } + + /// Clear `CAP_DAC_OVERRIDE` and `CAP_DAC_READ_SEARCH` from the calling + /// thread's effective capability set; the permitted set is kept. + #[cfg(target_os = "linux")] + fn drop_dac_override_on_this_thread() { + // ``, version 3: one header, two data structs. + #[repr(C)] + struct CapHeader { + version: u32, + pid: libc::c_int, + } + #[repr(C)] + #[derive(Clone, Copy)] + struct CapData { + effective: u32, + permitted: u32, + inheritable: u32, + } + const LINUX_CAPABILITY_VERSION_3: u32 = 0x2008_0522; + const CAP_DAC_OVERRIDE: u32 = 1; + const CAP_DAC_READ_SEARCH: u32 = 2; + + let mut header = CapHeader { + version: LINUX_CAPABILITY_VERSION_3, + pid: 0, + }; + let mut data = [CapData { + effective: 0, + permitted: 0, + inheritable: 0, + }; 2]; + // SAFETY: `header` and `data` have capget(2)'s version-3 layout (one + // header, two data structs); pid 0 is the calling thread. + let got = unsafe { libc::syscall(libc::SYS_capget, &raw mut header, data.as_mut_ptr()) }; + assert_eq!(got, 0, "capget: {}", std::io::Error::last_os_error()); + data[0].effective &= !((1 << CAP_DAC_OVERRIDE) | (1 << CAP_DAC_READ_SEARCH)); + // SAFETY: as above; capset(2) with pid 0 changes only this thread. + let set = unsafe { libc::syscall(libc::SYS_capset, &raw const header, data.as_ptr()) }; + assert_eq!(set, 0, "capset: {}", std::io::Error::last_os_error()); + } + + /// No per-thread capabilities to drop off Linux: there a root run still + /// lists a 0o000 directory, and the tests that use this fail their own + /// precondition rather than pass without asserting. + #[cfg(not(target_os = "linux"))] + fn drop_dac_override_on_this_thread() {} + /// Unit tests for the pure `doctor` reconciliation core (18a). Each test /// builds a `PhaseFacts` directly — no repository, no I/O — proving /// `reconcile_phase` is a predicate over facts alone. @@ -8815,15 +8881,6 @@ mod tests { fn doctor_reports_an_unlistable_socket_dir_instead_of_nothing() { use std::os::unix::fs::PermissionsExt as _; let cache = tempfile::tempdir().unwrap(); - // Root lists a 0o000 directory (the pinned container runs as - // uid 0), so the fixture cannot produce its precondition there; - // skip rather than assert it (the 999.113 pattern, 50-16). - if std::os::unix::fs::MetadataExt::uid(&std::fs::metadata(cache.path()).unwrap()) == 0 { - eprintln!( - "skipping doctor_reports_an_unlistable_socket_dir_instead_of_nothing: running as root, which bypasses the 0o000 fixture" - ); - return; - } assert_eq!(socket_dir_unlistable(cache.path()), None, "absent"); assert_eq!(render_orphan_supervisors_text(&[], None), ""); let sock = registry::socket_dir_in(cache.path()); @@ -8831,9 +8888,16 @@ mod tests { assert_eq!(socket_dir_unlistable(cache.path()), None, "listable"); std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o000)).unwrap(); - let unlistable = socket_dir_unlistable(cache.path()); + // Without DAC override, so uid 0 (the pinned container) is + // refused too (WINDOWS.md #19). + let unlistable = without_dac_override(|| socket_dir_unlistable(cache.path())); std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o700)).unwrap(); let why = unlistable.expect("a directory without read permission is unlistable"); + assert_eq!( + why, + std::io::ErrorKind::PermissionDenied.to_string(), + "the 0o000 directory itself refuses the listing" + ); assert!(!why.contains('/'), "path-free: {why}"); let text = render_orphan_supervisors_text(&[], Some(&why)); assert!( @@ -10958,25 +11022,22 @@ mod tests { let phase = PhaseId::new(1501); let root = root_with_an_unloadable_state(phase); let cache = tempfile::tempdir().unwrap(); - // Root lists a 0o000 directory (the pinned container runs as - // uid 0), so the fixture cannot produce its precondition there; - // skip rather than assert it (the 999.113 pattern, 50-16). - if std::os::unix::fs::MetadataExt::uid(&std::fs::metadata(cache.path()).unwrap()) == 0 { - eprintln!( - "skipping the_launch_guard_refuses_an_unlistable_socket_dir_over_an_unloadable_state: running as root, which bypasses the 0o000 fixture" - ); - return; - } let sock = registry::socket_dir_in(cache.path()); std::fs::create_dir_all(&sock).unwrap(); std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o000)).unwrap(); - let listable = std::fs::read_dir(&sock).is_ok(); - let result = refuse_launch_over_a_live_run( - root.path(), - phase, - LaunchVerb::Start, - Some(cache.path()), - ); + // Both reads run without DAC override, so uid 0 (the pinned + // container) is refused too (WINDOWS.md #19). + let (listable, result) = without_dac_override(|| { + ( + std::fs::read_dir(&sock).is_ok(), + refuse_launch_over_a_live_run( + root.path(), + phase, + LaunchVerb::Start, + Some(cache.path()), + ), + ) + }); std::fs::set_permissions(&sock, std::fs::Permissions::from_mode(0o700)).unwrap(); assert!( !listable, @@ -10987,7 +11048,10 @@ mod tests { }; for fragment in [ sock.display().to_string(), - "cannot be listed".to_string(), + format!( + "cannot be listed ({})", + std::io::ErrorKind::PermissionDenied + ), "refusing to start; nothing was written".to_string(), ] { assert!(text.contains(&fragment), "missing {fragment:?}: {text}"); From c1ee968d4cd84395ab3e3e97b42ff7a904d340f3 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Mon, 19 Oct 2026 07:38:09 -0400 Subject: [PATCH 092/112] docs(50-13): describe the supervisor's cancellable gate poll - ARCHITECTURE.md's gate section described only Gates::poll_response's 1s -> 60s backoff. Inside a supervisor's advance (stop token engaged) wait_for_gate_response uses poll_response_cancellable: owner gate_wait, every sleep capped at 1s, the stop token checked before each sleep, so a socket shutdown or SIGTERM/SIGINT ends the wait in about a second, clears the gate, records `stopped` and keeps the state - The uncancellable backoff poll is kept for a run with no stop token (the hidden `devflow advance` from a shell, a foreground gate) - run_gate() lives in pipeline_gate.rs, not main.rs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HRWwk1oVv9oLFxjb9cSiCm --- ARCHITECTURE.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3ae58f32..25a7cdb4 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -220,12 +220,27 @@ files per gated stage, all written atomically (temp file + rename): (`Gates::ack()`) once the response is consumed, so a poller knows to clean up. -`run_gate()` (`crates/devflow-cli/src/main.rs`) writes the gate, fires the -notify hook (`fire_gate_notify()` — runs `$DEVFLOW_GATE_NOTIFY_CMD` via -`sh -c` with gate metadata as environment variables, never interpolated +`run_gate()` (`crates/devflow-cli/src/pipeline_gate.rs`) writes the gate, +fires the notify hook (`fire_gate_notify()` — runs `$DEVFLOW_GATE_NOTIFY_CMD` +via `sh -c` with gate metadata as environment variables, never interpolated into the command string; a silent no-op when the variable is unset), then -blocks in `Gates::poll_response()` with exponential backoff (1s → 2s → … -capped at 60s) until a response appears or `DEVFLOW_GATE_TIMEOUT_SECS` +waits in `wait_for_gate_response()` (`crates/devflow-cli/src/pipeline_launch.rs`), +which polls the response file one of two ways: + +- **Inside a supervisor's advance** (its stop token is engaged, 50-13): the + run's owner reads `gate_wait` in a `ping` reply for the length of the wait, + and `Gates::poll_response_cancellable()` reads the response file about once + a second (every sleep is capped at 1s) and checks the stop token before each + sleep. A stop — a `shutdown` request on the supervisor's socket, or + SIGTERM/SIGINT to the supervisor — ends the wait within about a second: the + gate's files are removed, `stopped` is recorded in the state, which is kept + for `devflow resume` (not an abort), and the advance unwinds without acting + on the gate. A response already on disk wins over a stop. +- **Anywhere else** (no stop token: the hidden `devflow advance` run from a + shell, or a gate a foreground command fires): `Gates::poll_response()` with + exponential backoff (1s → 2s → … capped at 60s), which nothing cancels. + +Either wait ends when a response appears or `DEVFLOW_GATE_TIMEOUT_SECS` (default 259200 = 3 days, held equal to `DEVFLOW_GATE_MAX_UNATTENDED_AGE_SECS` so `devflow gate sweep` cannot reap a gate this poll is still watching) elapses. `GateAction::from_response()` turns an From 28b57c5b445af3dbffa88b68932c8fd4f73517b8 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Mon, 19 Oct 2026 07:39:29 -0400 Subject: [PATCH 093/112] fix(50-14): name `devflow stop` in cleanup's live-run refusal - cleanup's refusal for a live agent or an ALIVE supervisor told the operator to run `devflow resume --phase N`, which refuses while a run is live; since 50-14 `devflow stop --phase N` is the verb that ends it - cleanup_force_refuses_on_live_agent_unknown_monitor and cleanup_force_refuses_on_dead_monitor_live_agent now assert the refusal names `devflow stop --phase N` and no `devflow resume`; both failed on the old text first - OPERATIONS.md (50-17) no longer says the CLI text is out of step Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HRWwk1oVv9oLFxjb9cSiCm --- OPERATIONS.md | 6 +++--- crates/devflow-cli/src/commands.rs | 5 +++-- crates/devflow-cli/tests/phase7_cli.rs | 12 ++++++++---- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index 3f27b4a3..fca9fd92 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -104,9 +104,9 @@ How each command shows them: `doctor`'s repair is `devflow status`, `stop` refuses and signals nothing, and `cleanup` keeps the worktree unless `--force`. - **`devflow cleanup`** refuses an ALIVE run even with `--force` — end it - first with `devflow stop --phase N`, or wait for it (the refusal's own text - still says `devflow resume`, which does not end a live run). It keeps a - STALE run's worktree, or one whose probe failed, unless `--force`. + first with `devflow stop --phase N`, the line its refusal names, or wait + for it. It keeps a STALE run's worktree, or one whose probe failed, unless + `--force`. - **`devflow recover --clean --phase N`** refuses while the phase's supervisor is ALIVE and deletes nothing. - **`devflow start` / `devflow resume`** refuse while a run is live: an ALIVE diff --git a/crates/devflow-cli/src/commands.rs b/crates/devflow-cli/src/commands.rs index b03a21a9..65d0d6d4 100644 --- a/crates/devflow-cli/src/commands.rs +++ b/crates/devflow-cli/src/commands.rs @@ -932,13 +932,14 @@ pub(crate) fn cleanup(project_root: &Path, force: bool) -> Result<(), CliError> // Fail-closed on a live run: refuse whenever the agent is alive // (whatever the supervisor's liveness) OR the supervisor answers - // ALIVE, `--force` or not. + // ALIVE, `--force` or not. The verb that ends a live run is `stop` + // (50-14); `resume` refuses while one is live. if agent_alive || matches!(phase_liveness, PhaseLiveness::Alive { .. }) { let phase_label = phase .map(|p| p.to_string()) .unwrap_or_else(|| "?".to_string()); return Err(CliError::Message(format!( - "refusing to remove worktree {} for phase {phase_label} ({}) — run `devflow resume --phase {phase_label}` or wait for it to finish", + "refusing to remove worktree {} for phase {phase_label} ({}) — run `devflow stop --phase {phase_label}` or wait for it to finish", wt.path.display(), phase_liveness.describe(), ))); diff --git a/crates/devflow-cli/tests/phase7_cli.rs b/crates/devflow-cli/tests/phase7_cli.rs index 547a6ff1..d31eea4b 100644 --- a/crates/devflow-cli/tests/phase7_cli.rs +++ b/crates/devflow-cli/tests/phase7_cli.rs @@ -1063,9 +1063,11 @@ fn cleanup_force_refuses_on_live_agent_unknown_monitor() { String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); + // 50-14: `stop` ends a live run; `resume` refuses while one is live. assert!( - combined.contains("devflow resume"), - "refusal must name `devflow resume` as the unblocking action, got:\n{combined}" + combined.contains("run `devflow stop --phase 8`") && !combined.contains("devflow resume"), + "refusal must name `devflow stop --phase 8` (not `devflow resume`) as the \ + unblocking action, got:\n{combined}" ); assert!( wt_path.is_dir(), @@ -1135,9 +1137,11 @@ fn cleanup_force_refuses_on_dead_monitor_live_agent() { String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); + // 50-14: `stop` ends a live run; `resume` refuses while one is live. assert!( - combined.contains("devflow resume"), - "refusal must name `devflow resume` as the unblocking action, got:\n{combined}" + combined.contains("run `devflow stop --phase 9`") && !combined.contains("devflow resume"), + "refusal must name `devflow stop --phase 9` (not `devflow resume`) as the \ + unblocking action, got:\n{combined}" ); assert!( wt_path.is_dir(), From 0c66b16e6b3332f7fcb062b033bc1036db61ac3d Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Mon, 19 Oct 2026 07:40:41 -0400 Subject: [PATCH 094/112] fix(50-11): name the capture supervisor in the auto-mode refusal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The preflight's launch-shape refusal said the chain-flag guard binds only inside the pipe-owning monitor, "which this launch shape never starts". Since 50-11 a capture launch runs under the same Rust supervisor; run_monitor just never engages the guard for it - The reason now ends "— so Code would launch on the capture shape, whose supervisor never engages the chain-flag guard (only a stream-shape supervisor does)", for every cause; the doc comment no longer calls the legacy arm a detached shell script - unattended_launch_shape_refusal_names_the_capture_supervisor pins the sentence for the legacy opt-out and a codex agent; it failed on the old text first Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HRWwk1oVv9oLFxjb9cSiCm --- crates/devflow-cli/src/preflight.rs | 52 +++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/crates/devflow-cli/src/preflight.rs b/crates/devflow-cli/src/preflight.rs index fab5dd14..df734a4c 100644 --- a/crates/devflow-cli/src/preflight.rs +++ b/crates/devflow-cli/src/preflight.rs @@ -1008,11 +1008,13 @@ fn unattended_config_condition(launch_root: &Path) -> ConditionState { /// C2: the Code stage would launch on the pipe-owning arm AND the driver has /// a real dogfooded run (round-3 F5/D-04). /// -/// The chain-flag guard lives in a Rust process's stack frame; the legacy arm -/// is a detached shell script with no frame to hang a `Drop` on -/// (`35.1-RESEARCH.md` Pitfall 4). This turns that accepted gap from a silent -/// stall hours into an unattended run into a refusal before the first agent is -/// spawned. +/// The chain-flag guard is engaged by the stage supervisor (`run_monitor`) +/// for a stream launch only, the one shape that bounds the flag's lifetime to +/// the agent's. Since 50-11 a capture launch runs under the same Rust +/// supervisor but never engages it (before that the legacy arm was a detached +/// shell script with no frame to hang a `Drop` on, `35.1-RESEARCH.md` +/// Pitfall 4). This turns that accepted gap from a silent stall hours into an +/// unattended run into a refusal before the first agent is spawned. /// /// Asks [`stream_launch_enabled`] — the launch path's OWN predicate — /// rather than re-deriving the answer, so the preflight and the launch cannot @@ -1048,8 +1050,8 @@ fn unattended_launch_shape_condition(state: &State) -> ConditionState { // The reason names the CAUSE that applied, and then the consequence in // terms true of every cause. ConditionState::DoesNotHold(format!( - "{} — the chain-flag guard binds only inside the pipe-owning monitor, which this \ - launch shape never starts", + "{} — so Code would launch on the capture shape, whose supervisor never engages the \ + chain-flag guard (only a stream-shape supervisor does)", causes.join(" and ") )) } @@ -4657,4 +4659,40 @@ mod tests { "the driver still resolves the agy program (unchanged path)" ); } + + /// Since 50-11 both launch shapes run under the same Rust supervisor, + /// and only a stream launch engages the chain-flag guard. The refusal + /// says so, for every cause, and no longer says the capture shape + /// "never starts" a pipe-owning monitor. + #[test] + fn unattended_launch_shape_refusal_names_the_capture_supervisor() { + let dir = tempfile::tempdir().unwrap(); + let mut legacy = State::new( + PhaseId::new(43), + AgentKind::Claude, + Mode::Auto, + dir.path().to_path_buf(), + ); + legacy.legacy_claude_launch = true; + let codex = State::new( + PhaseId::new(44), + AgentKind::Codex, + Mode::Auto, + dir.path().to_path_buf(), + ); + for state in [legacy, codex] { + let ConditionState::DoesNotHold(reason) = unattended_launch_shape_condition(&state) + else { + panic!("{} --mode auto must be refused", state.agent); + }; + assert!( + reason.ends_with( + " — so Code would launch on the capture shape, whose supervisor never \ + engages the chain-flag guard (only a stream-shape supervisor does)" + ), + "{reason}" + ); + assert!(!reason.contains("never starts"), "{reason}"); + } + } } From f090a5e84a54eefd43d9bf4e855fde3f987b557b Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Wed, 21 Oct 2026 07:47:24 -0400 Subject: [PATCH 095/112] test(50): reproduce a stop lost to the --yes-ship merge (CR-01) Two held-advance pairs on a --yes-ship Ship stage: a socket stop taken after safe point 1 (test hold after_safe_point_1) and one taken just before finalization (test hold before_finalization) must ship nothing, record the stop at safe point 5 and 6 respectively, and answer the caller stopped. Each has a no-stop control that ships. Both stop tests fail on the current code: the run emits gate_resolved (--yes-ship), runs every after-ship hook and emits workflow_shipped. The hold points are cfg(test) only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGzZZbfn3r9638yptqPL47 --- crates/devflow-cli/src/pipeline_gate.rs | 2 + crates/devflow-cli/src/pipeline_launch.rs | 149 +++++++++++++++++++++- 2 files changed, 150 insertions(+), 1 deletion(-) diff --git a/crates/devflow-cli/src/pipeline_gate.rs b/crates/devflow-cli/src/pipeline_gate.rs index f6129ea7..a05a0054 100644 --- a/crates/devflow-cli/src/pipeline_gate.rs +++ b/crates/devflow-cli/src/pipeline_gate.rs @@ -233,6 +233,8 @@ pub(crate) fn finish_workflow_with_gate_timeout( state: &mut State, gate_timeout_secs: u64, ) -> Result<(), CliError> { + #[cfg(test)] + crate::pipeline_launch::test_hold("before_finalization"); loop { if run_checkout_hooks(project_root, state, &hooks::hooks_after_ship(), Stage::Ship) { break; diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index d9a3a63e..e5dc7aa8 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -1142,7 +1142,7 @@ const TEST_HOLD_WAIT: Duration = Duration::from_secs(30); /// window. A launch made outside an advance (the test's own first launch) /// never holds. Compiled out of non-test builds, together with every call. #[cfg(test)] -fn test_hold(point: &str) { +pub(crate) fn test_hold(point: &str) { if with_stop_token(|_| ()).is_none() { return; } @@ -2471,6 +2471,8 @@ pub(crate) fn advance_with( if stopped_at_safe_point(&mut state, 1)? { return Ok(()); } + #[cfg(test)] + test_hold("after_safe_point_1"); // 999.138 residual: a stopped run never advances. `resume` clears // `stopped` before it launches, so no caller advances a stopped state // legitimately. @@ -7790,6 +7792,12 @@ mod tests { /// Launch stage Define for `phase` (capture shape, Supervise mode, /// `--until plan`) and return once N is recorded. fn launch(phase: PhaseId) -> Self { + Self::launch_with(phase, |_, _| {}) + } + + /// [`launch`](Self::launch), with `configure` applied to the repo + /// and the state before it is saved and stage N launched. + fn launch_with(phase: PhaseId, configure: impl FnOnce(&Path, &mut State)) -> Self { let cache = PathBuf::from( std::env::var_os("DEVFLOW_CACHE_DIR").expect("the parent passes DEVFLOW_CACHE_DIR"), ); @@ -7805,6 +7813,7 @@ mod tests { let mut state = State::new(phase, AgentKind::Claude, Mode::Supervise, root.clone()); state.legacy_claude_launch = true; state.stop_until = Some(Stage::Plan); + configure(&root, &mut state); workflow::save_state(&state).unwrap(); let launched = launch_stage(&mut state, None, None); let n = state.supervisor.clone(); @@ -8779,6 +8788,144 @@ mod tests { assert!(events_named(&run.root, "advance_stopped").is_empty()); } + // ---- CR-01 (50-REVIEW.md): a stop before a `--yes-ship` merge ---- + + /// Whether `branch` exists in the repo at `root`. + fn branch_exists(root: &Path, branch: &str) -> bool { + devflow_core::test_support::git_command(root) + .args(["rev-parse", "--verify", "--quiet", branch]) + .status() + .unwrap() + .success() + } + + /// The body of the `--yes-ship` pair: stage N is Ship with `--yes-ship` + /// and a feature branch finalization can merge, and its advance is held + /// at the configured point. With `stop`, a socket stop is taken there + /// (owner `advancing`, so it is only flagged) before the release: the + /// advance must ship nothing, record the stop at `safe_point` and answer + /// the caller `stopped`. Without it (the control) the run ships. + fn held_yes_ship_advance(phase: PhaseId, stop: bool, safe_point: u8) { + let _guard = env_lock(); + let branch = format!("feature/phase-{}", phase.padded()); + let run = HandoffRun::launch_with(phase, |root, state| { + assert!( + devflow_core::test_support::git_command(root) + .args(["branch", &branch, "develop"]) + .status() + .unwrap() + .success() + ); + state.stage = Stage::Ship; + state.yes_ship = true; + state.stop_until = None; + }); + run.go(); + run.wait_held(); + if stop { + let (accepted, last) = shutdown_accepted_then( + &run.n.socket_path, + || run.release(), + Duration::from_secs(30), + ); + assert_eq!(accepted["owner"], "advancing", "{accepted}"); + assert!( + events_named(&run.root, "workflow_shipped").is_empty(), + "a stop accepted before the --yes-ship merge must ship nothing: {}", + run.events() + ); + assert!( + branch_exists(&run.root, &branch), + "no after-ship hook may run: the feature branch is kept" + ); + let state = run.state(); + assert!( + state.stopped, + "state.stopped after a stop at safe point {safe_point}" + ); + assert_eq!(state.stop_reason.as_deref(), Some(ADVANCE_STOP_REASON)); + assert!(!state.gate_pending, "no gate is left pending"); + assert!(!Gates::response_path(&run.root, run.phase, Stage::Ship).exists()); + let stopped = events_named(&run.root, "advance_stopped"); + assert_eq!(stopped.len(), 1, "{}", run.events()); + assert_eq!(stopped[0]["safe_point"], safe_point, "{}", stopped[0]); + assert_eq!( + last["stopped"], true, + "the stop's caller reads stopped: {last}" + ); + } else { + run.release(); + assert!( + poll_until(HANDOFF_STEP_WAIT, || events_named( + &run.root, + "workflow_shipped" + ) + .len() + == 1), + "control: without a stop the --yes-ship run ships: {}", + run.events() + ); + assert!( + !branch_exists(&run.root, &branch), + "control: the after-ship hooks ran (the feature branch is cleaned up)" + ); + assert!(events_named(&run.root, "advance_stopped").is_empty()); + } + } + + /// CR-01, safe point 5: a stop accepted after safe point 1 and before + /// `--yes-ship` writes its approval stops the run there. The pre-written + /// approval would otherwise beat the stop in the gate wait and let + /// finalization merge, bump, tag and push. + #[test] + fn a_stop_after_safe_point_1_is_honoured_before_the_yes_ship_approval() { + const NAME: &str = "pipeline_launch::tests::a_stop_after_safe_point_1_is_honoured_before_the_yes_ship_approval"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, Some("after_safe_point_1"), SUCCESS_MARKER); + return; + } + held_yes_ship_advance(PhaseId::new(1314), true, 5); + } + + /// The control for safe point 5: the same hold and no stop, and the + /// `--yes-ship` run ships. + #[test] + fn without_a_stop_after_safe_point_1_the_yes_ship_run_ships() { + const NAME: &str = + "pipeline_launch::tests::without_a_stop_after_safe_point_1_the_yes_ship_run_ships"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, Some("after_safe_point_1"), SUCCESS_MARKER); + return; + } + held_yes_ship_advance(PhaseId::new(1315), false, 5); + } + + /// CR-01, safe point 6: a stop accepted after the Ship gate was answered + /// and before finalization stops the run before the merge. + #[test] + fn a_stop_before_finalization_is_honoured_before_the_merge() { + const NAME: &str = + "pipeline_launch::tests::a_stop_before_finalization_is_honoured_before_the_merge"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, Some("before_finalization"), SUCCESS_MARKER); + return; + } + held_yes_ship_advance(PhaseId::new(1316), true, 6); + } + + /// The control for safe point 6: the same hold and no stop, and + /// finalization ships. + #[test] + fn without_a_stop_before_finalization_the_run_ships() { + const NAME: &str = + "pipeline_launch::tests::without_a_stop_before_finalization_the_run_ships"; + if !devflow_core::test_support::in_child_test(NAME) { + run_handoff_child(NAME, Some("before_finalization"), SUCCESS_MARKER); + return; + } + held_yes_ship_advance(PhaseId::new(1317), false, 6); + } + /// R-10: a SIGKILL of N between N+1's publish and N's unlink (N parked /// at `after_publish`) leaves the state naming the live N+1, and N's /// leftover socket probes Stale with a sidecar whose launch id differs From 8d16e3f3f9372e393b117de7aeda4325ea3b6ae6 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Wed, 21 Oct 2026 07:50:25 -0400 Subject: [PATCH 096/112] fix(50): honour a stop before yes-ship approval and finalization (CR-01) A stop accepted while a supervisor's advance runs (owner advancing) was read only at safe points 1-4. With --yes-ship, run_gate_with_timeout wrote the approval before the gate wait, and a response on disk wins over a cancel, so a stop landing after safe point 1 let finalization merge, bump, tag and push. Safe point 5 sits right before a pre-authorized response is written: the gate is cleared, the stop recorded and an error unwinds the advance, as at safe point 4. Safe point 6 opens finish_workflow_with_gate_timeout and clears the consumed Ship approval with the stop. Both record advance_stopped with their number, so a parked stop caller is answered stopped. Outside a supervisor's advance (ship_override) no token is engaged and both proceed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGzZZbfn3r9638yptqPL47 --- crates/devflow-cli/src/pipeline_gate.rs | 26 +++++++++++++++++++++++ crates/devflow-cli/src/pipeline_launch.rs | 8 +++++++ 2 files changed, 34 insertions(+) diff --git a/crates/devflow-cli/src/pipeline_gate.rs b/crates/devflow-cli/src/pipeline_gate.rs index a05a0054..e99cdb69 100644 --- a/crates/devflow-cli/src/pipeline_gate.rs +++ b/crates/devflow-cli/src/pipeline_gate.rs @@ -235,6 +235,19 @@ pub(crate) fn finish_workflow_with_gate_timeout( ) -> Result<(), CliError> { #[cfg(test)] crate::pipeline_launch::test_hold("before_finalization"); + // Safe point 6 (CR-01, 50-REVIEW.md): a stop taken during the advance + // after the Ship gate was answered is honoured before the merge, version + // bump, tag and push. The consumed approval is cleared with it, so a + // resumed run asks again. Outside a supervisor's advance (the foreground + // `ship_override`) no stop token is engaged and this proceeds. + if crate::pipeline_launch::stop_requested_at_safe_point() { + let _ = Gates::cleanup(project_root, state.phase, Stage::Ship); + state.gate_pending = false; + crate::pipeline_launch::record_advance_stop(state, 6)?; + return Err(CliError::Message(format!( + "finalization did not run: {ADVANCE_STOP_REASON}" + ))); + } loop { if run_checkout_hooks(project_root, state, &hooks::hooks_after_ship(), Stage::Ship) { break; @@ -423,6 +436,19 @@ pub(crate) fn run_gate_with_timeout( // `Gates::write_gate` above, and `poll_response` would never observe a // response written after it returns). if let Some(response) = auto_response { + // Safe point 5 (CR-01, 50-REVIEW.md): a response already on disk + // wins over a stop in the wait below, so a stop taken during the + // advance is honoured before the pre-authorized one is written. As + // at safe point 4, the gate is cleared, the stop recorded and an + // error unwinds every caller. + if crate::pipeline_launch::stop_requested_at_safe_point() { + let _ = Gates::cleanup(project_root, state.phase, stage); + state.gate_pending = false; + crate::pipeline_launch::record_advance_stop(state, 5)?; + return Err(CliError::Message(format!( + "the gate for stage {stage} was not auto-answered: {ADVANCE_STOP_REASON}" + ))); + } match Gates::respond(project_root, state.phase, stage, response) { Ok(_) => {} // A human or 23b's stale-gate sweep may have answered first; diff --git a/crates/devflow-cli/src/pipeline_launch.rs b/crates/devflow-cli/src/pipeline_launch.rs index e5dc7aa8..325073a9 100644 --- a/crates/devflow-cli/src/pipeline_launch.rs +++ b/crates/devflow-cli/src/pipeline_launch.rs @@ -1049,6 +1049,14 @@ fn stopped_at_safe_point(state: &mut State, safe_point: u8) -> Result bool { + with_stop_token(StopToken::safe_point) == Some(SafePoint::StopHere) +} + /// Persist a stop an advance honoured at `safe_point`: `stopped` with /// [`ADVANCE_STOP_REASON`], and an `advance_stopped` event naming the point. pub(crate) fn record_advance_stop(state: &mut State, safe_point: u8) -> Result<(), CliError> { From d6a73357415244a7310932ea44b7442dace0f5f7 Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Wed, 21 Oct 2026 15:48:57 -0400 Subject: [PATCH 097/112] test(50-26): add a test-support hold before the stream-thread wait - stream_wait_test_hold parks a stream supervisor after its exit file and before its stream threads are waited for; a stop or the release file ends it - compiled only under cfg(test) or the test-support feature, with its call - source scan G pins the gate, the gated call and the literal env read - DEVFLOW_TEST_STREAM_WAIT_HOLD allowlisted (doc_check failed naming it first) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGzZZbfn3r9638yptqPL47 --- crates/devflow-core/src/monitor.rs | 178 +++++++++++++++++++++++++++++ doc-check-allowlist.toml | 5 + 2 files changed, 183 insertions(+) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index dbea0a92..415a8deb 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -1350,6 +1350,10 @@ pub fn run_supervised_stage(spec: StageSpec) -> Result { }; std::fs::write(&exit_file, format!("{code}\n"))?; if let Some(threads) = threads { + // Test-only (50-26): park here, after the exit file and before the + // stream threads are waited for. Compiled out of release builds. + #[cfg(any(test, feature = "test-support"))] + stream_wait_test_hold(&supervisor.token()); threads.join(); } info!( @@ -1436,6 +1440,37 @@ fn monitor_test_hold(point: &str) { } } +/// How long [`stream_wait_test_hold`] waits for its release file: above the +/// bounded waits of the CLI test that uses it, combined. +#[cfg(any(test, feature = "test-support"))] +const STREAM_WAIT_HOLD_WAIT: Duration = Duration::from_secs(60); + +/// Test-only hold (50-26). When `DEVFLOW_TEST_STREAM_WAIT_HOLD` names a +/// path: create `.held`, then wait, polling every 20 ms and bounded by +/// [`STREAM_WAIT_HOLD_WAIT`], until `` exists or a stop is requested. +/// [`run_supervised_stage`] calls it for the stream shape after the exit +/// file is written and before the stream threads are waited for, so a +/// completed or stopped supervisor parks there: a CLI test can land +/// `devflow stop` while the owner is `advancing` and the agent's stdout is +/// still held, and the stop releases it. Returns at once when the variable +/// is absent. Compiled only under `cfg(test)` or the `test-support` feature, +/// together with its one call; a release build compiles neither (pinned by +/// `monitor::tests::the_stream_wait_hold_is_test_only_and_read_literally`). +#[cfg(any(test, feature = "test-support"))] +fn stream_wait_test_hold(token: &crate::supervisor::runtime::StopToken) { + let Some(release) = std::env::var_os("DEVFLOW_TEST_STREAM_WAIT_HOLD") else { + return; + }; + let release = PathBuf::from(release); + let mut held = release.clone().into_os_string(); + held.push(".held"); + let _ = std::fs::write(&held, ""); + let deadline = std::time::Instant::now() + STREAM_WAIT_HOLD_WAIT; + while !release.exists() && !token.stop_requested() && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } +} + /// Spawn `spec`'s agent in its own process group (`process_group(0)`, R-3), /// through `hermetic_command` with the adapter env re-applied after its /// scrub and the gate notify command stripped. Returns the child and, for @@ -6309,4 +6344,147 @@ exit 0 `spawn_monitor`; found {waits}" ); } + + // ---- 50-26: WR-03 --------------------------------------------------- + // + // A stream supervisor whose agent exited while a process it started + // still holds the agent's stdout must not wait on that stdout. Every + // 50-26 test, guard and helper lives in this block, so the + // `fix(50-26):` hunks stay disjoint from later edits (agy-5b). + + const STREAM_HOLD_NO_LITERAL_READ: &str = + "no literal std::env::var_os read of DEVFLOW_TEST_STREAM_WAIT_HOLD"; + const STREAM_HOLD_FN_NOT_ONCE: &str = "fn stream_wait_test_hold( is not defined exactly once"; + const STREAM_HOLD_NOT_GATED: &str = "fn stream_wait_test_hold( is not directly under #[cfg(any(test, feature = \"test-support\"))]"; + const STREAM_HOLD_CALL_NOT_GATED: &str = "a call to stream_wait_test_hold( is not directly under #[cfg(any(test, feature = \"test-support\"))]"; + + /// The stream-wait hold's layout over `source`'s non-test code (cut at + /// its first `mod tests {` line, whole-line `//` comments dropped): the + /// variable is read literally (the form the doc check sees), the hold is + /// defined once directly under the test-support gate, and every call is + /// directly under that gate too. + fn stream_wait_hold_violations(source: &str) -> Vec { + const GATE: &str = r#"#[cfg(any(test, feature = "test-support"))]"#; + const DEFINITION: &str = "fn stream_wait_test_hold("; + let lines: Vec<&str> = source + .lines() + .take_while(|line| *line != "mod tests {") + .filter(|line| !line.trim_start().starts_with("//")) + .collect(); + let gated = |at: usize| { + at.checked_sub(1) + .is_some_and(|above| lines[above].trim() == GATE) + }; + let mut violations = Vec::new(); + if !lines + .iter() + .any(|line| line.contains(r#"std::env::var_os("DEVFLOW_TEST_STREAM_WAIT_HOLD")"#)) + { + violations.push(STREAM_HOLD_NO_LITERAL_READ.to_string()); + } + let defined: Vec = lines + .iter() + .enumerate() + .filter(|(_, line)| line.trim_start().starts_with(DEFINITION)) + .map(|(at, _)| at) + .collect(); + match defined.as_slice() { + [at] => { + if !gated(*at) { + violations.push(STREAM_HOLD_NOT_GATED.to_string()); + } + } + _ => violations.push(STREAM_HOLD_FN_NOT_ONCE.to_string()), + } + let ungated_call = lines.iter().enumerate().any(|(at, line)| { + line.contains("stream_wait_test_hold(") + && !line.trim_start().starts_with(DEFINITION) + && !gated(at) + }); + if ungated_call { + violations.push(STREAM_HOLD_CALL_NOT_GATED.to_string()); + } + violations + } + + /// G (50-26, permanent source scan, T-50-101): a release build never + /// compiles the stream-wait hold or its call, and the hold's variable is + /// read literally. The real file yields no violation; each planted + /// source yields the violation it plants. + #[test] + fn the_stream_wait_hold_is_test_only_and_read_literally() { + let source = include_str!("monitor.rs"); + assert!( + source.lines().any(|line| line == "mod tests {"), + "monitor.rs must keep its `mod tests {{` line, or the scan would read its own controls" + ); + let real = stream_wait_hold_violations(source); + assert!(real.is_empty(), "monitor.rs: {real:?}"); + + let gate = "#[cfg(any(test, feature = \"test-support\"))]\n"; + let read = " let _path = std::env::var_os(\"DEVFLOW_TEST_STREAM_WAIT_HOLD\");\n"; + let hold = |attribute: &str, body: &str| { + format!("{attribute}fn stream_wait_test_hold(token: &StopToken) {{\n{body}}}\n") + }; + let caller = |attribute: &str| { + format!( + "fn run_supervised_stage() {{\n // Test-only.\n {attribute} stream_wait_test_hold(&token);\n}}\n" + ) + }; + let gated_call = format!(" {gate}"); + let good = format!("{}{}", hold(gate, read), caller(&gated_call)); + let only = |violation: &str| vec![violation.to_string()]; + + assert_eq!( + stream_wait_hold_violations(&good), + Vec::::new(), + "a correct layout (the comment line above the gate is dropped)" + ); + assert_eq!( + stream_wait_hold_violations(&format!("{}{}", hold("", read), caller(&gated_call))), + only(STREAM_HOLD_NOT_GATED), + "an ungated definition" + ); + assert_eq!( + stream_wait_hold_violations(&format!( + "{}{}", + hold("#[cfg(test)]\n", read), + caller(&gated_call) + )), + only(STREAM_HOLD_NOT_GATED), + "a definition gated by cfg(test) only" + ); + assert_eq!( + stream_wait_hold_violations(&format!("{}{}", hold(gate, read), caller(""))), + only(STREAM_HOLD_CALL_NOT_GATED), + "an ungated call" + ); + assert_eq!( + stream_wait_hold_violations(&format!( + "const HOLD: &str = \"DEVFLOW_TEST_STREAM_WAIT_HOLD\";\n{}{}", + hold(gate, " let _path = std::env::var_os(HOLD);\n"), + caller(&gated_call) + )), + only(STREAM_HOLD_NO_LITERAL_READ), + "a read through a constant" + ); + assert_eq!( + stream_wait_hold_violations(&format!( + "{}{}{}", + hold(gate, read), + hold(gate, ""), + caller(&gated_call) + )), + only(STREAM_HOLD_FN_NOT_ONCE), + "two definitions" + ); + assert_eq!( + stream_wait_hold_violations(&format!("mod tests {{\n{good}")), + vec![ + STREAM_HOLD_NO_LITERAL_READ.to_string(), + STREAM_HOLD_FN_NOT_ONCE.to_string() + ], + "the read and the hold only after the test-module cut" + ); + } } diff --git a/doc-check-allowlist.toml b/doc-check-allowlist.toml index 6da41482..bc18067c 100644 --- a/doc-check-allowlist.toml +++ b/doc-check-allowlist.toml @@ -69,3 +69,8 @@ reason = "Test-only: #[cfg(test)] hold points in the devflow-core monitor (50-24 kind = "source_to_docs" token = "DEVFLOW_TEST_STOPPED_OUTCOME_HOLD" reason = "Test-only: under the devflow-core test-support feature, Supervisor::wait_agent waits while it names a path that does not exist, after a stopped agent exits and before the stopped outcome is decided, so a CLI test can order that decision against the next stage's publish; compiled out of release builds, so it is not an operator setting (Phase 50, plan 50-23)." + +[[exceptions]] +kind = "source_to_docs" +token = "DEVFLOW_TEST_STREAM_WAIT_HOLD" +reason = "Test-only: under the devflow-core test-support feature, run_supervised_stage waits while it names a path that does not exist, or until a stop is requested, after a stream agent's exit is recorded and before its stream threads are waited for; compiled out of release builds, so it is not an operator setting (Phase 50, plan 50-26)." From 7af56e2fd23423764bff183b1c184ac8cc49087c Mon Sep 17 00:00:00 2001 From: Dennis Kim Date: Wed, 21 Oct 2026 15:54:53 -0400 Subject: [PATCH 098/112] test(50-26): reproduce a supervisor stuck on a held stdout (WR-03) RED record, on the unfixed code (nextest exit codes): - A a_completed_stream_run_returns_while_a_leftover_process_holds_the_agents_stdout: 100, WR-03: supervisor still runs 20 s after agent release, owner agent_running - B a_completed_stream_run_returns_once_its_leftover_process_released_the_agents_stdout: 0 (control: holder on /dev/null) - C no_line_reaches_the_capture_after_the_stdout_wait_expires: 100, WR-03: Ok(Ok(0)) only after 20.0 s (when the holder exited) - F a_stream_run_returns_while_a_leftover_process_keeps_writing_to_the_agents_stdout: 100, WR-03: Err(Timeout) after 20 s while the leftover kept writing - HeldStdout records the holder's pid and start time from the agent itself Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RGzZZbfn3r9638yptqPL47 --- crates/devflow-core/src/monitor.rs | 391 +++++++++++++++++++++++++++++ 1 file changed, 391 insertions(+) diff --git a/crates/devflow-core/src/monitor.rs b/crates/devflow-core/src/monitor.rs index 415a8deb..bca2004e 100644 --- a/crates/devflow-core/src/monitor.rs +++ b/crates/devflow-core/src/monitor.rs @@ -6487,4 +6487,395 @@ exit 0 "the read and the hold only after the test-module cut" ); } + + /// The process a fake agent left holding its stdout, as the agent itself + /// recorded it while the holder was still its unreaped child + /// (DeepSeek-3): ` `, where `` is `/proc//stat` + /// field 22 (set at fork, unchanged by `exec`). Dropped, it SIGKILLs the + /// holder by that recorded pid only, and only while it is still that + /// process; never by a name pattern. + struct HeldStdout { + pid: u32, + start: u64, + } + + impl HeldStdout { + /// Wait up to 10 s for the agent's holder record at `record`, and + /// require that it names a live process. + fn take(record: &Path) -> Self { + if !stage_wait_until(Duration::from_secs(10), || record.exists()) { + panic!( + "FIXTURE: the agent wrote no holder record at {} within 10 s", + record.display() + ); + } + let text = std::fs::read_to_string(record) + .unwrap_or_else(|err| panic!("FIXTURE: read {}: {err}", record.display())); + let fields: Vec<&str> = text.split_whitespace().collect(); + let parsed = match fields.as_slice() { + [pid, start] => pid.parse::().ok().zip(start.parse::().ok()), + _ => None, + }; + let Some((pid, start)) = parsed else { + panic!("FIXTURE: the holder record is not ` `: {text:?}"); + }; + if pid <= 1 || crate::agent::process_start_time(pid) != Some(start) { + panic!("FIXTURE: the holder record {text:?} names no live process"); + } + Self { pid, start } + } + + /// Whether the recorded holder is still that process, against the + /// start time recorded before the supervisor could return. + fn is_alive(&self) -> bool { + crate::agent::is_same_process(self.pid, self.start) + } + } + + impl Drop for HeldStdout { + fn drop(&mut self) { + let Ok(pid) = libc::pid_t::try_from(self.pid) else { + return; + }; + if pid > 1 && self.is_alive() { + // SAFETY: `pid > 1` is the holder this test's agent recorded, + // confirmed by its start time to still be that process. + unsafe { libc::kill(pid, libc::SIGKILL) }; + } + } + } + + /// The shell lines a fake agent runs right after starting its holder as + /// a background job: record ` ` while the holder is still + /// its unreaped child, through a temp file and `mv`. + fn record_holder(record: &Path) -> String { + format!( + "holder=$!\n\ + echo \"$holder $(cut -d' ' -f22 /proc/$holder/stat)\" > '{record}.tmp'\n\ + mv '{record}.tmp' '{record}'\n", + record = record.display() + ) + } + + /// Every `stream_wait_bounded` event for `phase` in `root`'s event log, + /// each as its JSON object. Fields are indexed on the map, so a missing + /// one panics instead of reading as a default. No log is zero events. + fn stream_wait_events( + root: &Path, + phase: PhaseId, + ) -> Vec> { + let path = crate::events::events_path(root); + let text = match std::fs::read_to_string(&path) { + Ok(text) => text, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Vec::new(), + Err(err) => panic!("read {}: {err}", path.display()), + }; + let phase = serde_json::json!(phase); + text.lines() + .map( + |line| match serde_json::from_str::(line) { + Ok(serde_json::Value::Object(map)) => map, + other => panic!("an event line is not a JSON object: {line:?} ({other:?})"), + }, + ) + .filter(|map| map["event"] == "stream_wait_bounded" && map["phase"] == phase) + .collect() + } + + /// Require exactly one `stream_wait_bounded` event for `phase`, naming + /// only the reader as unfinished, with the capture closed. + fn assert_one_reader_bound(root: &Path, phase: PhaseId, context: &str) { + let events = stream_wait_events(root, phase); + assert!( + events.len() == 1, + "WR-03: {context}: expected exactly one stream_wait_bounded event, found {events:?}" + ); + let event = &events[0]; + assert_eq!( + event["unfinished"], + serde_json::json!(["reader"]), + "{context}: {event:?}" + ); + assert_eq!( + event["capture_closed"], + serde_json::json!(true), + "{context}: {event:?}" + ); + } + + /// A fake Claude stream agent: it reads its first turn, prints + /// [`RESULT_WITH_MARKER`], starts `holder` as a background job (it + /// inherits the agent's stdout unless `holder` redirects it, and stays + /// in the agent's group), records it at `record`, waits for `go` and + /// exits 0. + fn holder_agent(fixture: &StageFixture, holder: &str, record: &Path, go: &Path) -> PathBuf { + fixture.agent( + "claude", + &format!( + "{preamble}printf '%s\\n' '{RESULT_WITH_MARKER}'\n\ + {holder} &\n\ + {record_lines}\ + while [ ! -e '{go}' ]; do sleep 0.05; done\n\ + exit 0", + preamble = Shape::Stream.preamble(), + record_lines = record_holder(record), + go = go.display(), + ), + ) + } + + /// A and B: a supervised Claude stream stage (production 120 s idle + /// timeout) whose agent leaves `holder` running, then exits 0. The + /// supervisor must exit within 20 s of the agent's exit and record a + /// normal completion; `stdout_held` says whether exactly one + /// `stream_wait_bounded` event or none is expected. + fn completed_run_with_a_leftover(test_name: &str, holder: &str, stdout_held: bool) { + let fixture = StageFixture::new(); + let go = fixture.path("go"); + let record = fixture.path("holder"); + let program = holder_agent(&fixture, holder, &record, &go); + let mut run = launch_stage_with( + test_name, + &fixture, + StageLaunchArgs { + shape: Shape::Stream, + agent: AgentKind::Claude, + program: &program, + adapter_env: "", + args: &[], + extra_env: &[], + }, + ); + let held = HeldStdout::take(&record); + std::fs::write(&go, "").expect("let the agent exit"); + if !stage_wait_until(Duration::from_secs(20), || run.child.try_exited()) { + panic!( + "WR-03: the supervisor still runs 20 s after its agent was released to exit; \ + holder {} alive: {}; socket: {:?}", + held.pid, + held.is_alive(), + crate::supervisor::probe(run.socket.as_path()) + ); + } + let outcome = run.finish(test_name); + assert_eq!( + outcome, + "Completed(0) advancing", + "stderr: {:?}", + fixture.stderr() + ); + let stdout = fixture.stdout(); + assert!( + stdout.lines().any(|line| line == RESULT_WITH_MARKER), + "the capture holds the result line: {stdout:?}" + ); + assert_eq!(fixture.exit_code(), "0"); + assert!( + !crate::agent_result::idle_timeout_path(&fixture.root, STAGE_PHASE).exists(), + "no idle-timeout verdict for a completed run" + ); + assert_eq!(run.liveness(), crate::supervisor::Liveness::Gone); + assert!( + held.is_alive(), + "the holder still runs: the supervisor's wait ended, not the holder" + ); + if stdout_held { + assert_one_reader_bound(&fixture.root, STAGE_PHASE, test_name); + } else { + let events = stream_wait_events(&fixture.root, STAGE_PHASE); + assert!( + events.is_empty(), + "a released stdout is not a bounded wait: {events:?}" + ); + } + drop(run.group); + } + + const HELD_STDOUT_TEST: &str = "monitor::tests::a_completed_stream_run_returns_while_a_leftover_process_holds_the_agents_stdout"; + + /// A (fail-first): a completed stream run whose agent left `sleep 300` + /// holding its stdout returns at the agent's exit, under the production + /// 120 s idle timeout. Before the fix the supervisor waited 120 s of + /// silence in `drive_stream_io` and then forever in the thread join. + #[test] + fn a_completed_stream_run_returns_while_a_leftover_process_holds_the_agents_stdout() { + if crate::test_support::in_child_test(HELD_STDOUT_TEST) { + supervise_stage_in_child(); + return; + } + completed_run_with_a_leftover(HELD_STDOUT_TEST, "sleep 300", true); + } + + const RELEASED_STDOUT_TEST: &str = "monitor::tests::a_completed_stream_run_returns_once_its_leftover_process_released_the_agents_stdout"; + + /// B (control, the verifier's `/dev/null` control): the same run with a + /// holder on `/dev/null` returns normally on the unfixed and the fixed + /// code, and records no bounded wait. + #[test] + fn a_completed_stream_run_returns_once_its_leftover_process_released_the_agents_stdout() { + if crate::test_support::in_child_test(RELEASED_STDOUT_TEST) { + supervise_stage_in_child(); + return; + } + completed_run_with_a_leftover( + RELEASED_STDOUT_TEST, + "sleep 300 /dev/null 2>&1", + false, + ); + } + + /// Run [`supervise_stream_child`] for a Claude `sh -c