Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 22 additions & 6 deletions crates/crew/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,12 @@ fn running_as_agent() -> bool {
}

pub fn ensure_daemon() -> anyhow::Result<()> {
// Someone else is mid-boot: the socket is bound but the roster is still
// opening. Starting another would be refused anyway, and the version is
// not readable until it is up, so wait rather than judge it.
if paths::is_socket_live() && !paths::daemon_is_ready() {
wait_ready(Duration::from_secs(4));
}
if paths::is_socket_live() {
if paths::daemon_version_matches(env!("CARGO_PKG_VERSION")) {
return Ok(());
Expand Down Expand Up @@ -160,12 +166,11 @@ pub fn ensure_daemon() -> anyhow::Result<()> {
}
cmd.spawn().context("spawn crew daemon")?;

let deadline = Instant::now() + Duration::from_secs(4);
while Instant::now() < deadline {
if paths::is_socket_live() {
return Ok(());
}
thread::sleep(Duration::from_millis(50));
// Ready, not merely bound: returning on the bind would hand the caller a
// socket that accepts the connection and answers nothing until the roster
// has finished opening.
if wait_ready(Duration::from_secs(4)) {
return Ok(());
}
let tail = std::fs::read_to_string(paths::log_path()).unwrap_or_default();
let tail = tail
Expand All @@ -180,6 +185,17 @@ pub fn ensure_daemon() -> anyhow::Result<()> {
bail!("daemon did not start. log tail:\n{tail}")
}

fn wait_ready(timeout: Duration) -> bool {
let deadline = Instant::now() + timeout;
while Instant::now() < deadline {
if paths::daemon_is_ready() {
return true;
}
thread::sleep(Duration::from_millis(50));
}
false
}

pub fn stop_daemon() -> anyhow::Result<()> {
if paths::is_socket_live() {
match rpc(Request::Shutdown) {
Expand Down
84 changes: 78 additions & 6 deletions crates/crew/src/daemon.rs
Original file line number Diff line number Diff line change
Expand Up @@ -896,7 +896,32 @@ pub async fn run() -> anyhow::Result<()> {
);
}
paths::remove_stale_socket();
// Cleared here and written once the accept loop is up, so it reads as
// "serving" rather than "was serving at some point".
paths::remove_daemon_version();

// Bind before anything is spawned. A live socket is how a second daemon —
// and `client::ensure_daemon` — decides one is already coming up, and
// binding last left the whole roster opening inside a window where that
// check still said no. Two daemons could each spawn every agent onto the
// same cwd and transcripts, and the one that lost the bind returned
// through `?` without reaching `shutdown_agents`, abandoning its children.
let listener = UnixListener::bind(paths::socket_path())
.with_context(|| format!("bind {}", paths::socket_path().display()))?;

// The socket is ours from here, so no path out may leave it behind:
// `is_socket_live` would keep answering yes and `ensure_daemon` would wait
// out its deadline against a daemon that is gone.
let served = serve(listener).await;
shutdown_agents();
paths::remove_stale_socket();
paths::remove_pid();
paths::remove_daemon_version();
let _ = events().send(Event::Shutdown);
served
}

async fn serve(listener: UnixListener) -> anyhow::Result<()> {
crate::transcript::set_seal_hook(on_assistant_sealed);

let cfg = Config::load()?;
Expand Down Expand Up @@ -940,8 +965,6 @@ pub async fn run() -> anyhow::Result<()> {
}
let _ = write_roster(&cfg.agents, &cfg.channels);

let listener = UnixListener::bind(paths::socket_path())
.with_context(|| format!("bind {}", paths::socket_path().display()))?;
paths::write_pid(std::process::id())?;
paths::write_daemon_version(env!("CARGO_PKG_VERSION"))?;
eprintln!(
Expand Down Expand Up @@ -983,10 +1006,6 @@ pub async fn run() -> anyhow::Result<()> {
}
}

shutdown_agents();
paths::remove_stale_socket();
paths::remove_pid();
let _ = events().send(Event::Shutdown);
Ok(())
}

Expand Down Expand Up @@ -2782,6 +2801,59 @@ mod daemon_tests {
});
}

/// The socket, pid and version file belong to this process only while it
/// is running. Leaving any of them behind makes `is_socket_live` answer
/// yes for a daemon that is gone, and `ensure_daemon` then waits out its
/// whole deadline before saying so.
#[test]
fn a_daemon_leaves_nothing_behind_when_it_stops() {
paths::testing::with_home("stop", || {
paths::ensure_home().expect("home");
fs::write(paths::agents_path(), r#"{"agents":[],"channels":[]}"#).expect("write");

let boot = std::thread::spawn(|| {
let rt = tokio::runtime::Runtime::new().expect("runtime");
rt.block_on(run())
});
for _ in 0..200 {
if paths::is_socket_live() {
break;
}
std::thread::sleep(Duration::from_millis(25));
}
assert!(paths::is_socket_live(), "the daemon should be listening");

crate::client::rpc(crate::protocol::Request::Shutdown).expect("shutdown");
boot.join().expect("join").expect("clean exit");

assert!(!paths::socket_path().exists(), "socket outlived the daemon");
assert!(!paths::pid_path().exists(), "pid outlived the daemon");
assert!(
!paths::daemon_version_path().exists(),
"version outlived the daemon"
);
});
}

/// Boot binds before it opens anything, so the second daemon has to see
/// the first through a socket that is bound and not yet accepting — which
/// is the whole point of binding first.
#[test]
fn a_bound_socket_stops_a_second_daemon_before_it_spawns() {
paths::testing::with_home("bind", || {
paths::ensure_home().expect("home");
let held = std::os::unix::net::UnixListener::bind(paths::socket_path()).expect("bind");
assert!(
paths::is_socket_live(),
"a bound listener counts as live before it accepts"
);
let rt = tokio::runtime::Runtime::new().expect("runtime");
let err = rt.block_on(run()).expect_err("a second daemon must bail");
assert!(err.to_string().contains("already running"), "{err:#}");
drop(held);
});
}

#[test]
fn a_hash_prefix_addresses_a_channel() {
assert!(matches!(routine_host("alpha"), RoutineHost::Agent(id) if id == "alpha"));
Expand Down
9 changes: 9 additions & 0 deletions crates/crew/src/paths.rs
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,15 @@ pub fn daemon_version_matches(current: &str) -> bool {
read_daemon_version().as_deref() == Some(current)
}

/// A daemon that is bound *and* serving. The socket binds before the roster
/// opens, so that a second daemon cannot start one too — which leaves a window
/// where connecting succeeds but nothing answers yet. The version file is
/// written once the accept loop is up and removed on the way out, so it is
/// what "ready" means.
pub fn daemon_is_ready() -> bool {
daemon_version_path().exists() && is_socket_live()
}

pub fn create_cwd(path: &Path) -> anyhow::Result<()> {
fs::create_dir_all(path)?;
Ok(())
Expand Down
Loading