From 08a31bd75b40a82e36616dd2c38031013d2d90f4 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 14:10:18 +0500 Subject: [PATCH 01/15] feat: degrade to in-memory reporting on read-only/ephemeral file systems (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve the data directory as explicit option, BUGSEE_DATA_DIR, platform default, then temp dir, taking the first that is actually writable. If none is, or the disk fails mid-run (full, read-only, removed), launch still succeeds and handled errors / manual reports are assembled without capture context and delivered from a bounded in-memory backlog. Native crash handler and panic snapshots are skipped in memory mode. Adds Bugsee::storage_status() so a host can check crash recovery is available and whether the dir may be ephemeral. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 12 + crates/bugsee-core/src/reporting.rs | 28 +- crates/bugsee-core/src/runtime.rs | 410 +++++++++++++++---- crates/bugsee-core/src/util.rs | 11 + crates/bugsee-core/tests/storage_failures.rs | 239 +++++++++++ crates/bugsee/src/api.rs | 81 +++- crates/bugsee/src/lib.rs | 4 +- crates/bugsee/src/options.rs | 278 ++++++++++++- crates/bugsee/tests/storage_fallback.rs | 107 +++++ 9 files changed, 1082 insertions(+), 88 deletions(-) create mode 100644 crates/bugsee-core/tests/storage_failures.rs create mode 100644 crates/bugsee/tests/storage_fallback.rs diff --git a/DESIGN.md b/DESIGN.md index badb64a..3029573 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -181,6 +181,18 @@ Global base scope behind `RwLock`; `with_scope` pushes a thread-local overlay. * **Durability envelope (be honest):** buffered append survives **process death** (panic, `abort`, signal, `kill -9`) because completed `write()`s live in the kernel page cache independent of the crashing process β€” this is what the `recovery_e2e` / `kill -9` tests exercise. It does **not** guarantee survival of **power loss / kernel panic**, since nothing is `fsync`'d; a hardening tier (periodic `fsync`, or the originally-envisioned `mmap(MAP_SHARED)` + `msync`) is **future work**, not current behavior. +**Data directory resolution and failure modes (read-only / ephemeral hosts).** The facade picks the first **writable** candidate (create + write-probe, not mere existence), skipping and recording any that fail: (1) `LaunchOptions::data_dir`; (2) `BUGSEE_DATA_DIR`; (3) platform default `bugsee/` under `$XDG_DATA_HOME`/`~/.local/share` (Linux), `~/Library/Application Support` (macOS), `%LOCALAPPDATA%` (Windows), skipped if the base is unset (container with no home); (4) `/bugsee/`. Launch never fails for storage reasons. Outcomes: + +| Failure | Outcome | +|---|---| +| Preferred candidate unusable, a later one works | falls through; one stderr line; `StorageStatus::skipped` lists why | +| No candidate usable, or `Session::begin`/`PartStore::new` fails | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from a bounded in-memory backlog (16; overflow drops the newest with `DropReason::StorageUnavailable`); an unsent report dies with the process | +| Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and silently dropped; a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out | +| Disk full / permission denied at enqueue, snapshot, marker | same as above; the `reports/` snapshot dir is reclaimed by `SnapshotHandle::drop` | +| Crash handler cannot write its marker | the raw `open`/`write` return values are ignored; the process still dies with the original signal (watchdog bounds the handler) | + +Crash recovery needs the data directory to **outlive the process**: on a container, mount a volume and point `data_dir`/`BUGSEE_DATA_DIR` at it. The temp fallback is commonly wiped on restart. `Bugsee::storage_status()` reports `mode`, `data_dir`, `source`, `skipped`, `crash_recovery_available()` and `may_be_ephemeral()` (the SDK cannot tell a volume from a scratch disk; the temp fallback is the only case it flags). Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised; failures are injected by blocking path components and removing the directory mid-run. + **Generations:** each launch bumps `gen`; next-launch recovery reads the **previous** gen's parts before advancing. **Snapshot (no seal, no copy):** pin `[start,end]` (`end`=snapshot instant, `start`=`endβˆ’window`) + **hard-link current parts including the active one** into the report dir (`PartStore::snapshot_into`, after a `flush`). At export, read linked parts and include only entries with `start ≀ timestamp ≀ end` (monotonic order β†’ early-stop at the tail; the self-terminating `RecordIter` cleanly ignores any torn trailing write on the active part). Bounds become `manifest.time.start/end`. Cost: 2 timestamps + O(#parts) `link` syscalls. Fallback to copy on FAT/exFAT/cross-volume (capability-probed once). Reflink (APFS `clonefile` / Linux `FICLONE`) a noted future tier. diff --git a/crates/bugsee-core/src/reporting.rs b/crates/bugsee-core/src/reporting.rs index 1fdcb4c..3116578 100644 --- a/crates/bugsee-core/src/reporting.rs +++ b/crates/bugsee-core/src/reporting.rs @@ -14,7 +14,7 @@ use std::path::Path; use serde_json::{Map, Value}; use crate::bundle::{bundle_filename, write_bundle, BundleEntry}; -use crate::capture::export_report; +use crate::capture::{export_report, ChannelDocument}; use crate::model::enums::{IssueType, Severity}; use crate::model::environment::Environment; use crate::model::report::{FileDescriptor, IssueRequest, Manifest, Source, TimeWindow}; @@ -99,7 +99,33 @@ pub fn assemble_with_extras( created_on_ms: i64, ) -> std::io::Result { let docs = export_report(report_dir, window.start, window.end)?; + assemble_docs( + docs, + window, + meta, + env, + crash_json, + extra_files, + app_token, + created_on_ms, + ) +} +/// Like [`assemble_with_extras`] but from already-exported capture `docs` +/// instead of a snapshot directory. With an empty `docs` it builds a bundle that +/// needs no file system at all: the in-memory delivery path used when the data +/// directory is unusable (read-only, full, gone). +#[allow(clippy::too_many_arguments)] +pub fn assemble_docs( + docs: Vec, + window: TimeWindow, + meta: &ReportMeta, + env: &Environment, + crash_json: Option>, + extra_files: &[ExtraFile], + app_token: &str, + created_on_ms: i64, +) -> std::io::Result { let mut files: Vec = Vec::new(); let mut entries: Vec = Vec::new(); diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index fcb2ecf..faf62cc 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -61,6 +61,30 @@ const DEFAULT_MAX_QUEUED: usize = 8192; /// NON-crash reports are dropped-newest; crash reports are never dropped. const DEFAULT_MAX_REPORT_QUEUED: usize = 256; +/// Where a recorder keeps its state. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StorageMode { + /// The data directory is usable: capture parts, the liveness marker and the + /// retry queue are on disk, so a crash can be recovered on the next launch. + Disk, + /// No usable data directory (read-only, missing, full, ...). Handled errors + /// and manual reports are still delivered, from memory, but there is no + /// capture window and nothing survives the process: fatal crashes cannot be + /// recovered and an unsent report is lost on exit. + Memory, +} + +/// Most reports held in memory awaiting delivery. Reports are rare and a bundle +/// is small, so this only bounds a pathological storm while offline. +const MAX_MEMORY_REPORTS: usize = 16; + +/// A report waiting in memory for delivery, because it could not be queued on disk. +struct MemReport { + report: AssembledReport, + retry: u32, + next_attempt_ms: i64, +} + /// A callback run on every report before delivery. Mutate the metadata in place; /// return `false` to drop the report entirely. pub type BeforeSend = Box bool + Send + Sync>; @@ -83,6 +107,10 @@ pub enum DropReason { TooManySimilar, /// Delivery kept failing transiently until the durable retry cap. RetriesExhausted, + /// The report could not be stored on disk and the in-memory backlog was + /// full, so it was dropped rather than queued. Only seen with an unusable + /// data directory. + StorageUnavailable, } /// Called when a report is abandoned. See [`DropReason`]. @@ -136,6 +164,9 @@ pub struct RecorderConfig { pub sample_rate: f64, /// Max in-flight capture entries before new ones are dropped (back-pressure). pub max_queued_entries: usize, + /// Skip the disk entirely and run in [`StorageMode::Memory`]. For a host + /// layer that already established no data directory is usable. + pub memory_only: bool, } impl RecorderConfig { @@ -156,6 +187,7 @@ impl RecorderConfig { before_breadcrumb: None, sample_rate: 1.0, max_queued_entries: DEFAULT_MAX_QUEUED, + memory_only: false, } } } @@ -199,6 +231,8 @@ struct Shared { max_report_queued: usize, /// Whether telemetry sampling is paused (mirrors the facade's pause state). paused: AtomicBool, + /// Reports that could not be queued on disk, delivered from memory. + mem_queue: ForkLock>, } /// The recorder's own locks join the `fork()` lock protocol, so a forked child @@ -208,8 +242,10 @@ impl ForkLockable for Shared { fn fork_lock(&self) { self.scope.fork_lock(); self.session.fork_lock(); + self.mem_queue.fork_lock(); } fn fork_unlock(&self) { + self.mem_queue.fork_unlock(); self.session.fork_unlock(); self.scope.fork_unlock(); } @@ -253,7 +289,10 @@ pub struct Recorder { shared: Arc, caps: WindowCaps, data_dir: PathBuf, - session: Session, + /// `None` in memory mode: there is no liveness marker to own. + session: Option, + storage: StorageMode, + storage_reason: Option, rotate_interval: Duration, backoff_base: Duration, worker: Option>, @@ -294,17 +333,40 @@ impl Recorder { report_queued: AtomicUsize::new(0), max_report_queued: DEFAULT_MAX_REPORT_QUEUED, paused: AtomicBool::new(false), + mem_queue: ForkLock::new(Vec::new()), }); crate::fork::install_fork_hook(); crate::fork::register_weak(&shared); - let session = Session::begin(&config.data_dir)?; let data_dir = config.data_dir.clone(); + // An unusable data directory degrades to memory mode instead of failing + // the launch: the host must not lose error reporting (or its startup) + // because a volume is read-only or full. + let (disk, storage_reason) = if config.memory_only { + (None, Some("no usable data directory".to_string())) + } else { + match open_disk(&data_dir, config.caps) { + Ok(disk) => (Some(disk), None), + Err(e) => (None, Some(format!("{}: {e}", data_dir.display()))), + } + }; + if let Some(reason) = &storage_reason { + crate::util::warn_once(&format!( + "data directory unusable ({reason}); running in memory only: handled errors \ + and manual reports are still sent, but crashes cannot be recovered and \ + nothing survives this process" + )); + } + let (session, store) = match disk { + Some((session, store)) => (Some(session), Some(store)), + None => (None, None), + }; let pipeline = spawn_pipeline( &shared, - &data_dir, - session.generation(), + store.is_some().then_some(data_dir.as_path()), + session.as_ref().map_or(0, Session::generation), + store, config.caps, config.rotate_interval, config.upload_backoff_base, @@ -318,6 +380,12 @@ impl Recorder { shared, caps: config.caps, data_dir, + storage: if session.is_some() { + StorageMode::Disk + } else { + StorageMode::Memory + }, + storage_reason, session, rotate_interval: config.rotate_interval, backoff_base: config.upload_backoff_base, @@ -359,11 +427,19 @@ impl Recorder { self.shared.queued.store(0, Ordering::SeqCst); self.shared.report_queued.store(0, Ordering::SeqCst); - let session = Session::begin(&self.data_dir)?; + let disk = match self.storage { + StorageMode::Disk => Some(open_disk(&self.data_dir, self.caps)?), + StorageMode::Memory => None, + }; + let (session, store) = match disk { + Some((session, store)) => (Some(session), Some(store)), + None => (None, None), + }; let pipeline = spawn_pipeline( &self.shared, - &self.data_dir, - session.generation(), + store.is_some().then_some(self.data_dir.as_path()), + session.as_ref().map_or(0, Session::generation), + store, self.caps, self.rotate_interval, self.backoff_base, @@ -462,6 +538,18 @@ impl Recorder { &self.data_dir } + /// Where this recorder keeps its state. In [`StorageMode::Memory`] crash + /// capture is pointless (nothing could be recovered), so the host layer + /// should not install the native handler. + pub fn storage_mode(&self) -> StorageMode { + self.storage + } + + /// Why the recorder fell back to [`StorageMode::Memory`], if it did. + pub fn storage_reason(&self) -> Option<&str> { + self.storage_reason.as_deref() + } + /// Path where the native crash handler should write its crash-info marker. pub fn crash_info_path(&self) -> PathBuf { self.gen_dir().join(crate::recovery::CRASH_INFO_NAME) @@ -473,9 +561,12 @@ impl Recorder { } fn gen_dir(&self) -> PathBuf { - self.data_dir - .join("parts") - .join(self.session.generation().to_string()) + self.data_dir.join("parts").join( + self.session + .as_ref() + .map_or(0, Session::generation) + .to_string(), + ) } /// Configured window caps. @@ -525,7 +616,9 @@ impl Drop for Recorder { // unwinding, and its snapshot is deleted by `report_caught`, so this // cannot turn a handled panic into a phantom crash. if worker_acked && uploader_acked && !std::thread::panicking() { - self.session.end(); + if let Some(session) = &self.session { + session.end(); + } } } } @@ -660,9 +753,19 @@ fn report_into( } } -/// Recover any prior generation that ended abnormally, queueing it for delivery. -/// The crashed session's state is discarded ONLY after the report is durably -/// enqueued β€” a failure leaves it on disk for a later attempt (no crash loss). +/// Claim a session and open its part store, or fail without leaving a liveness +/// marker behind (a marker with no store would be recovered as a phantom crash). +fn open_disk(data_dir: &Path, caps: WindowCaps) -> std::io::Result<(Session, PartStore)> { + let session = Session::begin(data_dir)?; + match PartStore::new(data_dir, session.generation(), caps) { + Ok(store) => Ok((session, store)), + Err(e) => { + session.end(); + Err(e) + } + } +} + /// Start the uploader and the capture worker for `generation`. /// /// `fresh_launch` is true for a process's first pipeline and false for one @@ -671,8 +774,9 @@ fn report_into( #[allow(clippy::too_many_arguments)] fn spawn_pipeline( shared: &Arc, - data_dir: &Path, + data_dir: Option<&Path>, generation: u64, + store: Option, caps: WindowCaps, rotate_interval: Duration, backoff_base: Duration, @@ -682,7 +786,7 @@ fn spawn_pipeline( // Uploader thread: drains the durable queue with retry. let (upload_tx, upload_rx) = channel(); let uploader_shared = Arc::clone(shared); - let uploader_data_dir = data_dir.to_path_buf(); + let uploader_data_dir = data_dir.map(Path::to_path_buf); let uploader = std::thread::Builder::new() .name("bugsee-uploader".into()) .spawn(move || { @@ -697,9 +801,8 @@ fn spawn_pipeline( // Capture worker: owns the part store; enqueues reports. let (tx, rx) = channel(); - let store = PartStore::new(data_dir, generation, caps)?; let worker_shared = Arc::clone(shared); - let worker_data_dir = data_dir.to_path_buf(); + let worker_data_dir = data_dir.map(Path::to_path_buf); let worker_upload_tx = upload_tx.clone(); let worker = std::thread::Builder::new() @@ -713,15 +816,17 @@ fn spawn_pipeline( // first (runs before this session enqueues anything). Only a fresh // launch: a fork child's parent may be mid-enqueue right now, and // its in-flight sidecars would look orphaned. - if fresh_launch { - queue::gc_orphans(&worker_data_dir); + if let (true, Some(dir)) = (fresh_launch, &worker_data_dir) { + queue::gc_orphans(dir); } // A fork child recovers too. A prefork master recovers only at ITS // launch, and its own generation is the lowest, so it can never // recover a worker that died later; the next worker started from // it must, or that crash waits for the master to restart. Claims // keep siblings from recovering the same session twice. - run_recovery(&worker_shared, &worker_data_dir, generation); + if let Some(dir) = &worker_data_dir { + run_recovery(&worker_shared, dir, generation); + } })); let _ = worker_upload_tx.send(UploadMsg::Wake); worker_loop( @@ -744,6 +849,9 @@ fn spawn_pipeline( }) } +/// Recover any prior generation that ended abnormally, queueing it for delivery. +/// The crashed session's state is discarded ONLY after the report is durably +/// enqueued β€” a failure leaves it on disk for a later attempt (no crash loss). fn run_recovery(shared: &Shared, data_dir: &Path, current_generation: u64) { for pending in recovery::find_pending(data_dir, current_generation) { // Several processes share this directory (a prefork master and its @@ -827,10 +935,10 @@ fn merge_scope(shared: &Shared, meta: &mut ReportMeta) { #[allow(clippy::too_many_arguments)] fn worker_loop( rx: Receiver, - mut store: PartStore, + mut store: Option, shared: Arc, rotate_interval: Duration, - data_dir: PathBuf, + data_dir: Option, caps: WindowCaps, mut sampler: Option>, upload_tx: Sender, @@ -850,6 +958,8 @@ fn worker_loop( shared.queued.fetch_sub(1, Ordering::Relaxed); // A panic while serializing/appending must not kill the worker. let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + // Memory mode has no capture window: the entry is dropped. + let Some(store) = store.as_mut() else { return }; if let Ok(value) = entry.to_json() { if let Ok(bytes) = serde_json::to_vec(&value) { let _ = store.append(entry.wire_type(), entry.timestamp(), &bytes); @@ -865,7 +975,13 @@ fn worker_loop( shared.report_queued.fetch_sub(1, Ordering::Relaxed); let ok = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { process_report( - &mut store, &shared, &data_dir, caps, meta, window_end, crash_json, + store.as_mut(), + &shared, + data_dir.as_deref(), + caps, + meta, + window_end, + crash_json, ) .is_ok() })) @@ -876,7 +992,7 @@ fn worker_loop( } Ok(Msg::Snapshot { window_end, ack }) => { let handle = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - take_snapshot(&mut store, &data_dir, caps, window_end) + take_snapshot(store.as_mut()?, data_dir.as_deref()?, caps, window_end) })) .ok() .flatten(); @@ -884,7 +1000,7 @@ fn worker_loop( } Ok(Msg::UploadSnapshot { handle, meta }) => { let ok = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - deliver_snapshot(&shared, &data_dir, handle, meta) + deliver_snapshot(&shared, data_dir.as_deref(), handle, meta) })) .unwrap_or(false); if ok { @@ -892,11 +1008,15 @@ fn worker_loop( } } Ok(Msg::Flush(ack)) => { - let _ = store.flush(); + if let Some(store) = store.as_mut() { + let _ = store.flush(); + } let _ = ack.send(()); } Ok(Msg::Stop(ack)) => { - let _ = store.flush(); + if let Some(store) = store.as_mut() { + let _ = store.flush(); + } let _ = ack.send(()); break; } @@ -906,7 +1026,7 @@ fn worker_loop( // Run the rotation/eviction/telemetry tick whenever the interval has // elapsed β€” via a message OR a timeout β€” so steady traffic can't starve it. if Instant::now() >= next_rotate { - rotate_tick(&mut store, &shared, &mut sampler); + rotate_tick(store.as_mut(), &shared, &mut sampler); next_rotate = Instant::now() + rotate_interval; } } @@ -916,7 +1036,7 @@ fn worker_loop( /// window. Panic-contained so a host `TelemetrySampler` panic can't kill the /// worker. fn rotate_tick( - store: &mut PartStore, + mut store: Option<&mut PartStore>, shared: &Shared, sampler: &mut Option>, ) { @@ -934,13 +1054,15 @@ fn rotate_tick( value, custom: Default::default(), }; - if let Ok(bytes) = serde_json::to_vec(&entry) { + if let (Some(store), Ok(bytes)) = (store.as_mut(), serde_json::to_vec(&entry)) { let _ = store.append("traces.system", ts, &bytes); } } } } - let _ = store.rotate(); + if let Some(store) = store.as_mut() { + let _ = store.rotate(); + } })); } @@ -971,7 +1093,7 @@ fn take_snapshot( /// Assemble + enqueue a report from a pre-taken snapshot (deferred upload path). fn deliver_snapshot( shared: &Shared, - data_dir: &Path, + data_dir: Option<&Path>, handle: SnapshotHandle, mut meta: ReportMeta, ) -> bool { @@ -982,7 +1104,7 @@ fn deliver_snapshot( } } // The snapshot dir is reclaimed by `handle`'s Drop on return (any path). - reporting::assemble( + let assembled = reporting::assemble( &handle.dir, handle.window, &meta, @@ -991,14 +1113,73 @@ fn deliver_snapshot( &shared.app_token, handle.window.end, ) - .and_then(|assembled| queue::enqueue(data_dir, &assembled)) - .is_ok() + // The snapshot became unreadable (disk gone): still send what we know. + .or_else(|_| assemble_in_memory(shared, &meta, handle.window, None)); + match assembled { + Ok(assembled) => persist_or_hold(shared, data_dir, assembled), + Err(_) => false, + } +} + +/// Queue `assembled` on disk, or hold it in memory if the disk refuses it +/// (read-only, full, directory removed) or there is no disk. Always `true`: the +/// report is held either way. +fn persist_or_hold(shared: &Shared, data_dir: Option<&Path>, assembled: AssembledReport) -> bool { + if let Some(dir) = data_dir { + if queue::enqueue(dir, &assembled).is_ok() { + return true; + } + // A failed enqueue can leave a partial sidecar set; the next launch's + // orphan sweep reaps it. Do not retry the disk for this report. + } + hold_in_memory(shared, assembled); + true +} + +/// Assemble a bundle with no capture context and no file system access. +fn assemble_in_memory( + shared: &Shared, + meta: &ReportMeta, + window: TimeWindow, + crash_json: Option>, +) -> std::io::Result { + reporting::assemble_docs( + Vec::new(), + window, + meta, + &shared.env, + crash_json, + &[], + &shared.app_token, + window.end, + ) +} + +/// Park a report in the bounded in-memory queue for the uploader. Over the cap +/// the NEWEST report is dropped (and the host told), keeping the backlog bounded. +fn hold_in_memory(shared: &Shared, report: AssembledReport) { + let full = { + let mut queue = shared.mem_queue.lock(); + if queue.len() >= MAX_MEMORY_REPORTS { + true + } else { + queue.push(MemReport { + report, + retry: 0, + next_attempt_ms: 0, + }); + false + } + }; + if full { + notify_dropped(shared, DropReason::StorageUnavailable); + } } fn process_report( - store: &mut PartStore, + store: Option<&mut PartStore>, shared: &Shared, - data_dir: &Path, + data_dir: Option<&Path>, caps: WindowCaps, mut meta: ReportMeta, window_end: i64, @@ -1022,49 +1203,80 @@ fn process_report( } // Suppress reports the server blacklisted (12004 TooManySimilar). - if queue::any_blacklisted(data_dir, &meta.signatures) { - return Ok(()); + if let Some(dir) = data_dir { + if queue::any_blacklisted(dir, &meta.signatures) { + return Ok(()); + } } let window_start = window_end.saturating_sub(caps.max_window_ms); - let report_id = crate::util::random_hex(8); - let report_dir = data_dir.join("reports").join(&report_id); - std::fs::create_dir_all(&report_dir)?; - - // Own the report dir with a `SnapshotHandle` so EVERY exit path reclaims its - // hard links via Drop. The earlier manual `remove_dir_all` ran only after a - // successful assemble, so a `snapshot_into`/`assemble` failure (most plausibly - // disk-full) permanently orphaned a `reports/` dir full of hard-linked - // parts, and `reports/` was never GC'd (F7). let window = TimeWindow { start: window_start, end: window_end, }; + + // With a usable disk, snapshot + assemble from it. Any failure along the way + // (most plausibly disk-full or a vanished directory) falls back to a bundle + // without capture context, so the report still goes out. + let from_disk = match (store, data_dir) { + (Some(store), Some(data_dir)) => { + assemble_from_disk(store, shared, data_dir, &meta, window, crash_json.clone()) + } + _ => None, + }; + let assembled = match from_disk { + Some(assembled) => assembled, + None => assemble_in_memory(shared, &meta, window, crash_json)?, + }; + + // Persist to the durable queue; the uploader delivers it. Failing that, hold + // it in memory. + persist_or_hold(shared, data_dir, assembled); + Ok(()) +} + +/// Snapshot the window into a fresh report dir and assemble it, or `None` if the +/// disk refuses any step. +fn assemble_from_disk( + store: &mut PartStore, + shared: &Shared, + data_dir: &Path, + meta: &ReportMeta, + window: TimeWindow, + crash_json: Option>, +) -> Option { + let report_id = crate::util::random_hex(8); + let report_dir = data_dir.join("reports").join(&report_id); + // Own the report dir with a `SnapshotHandle` BEFORE creating it so EVERY exit + // path reclaims its hard links via Drop. The earlier manual `remove_dir_all` + // ran only after a successful assemble, so a `snapshot_into`/`assemble` + // failure (most plausibly disk-full) permanently orphaned a `reports/` dir + // full of hard-linked parts, and `reports/` was never GC'd (F7). let handle = SnapshotHandle { dir: report_dir, window, }; + std::fs::create_dir_all(&handle.dir).ok()?; - store.snapshot_into(&handle.dir, window_start, window_end)?; - let assembled = reporting::assemble( + store + .snapshot_into(&handle.dir, window.start, window.end) + .ok()?; + reporting::assemble( &handle.dir, window, - &meta, + meta, &shared.env, crash_json, &shared.app_token, - window_end, - )?; - - // Persist to the durable queue; the uploader delivers it. `handle`'s Drop - // reclaims the snapshot dir on return (success or the `?` paths above). - queue::enqueue(data_dir, &assembled) + window.end, + ) + .ok() } fn uploader_loop( rx: Receiver, shared: Arc, - data_dir: PathBuf, + data_dir: Option, backoff_base: Duration, fresh_launch: bool, ) { @@ -1075,19 +1287,19 @@ fn uploader_loop( // panicking Transport impl can't kill the uploader. if fresh_launch { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain(&shared, &data_dir, backoff_base, true) + drain(&shared, data_dir.as_deref(), backoff_base, true) })); } loop { match rx.recv_timeout(poll) { Ok(UploadMsg::Wake) => { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain(&shared, &data_dir, backoff_base, false) + drain(&shared, data_dir.as_deref(), backoff_base, false) })); } Ok(UploadMsg::Drain { deadline, ack }) => { let empty = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_until_empty(&shared, &data_dir, backoff_base, deadline) + drain_until_empty(&shared, data_dir.as_deref(), backoff_base, deadline) })) .unwrap_or(false); let _ = ack.send(empty); @@ -1098,7 +1310,7 @@ fn uploader_loop( } Err(RecvTimeoutError::Timeout) => { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain(&shared, &data_dir, backoff_base, false) + drain(&shared, data_dir.as_deref(), backoff_base, false) })); } Err(RecvTimeoutError::Disconnected) => break, @@ -1109,9 +1321,12 @@ fn uploader_loop( /// Attempt delivery of every due queued report once. On the flush path `force` /// is set so backing-off reports are retried immediately. Returns whether any /// report was attempted (used to pace `drain_until_empty`). -fn drain(shared: &Shared, data_dir: &Path, backoff_base: Duration, force: bool) -> bool { +fn drain(shared: &Shared, data_dir: Option<&Path>, backoff_base: Duration, force: bool) -> bool { let now = epoch_ms(); - let mut attempted = false; + let mut attempted = drain_memory(shared, backoff_base, force, true); + let Some(data_dir) = data_dir else { + return attempted; + }; for report in queue::list_pending(data_dir) { // Another process sharing the queue may be delivering this one. let Some(_claim) = queue::try_claim(&report) else { @@ -1242,6 +1457,61 @@ fn notify_dropped(shared: &Shared, reason: DropReason) { } } +/// Attempt delivery of the in-memory backlog. Mirrors [`deliver_queued`]: a +/// transient failure keeps the report (counting toward the retry cap only on the +/// backoff-paced path), anything terminal drops it and tells the host. The queue +/// is taken out for the duration so the lock is never held across the network. +fn drain_memory(shared: &Shared, backoff_base: Duration, force: bool, count_failure: bool) -> bool { + let pending = std::mem::take(&mut *shared.mem_queue.lock()); + if pending.is_empty() { + return false; + } + let now = epoch_ms(); + let mut attempted = false; + let mut keep = Vec::new(); + for mut item in pending { + if !force && now < item.next_attempt_ms { + keep.push(item); + continue; + } + attempted = true; + let mut endpoint_out = None; + match transport::deliver( + &*shared.transport, + &shared.app_token, + &shared.environment_json, + &shared.session, + &item.report, + None, + &mut endpoint_out, + ) { + Ok(()) => {} + Err(TransportError::Transient(_)) | Err(TransportError::SessionExpired) => { + if count_failure { + item.retry += 1; + if item.retry >= UPLOAD_RETRY_CAP { + notify_dropped(shared, DropReason::RetriesExhausted); + continue; + } + item.next_attempt_ms = now + backoff_delay(item.retry, backoff_base); + } + keep.push(item); + } + Err(TransportError::TooManySimilar { .. }) => { + notify_dropped(shared, DropReason::TooManySimilar); + } + Err(e) => notify_dropped(shared, drop_reason_for(e)), + } + } + if !keep.is_empty() { + // Reports held while we were delivering stay behind the retained ones. + let mut queue = shared.mem_queue.lock(); + keep.append(&mut queue); + *queue = keep; + } + attempted +} + /// Exponential backoff: `base * 2^(retry-1)`, capped at 300 s. fn backoff_delay(retry: u32, base: Duration) -> i64 { let shift = retry.saturating_sub(1).min(20); @@ -1260,14 +1530,15 @@ fn backoff_delay(retry: u32, base: Duration) -> i64 { /// whether the queue ended up empty. fn drain_until_empty( shared: &Shared, - data_dir: &Path, + data_dir: Option<&Path>, backoff_base: Duration, deadline: Duration, ) -> bool { let start = Instant::now(); loop { let now = epoch_ms(); - for report in queue::list_pending(data_dir) { + drain_memory(shared, backoff_base, true, false); + for report in data_dir.map(queue::list_pending).unwrap_or_default() { let Some(_claim) = queue::try_claim(&report) else { continue; // a peer is delivering it; the loop below waits it out }; @@ -1276,7 +1547,7 @@ fn drain_until_empty( }; deliver_queued( shared, - data_dir, + data_dir.unwrap_or(Path::new("")), backoff_base, &report, zip, @@ -1289,7 +1560,8 @@ fn drain_until_empty( break; } } - if queue::list_pending(data_dir).is_empty() { + let disk_empty = data_dir.is_none_or(|d| queue::list_pending(d).is_empty()); + if disk_empty && shared.mem_queue.lock().is_empty() { return true; } if start.elapsed() >= deadline { diff --git a/crates/bugsee-core/src/util.rs b/crates/bugsee-core/src/util.rs index 4a00dae..ed27ce0 100644 --- a/crates/bugsee-core/src/util.rs +++ b/crates/bugsee-core/src/util.rs @@ -10,6 +10,17 @@ use std::time::{SystemTime, UNIX_EPOCH}; +/// Print `message` to stderr the first time it is called in this process and do +/// nothing after. For conditions the SDK must announce (it otherwise does no +/// logging) but that would be noise if repeated, such as a degraded data dir. +pub fn warn_once(message: &str) { + use std::sync::atomic::{AtomicBool, Ordering}; + static WARNED: AtomicBool = AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + eprintln!("bugsee: {message}"); + } +} + /// Current wall-clock time in milliseconds since the Unix epoch. pub fn epoch_ms() -> i64 { match SystemTime::now().duration_since(UNIX_EPOCH) { diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs new file mode 100644 index 0000000..94f6b1b --- /dev/null +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -0,0 +1,239 @@ +// +// storage_failures.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! An unusable data directory must degrade, never fail the host. +//! +//! Each test drives a real `Recorder` through one failure mode and asserts the +//! documented outcome: no panic, no hang, and the report still reaches the +//! transport (from memory where the disk refuses it). +//! +//! Failures are injected by making a path component unusable (a regular file +//! where a directory is needed, a read-only directory, a directory removed +//! mid-run). A true ENOSPC needs a size-capped filesystem and root to mount one; +//! writing under a path blocked by a regular file fails the same way at the +//! same call sites (`create_dir_all` / `write` returning an `io::Error`). + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use bugsee_core::model::entry::{CaptureEntry, EventEntry}; +use bugsee_core::reporting::manual_upload_meta; +use bugsee_core::runtime::StorageMode; +use bugsee_core::util::epoch_ms; +use bugsee_core::{MockTransport, Recorder, RecorderConfig}; +use serde_json::Map; + +struct TempDir { + path: PathBuf, +} +impl TempDir { + fn new() -> Self { + let mut path = std::env::temp_dir(); + path.push(format!( + "bugsee-storage-{}", + bugsee_core::util::random_hex(8) + )); + std::fs::create_dir_all(&path).unwrap(); + TempDir { path } + } +} +impl Drop for TempDir { + fn drop(&mut self) { + restore_permissions(&self.path); + let _ = std::fs::remove_dir_all(&self.path); + } +} + +#[cfg(unix)] +fn restore_permissions(path: &Path) { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)); +} +#[cfg(not(unix))] +fn restore_permissions(_path: &Path) {} + +fn event(name: &str) -> CaptureEntry { + CaptureEntry::UserEvent(EventEntry { + timestamp: epoch_ms(), + name: Some(name.into()), + params: Map::new(), + custom: Map::new(), + }) +} + +/// Report once and wait for delivery; returns how many bundles arrived. +fn report_and_flush(recorder: &Recorder, transport: &MockTransport) -> usize { + recorder.capture(event("before")); + recorder.report(manual_upload_meta(), None); + assert!( + recorder.flush(Duration::from_secs(10)), + "flush must complete, not hang" + ); + let n = transport.uploaded_bundles.lock().unwrap().len(); + n +} + +/// A path that can never be created: a child of a regular file. +fn blocked_path(dir: &Path) -> PathBuf { + let file = dir.join("not-a-dir"); + std::fs::write(&file, b"x").unwrap(); + file.join("data") +} + +#[test] +fn missing_uncreatable_directory_runs_in_memory_and_still_reports() { + let dir = TempDir::new(); + let transport = Arc::new(MockTransport::default()); + + let recorder = Recorder::launch( + RecorderConfig::new(blocked_path(&dir.path), "T"), + transport.clone(), + ) + .expect("an unusable directory must not fail launch"); + + assert_eq!(recorder.storage_mode(), StorageMode::Memory); + assert!(recorder.storage_reason().is_some()); + assert_eq!(report_and_flush(&recorder, &transport), 1); +} + +#[test] +fn memory_only_config_skips_the_disk_entirely() { + let dir = TempDir::new(); + let transport = Arc::new(MockTransport::default()); + let data_dir = dir.path.join("must-stay-absent"); + let mut config = RecorderConfig::new(&data_dir, "T"); + config.memory_only = true; + + let recorder = Recorder::launch(config, transport.clone()).expect("launch"); + + assert_eq!(recorder.storage_mode(), StorageMode::Memory); + assert_eq!(report_and_flush(&recorder, &transport), 1); + drop(recorder); + assert!(!data_dir.exists(), "memory mode must not touch the disk"); +} + +#[cfg(unix)] +#[test] +fn read_only_directory_runs_in_memory_and_still_reports() { + use std::os::unix::fs::PermissionsExt; + let dir = TempDir::new(); + let ro = dir.path.join("ro"); + std::fs::create_dir(&ro).unwrap(); + std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o555)).unwrap(); + // root ignores permission bits, which would make this test vacuous. + if std::fs::write(ro.join("probe"), b"x").is_ok() { + eprintln!("skipped: permission bits are not enforced for this user"); + return; + } + let transport = Arc::new(MockTransport::default()); + + let recorder = Recorder::launch(RecorderConfig::new(&ro, "T"), transport.clone()) + .expect("a read-only directory must not fail launch"); + + assert_eq!(recorder.storage_mode(), StorageMode::Memory); + assert_eq!(report_and_flush(&recorder, &transport), 1); +} + +#[test] +fn directory_removed_mid_run_still_reports() { + let dir = TempDir::new(); + let data_dir = dir.path.join("data"); + let transport = Arc::new(MockTransport::default()); + let recorder = + Recorder::launch(RecorderConfig::new(&data_dir, "T"), transport.clone()).expect("launch"); + assert_eq!(recorder.storage_mode(), StorageMode::Disk); + recorder.capture(event("early")); + assert!(recorder.flush(Duration::from_secs(10))); + + // The volume vanishes under a running process. + std::fs::remove_dir_all(&data_dir).unwrap(); + + assert_eq!(report_and_flush(&recorder, &transport), 1); + // Capture and shutdown after the loss must not panic either. + recorder.capture(event("late")); + drop(recorder); +} + +#[test] +fn directory_replaced_by_a_file_mid_run_falls_back_to_memory_delivery() { + let dir = TempDir::new(); + let data_dir = dir.path.join("data"); + let transport = Arc::new(MockTransport::default()); + let recorder = + Recorder::launch(RecorderConfig::new(&data_dir, "T"), transport.clone()).expect("launch"); + + // Nothing under the data dir can be created any more: every disk write site + // (parts, report snapshot, queue) fails. + std::fs::remove_dir_all(&data_dir).unwrap(); + std::fs::write(&data_dir, b"a file now").unwrap(); + + assert_eq!(report_and_flush(&recorder, &transport), 1); + drop(recorder); +} + +#[test] +fn queue_write_failure_holds_the_report_in_memory() { + let dir = TempDir::new(); + let data_dir = dir.path.join("data"); + std::fs::create_dir_all(&data_dir).unwrap(); + // The queue directory cannot be created: the enqueue step fails while the + // capture parts and the liveness marker work. This is what a full disk or a + // quota looks like at the queue. + std::fs::write(data_dir.join("queue"), b"blocker").unwrap(); + let transport = Arc::new(MockTransport::default()); + + let recorder = + Recorder::launch(RecorderConfig::new(&data_dir, "T"), transport.clone()).expect("launch"); + assert_eq!(recorder.storage_mode(), StorageMode::Disk); + + assert_eq!(report_and_flush(&recorder, &transport), 1); +} + +#[test] +fn memory_backlog_is_bounded_and_overflow_is_reported() { + use bugsee_core::runtime::DropReason; + use bugsee_core::transport::{Transport, TransportError}; + use std::sync::Mutex; + + // Never delivers, so everything stays in the memory backlog. + struct Offline; + impl Transport for Offline { + fn register_session(&self, _: &str, _: &[u8]) -> Result { + Err(TransportError::Transient("offline".into())) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + Err(TransportError::Transient("offline".into())) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), TransportError> { + Err(TransportError::Transient("offline".into())) + } + } + + let dir = TempDir::new(); + let dropped = Arc::new(Mutex::new(Vec::new())); + let sink = dropped.clone(); + let mut config = RecorderConfig::new(blocked_path(&dir.path), "T"); + config.on_report_dropped = Some(Box::new(move |r| sink.lock().unwrap().push(r))); + let recorder = Recorder::launch(config, Arc::new(Offline)).expect("launch"); + + for _ in 0..40 { + recorder.report(manual_upload_meta(), None); + } + // Flush cannot succeed (offline) but must return promptly, not hang. + assert!(!recorder.flush(Duration::from_millis(500))); + + let seen = dropped.lock().unwrap().clone(); + assert!( + !seen.is_empty() && seen.iter().all(|r| *r == DropReason::StorageUnavailable), + "overflow must be surfaced, got {seen:?}" + ); +} diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 138dace..02b552d 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -23,8 +23,9 @@ use bugsee_core::util::epoch_ms; use bugsee_core::{Recorder, RecorderConfig}; use serde_json::{Map, Value}; -use crate::options::LaunchOptions; +use crate::options::{DataDirResolution, DataDirSource, LaunchOptions, StorageStatus}; use crate::report::Report; +use bugsee_core::runtime::StorageMode; // A `Mutex` (not `RwLock`) because `mpsc::Sender` inside `Recorder` is `Send` // but not `Sync`; the lock is held only long enough to enqueue. @@ -44,6 +45,14 @@ struct ReviveInfo { } static REVIVE_INFO: ForkMutex> = ForkMutex::new(|| None); +/// How the current launch resolved its data directory, for `storage_status`. +struct Resolved { + dir: Option, + source: DataDirSource, + skipped: Vec, +} +static RESOLUTION: ForkMutex> = ForkMutex::new(|| None); + /// The fork epoch this process last reconciled with. static SEEN_FORK_EPOCH: AtomicU64 = AtomicU64::new(0); @@ -71,8 +80,22 @@ impl Bugsee { pub fn launch_with(options: LaunchOptions) -> io::Result { bugsee_core::fork::install_fork_hook(); check_fork(); - let data_dir = options.resolved_data_dir(); - std::fs::create_dir_all(&data_dir)?; + // No writable directory is not an error: the recorder runs in memory so + // handled errors still go out (see `DataDirSource`). + let DataDirResolution { + dir, + source, + skipped, + } = options.resolve_data_dir(); + let memory_only = dir.is_none(); + let data_dir = dir.unwrap_or_else(|| std::env::temp_dir().join("bugsee")); + if !memory_only && !skipped.is_empty() { + bugsee_core::util::warn_once(&format!( + "data directory unusable ({}); using {}", + skipped.join("; "), + data_dir.display() + )); + } let transport = match options.transport.clone() { Some(t) => t, @@ -94,6 +117,7 @@ impl Bugsee { }; let mut config = RecorderConfig::new(data_dir, options.app_token.clone()); + config.memory_only = memory_only; config.caps = options.caps(); config.rotate_interval = options.rotate_interval; config.before_send = options.before_send; @@ -121,6 +145,18 @@ impl Bugsee { } let recorder = Recorder::launch(config, transport)?; + // Nothing can be recovered without a disk, so crash markers and panic + // snapshots have nowhere to go. + let persistent = recorder.storage_mode() == StorageMode::Disk; + *RESOLUTION.lock() = Some(Resolved { + dir: persistent.then(|| recorder.data_dir().to_path_buf()), + source: if persistent { + source + } else { + DataDirSource::None + }, + skipped, + }); #[cfg(feature = "panic")] let panic_info_path = recorder.panic_info_path(); @@ -128,7 +164,7 @@ impl Bugsee { // Install the native crash handler pointing at this generation's marker. // Replace under a brief lock, then drop the OLD handler outside it. #[cfg(feature = "native")] - if options.native_crash_capture { + if options.native_crash_capture && persistent { if let Ok(handler) = bugsee_native::install(recorder.crash_info_path()) { let old = NATIVE.lock().replace(handler); drop(old); @@ -161,7 +197,11 @@ impl Bugsee { // SIGABRT on the next launch β€” and so an uncaught UNWINDING panic is // recovered as a fatal panic (the marker is kept because // `Recorder::drop` sees `thread::panicking()`). - bugsee_panic::set_snapshot_path(panic_info_path); + if persistent { + bugsee_panic::set_snapshot_path(panic_info_path); + } else { + bugsee_panic::clear_snapshot_path(); + } // Opt-in: report immediately from the hook instead of via recovery. bugsee_panic::set_report_from_hook(options.report_panics_from_hook); } @@ -169,6 +209,33 @@ impl Bugsee { Ok(LaunchGuard { _private: () }) } + /// Where the SDK is keeping its state, or `None` before `launch`. + /// + /// Use it to confirm a deployment is set up for crash recovery: it needs + /// [`StorageStatus::crash_recovery_available`], and on a container a data + /// directory on a volume rather than ephemeral storage + /// ([`StorageStatus::may_be_ephemeral`]). + pub fn storage_status() -> Option { + check_fork(); + let (mode, reason) = { + let guard = RECORDER.lock(); + let recorder = guard.as_ref()?; + ( + recorder.storage_mode(), + recorder.storage_reason().map(String::from), + ) + }; + let resolved = RESOLUTION.lock(); + let resolved = resolved.as_ref()?; + Some(StorageStatus { + mode, + data_dir: resolved.dir.clone(), + source: resolved.source, + skipped: resolved.skipped.clone(), + reason, + }) + } + /// Stop the SDK and flush pending work (dropping the recorder joins the worker). pub fn stop() { check_fork(); @@ -496,7 +563,9 @@ fn revive_after_fork(epoch: u64) { } } #[cfg(feature = "panic")] - bugsee_panic::set_snapshot_path(recorder.panic_info_path()); + if recorder.storage_mode() == StorageMode::Disk { + bugsee_panic::set_snapshot_path(recorder.panic_info_path()); + } *RECORDER.lock() = Some(recorder); *failures = 0; diff --git a/crates/bugsee/src/lib.rs b/crates/bugsee/src/lib.rs index 867dde3..ce610ef 100644 --- a/crates/bugsee/src/lib.rs +++ b/crates/bugsee/src/lib.rs @@ -67,7 +67,7 @@ macro_rules! app_package_id { } pub use api::{Bugsee, LaunchGuard, ResultExt}; -pub use options::LaunchOptions; +pub use options::{DataDirSource, LaunchOptions, StorageStatus, DATA_DIR_ENV}; pub use perf::{Span, Transaction}; pub use report::Report; @@ -84,5 +84,5 @@ pub use bugsee_core::model::enums::{ }; pub use bugsee_core::model::perf::Status; pub use bugsee_core::reporting::{Attachment, ReportMeta}; -pub use bugsee_core::runtime::DropReason; +pub use bugsee_core::runtime::{DropReason, StorageMode}; pub use bugsee_core::transport::{Transport, TransportError}; diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index 9d64222..acd48e3 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -8,7 +8,8 @@ //! Typed launch configuration. Mirrors the mobile SDK option surface, adapted //! to a builder rather than a string-keyed registry. -use std::path::PathBuf; +use std::ffi::OsString; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -121,7 +122,12 @@ impl LaunchOptions { self } - /// Override the on-disk data directory (defaults to a per-app temp dir). + /// Set the on-disk data directory, taking precedence over everything else. + /// + /// The directory holds the capture window, the liveness marker and the retry + /// queue, and crash recovery works only if it OUTLIVES the process: on a + /// container, point it at a volume. See [`DataDirSource`] for the full + /// resolution order and what happens when it is not writable. pub fn data_dir(mut self, dir: impl Into) -> Self { self.data_dir = Some(dir.into()); self @@ -225,14 +231,162 @@ impl LaunchOptions { } } - pub(crate) fn resolved_data_dir(&self) -> PathBuf { - self.data_dir.clone().unwrap_or_else(|| { - let mut dir = std::env::temp_dir(); - dir.push("bugsee"); - // Isolate by app token so multiple apps don't collide. - dir.push(short_token(&self.app_token)); - dir - }) + /// Pick the data directory: the first candidate in [`DataDirSource`] order + /// that can actually be created and written to. + pub(crate) fn resolve_data_dir(&self) -> DataDirResolution { + resolve_data_dir( + self.data_dir.as_deref(), + &self.app_token, + &|name| std::env::var_os(name), + &std::env::temp_dir(), + ) + } +} + +/// Environment variable naming the data directory when +/// [`LaunchOptions::data_dir`] is not set. +pub const DATA_DIR_ENV: &str = "BUGSEE_DATA_DIR"; + +/// Which step of the resolution order produced the data directory. +/// +/// Candidates are tried in this order and the first one that is writable wins; +/// an unusable one is skipped (and recorded in +/// [`StorageStatus::skipped`](crate::StorageStatus::skipped)), never fatal: +/// +/// 1. [`Explicit`](DataDirSource::Explicit): [`LaunchOptions::data_dir`]. +/// 2. [`Environment`](DataDirSource::Environment): `BUGSEE_DATA_DIR`. +/// 3. [`PlatformDefault`](DataDirSource::PlatformDefault): `bugsee/` under +/// `$XDG_DATA_HOME` (else `~/.local/share`) on Linux, `~/Library/Application +/// Support` on macOS, `%LOCALAPPDATA%` on Windows. Skipped when the base is +/// unset, as in a container with no home directory. +/// 4. [`TempFallback`](DataDirSource::TempFallback): `bugsee/` under the +/// system temp directory. Often wiped on restart, so crash recovery may not +/// survive it; [`StorageStatus::may_be_ephemeral`](crate::StorageStatus::may_be_ephemeral) +/// reports it. +/// +/// If none is writable the SDK runs +/// [in memory](bugsee_core::runtime::StorageMode::Memory): handled errors and +/// manual reports are still sent, but there is no capture window and no crash +/// recovery. It logs one line to stderr when it degrades. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DataDirSource { + Explicit, + Environment, + PlatformDefault, + TempFallback, + /// No directory is in use (memory mode). + None, +} + +/// The outcome of [`resolve_data_dir`]. +pub(crate) struct DataDirResolution { + /// The chosen directory, or `None` if no candidate was writable. + pub dir: Option, + pub source: DataDirSource, + /// Candidates tried and rejected before the winner, with the reason. + pub skipped: Vec, +} + +/// Pure resolution core: `env` and `temp` are injected so tests need not touch +/// the process environment. +pub(crate) fn resolve_data_dir( + explicit: Option<&Path>, + app_token: &str, + env: &dyn Fn(&str) -> Option, + temp: &Path, +) -> DataDirResolution { + let mut candidates: Vec<(DataDirSource, PathBuf)> = Vec::new(); + if let Some(dir) = explicit { + candidates.push((DataDirSource::Explicit, dir.to_path_buf())); + } + if let Some(dir) = env(DATA_DIR_ENV).filter(|v| !v.is_empty()) { + candidates.push((DataDirSource::Environment, PathBuf::from(dir))); + } + // Isolate by app token so multiple apps don't collide. + let app = |base: PathBuf| base.join("bugsee").join(short_token(app_token)); + if let Some(base) = platform_data_base(env) { + candidates.push((DataDirSource::PlatformDefault, app(base))); + } + candidates.push((DataDirSource::TempFallback, app(temp.to_path_buf()))); + + let mut skipped = Vec::new(); + for (source, dir) in candidates { + match probe_writable(&dir) { + Ok(()) => { + return DataDirResolution { + dir: Some(dir), + source, + skipped, + } + } + Err(e) => skipped.push(format!("{}: {e}", dir.display())), + } + } + DataDirResolution { + dir: None, + source: DataDirSource::None, + skipped, + } +} + +/// The per-user data base directory for this platform, if its inputs are set. +fn platform_data_base(env: &dyn Fn(&str) -> Option) -> Option { + let set = |name: &str| env(name).filter(|v| !v.is_empty()).map(PathBuf::from); + if cfg!(windows) { + set("LOCALAPPDATA") + } else if cfg!(target_vendor = "apple") { + set("HOME").map(|h| h.join("Library").join("Application Support")) + } else { + set("XDG_DATA_HOME").or_else(|| set("HOME").map(|h| h.join(".local").join("share"))) + } +} + +/// Create `dir` and prove a file can be written in it. Checking writability (not +/// just existence) is what catches a read-only root file system or a volume +/// mounted read-only. +fn probe_writable(dir: &Path) -> std::io::Result<()> { + use std::io::Write; + std::fs::create_dir_all(dir)?; + let probe = dir.join(format!(".bugsee-probe-{}", std::process::id())); + let result = std::fs::File::create(&probe).and_then(|mut f| f.write_all(b"1")); + let _ = std::fs::remove_file(&probe); + result +} + +/// Where the SDK keeps its state right now; see +/// [`Bugsee::storage_status`](crate::Bugsee::storage_status). +/// +/// On ephemeral storage, crash recovery only works if the directory outlives +/// the process (a volume mount). The SDK cannot tell a volume from a scratch +/// disk, so check [`may_be_ephemeral`](StorageStatus::may_be_ephemeral) and +/// your own deployment. +#[derive(Debug, Clone)] +pub struct StorageStatus { + /// `Disk` when the data directory is usable; `Memory` when it is not. + pub mode: bugsee_core::runtime::StorageMode, + /// The directory in use; `None` in memory mode. + pub data_dir: Option, + /// Which resolution step produced `data_dir`. + pub source: DataDirSource, + /// Candidates rejected before the one in use, with the error for each. + pub skipped: Vec, + /// Why the SDK is in memory mode, if it is. + pub reason: Option, +} + +impl StorageStatus { + /// Whether a fatal crash can be recovered on the next launch: true only with + /// a usable data directory. (It also needs that directory to survive the + /// process; see [`may_be_ephemeral`](Self::may_be_ephemeral).) + pub fn crash_recovery_available(&self) -> bool { + self.mode == bugsee_core::runtime::StorageMode::Disk + } + + /// Whether the directory is the temp-dir fallback, which is commonly wiped + /// on restart and so may not preserve a crash for the next launch. Set + /// [`LaunchOptions::data_dir`] (or `BUGSEE_DATA_DIR`) to a volume to fix it. + pub fn may_be_ephemeral(&self) -> bool { + self.source == DataDirSource::TempFallback } } @@ -243,3 +397,107 @@ fn short_token(token: &str) -> String { .take(16) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch() -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "bugsee-resolve-{}", + bugsee_core::util::random_hex(8) + )); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + /// A path no directory can be created at: a child of a regular file. + fn blocked(scratch: &Path, name: &str) -> PathBuf { + let file = scratch.join(name); + std::fs::write(&file, b"x").unwrap(); + file.join("sub") + } + + fn env_of<'a>(vars: &'a [(&'a str, OsString)]) -> impl Fn(&str) -> Option + 'a { + move |name| { + vars.iter() + .find(|(k, _)| *k == name) + .map(|(_, v)| v.clone()) + } + } + + #[test] + fn explicit_directory_wins_when_writable() { + let s = scratch(); + let explicit = s.join("explicit"); + let env = [(DATA_DIR_ENV, s.join("from-env").into_os_string())]; + let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&env), &s.join("tmp")); + assert_eq!(r.source, DataDirSource::Explicit); + assert_eq!(r.dir.as_deref(), Some(explicit.as_path())); + assert!(r.skipped.is_empty()); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn unwritable_explicit_falls_through_to_the_environment_variable() { + let s = scratch(); + let bad = blocked(&s, "f1"); + let good = s.join("from-env"); + let env = [(DATA_DIR_ENV, good.clone().into_os_string())]; + let r = resolve_data_dir(Some(&bad), "tok", &env_of(&env), &s.join("tmp")); + assert_eq!(r.source, DataDirSource::Environment); + assert_eq!(r.dir, Some(good)); + assert_eq!(r.skipped.len(), 1, "the rejected candidate is recorded"); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn platform_default_is_used_when_home_is_set() { + let s = scratch(); + let home = s.join("home"); + let env = [ + ("HOME", home.clone().into_os_string()), + ("XDG_DATA_HOME", home.clone().into_os_string()), + ("LOCALAPPDATA", home.clone().into_os_string()), + ]; + let r = resolve_data_dir(None, "tok-1", &env_of(&env), &s.join("tmp")); + assert_eq!(r.source, DataDirSource::PlatformDefault); + assert!(r.dir.unwrap().starts_with(&home)); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn no_home_falls_back_to_the_temp_directory() { + let s = scratch(); + let tmp = s.join("tmp"); + let r = resolve_data_dir(None, "tok-1", &env_of(&[]), &tmp); + assert_eq!(r.source, DataDirSource::TempFallback); + assert_eq!(r.dir, Some(tmp.join("bugsee").join("tok1"))); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn nothing_writable_resolves_to_none_and_lists_every_attempt() { + let s = scratch(); + let env = [(DATA_DIR_ENV, blocked(&s, "f2").into_os_string())]; + let r = resolve_data_dir( + Some(&blocked(&s, "f3")), + "tok", + &env_of(&env), + &blocked(&s, "f4"), + ); + assert_eq!(r.source, DataDirSource::None); + assert!(r.dir.is_none()); + assert_eq!(r.skipped.len(), 3); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn an_empty_environment_variable_is_ignored() { + let s = scratch(); + let env = [(DATA_DIR_ENV, OsString::new())]; + let r = resolve_data_dir(None, "tok", &env_of(&env), &s.join("tmp")); + assert_eq!(r.source, DataDirSource::TempFallback); + let _ = std::fs::remove_dir_all(&s); + } +} diff --git a/crates/bugsee/tests/storage_fallback.rs b/crates/bugsee/tests/storage_fallback.rs new file mode 100644 index 0000000..ba82295 --- /dev/null +++ b/crates/bugsee/tests/storage_fallback.rs @@ -0,0 +1,107 @@ +// +// storage_fallback.rs +// bugsee +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! The facade on read-only / ephemeral hosts: `launch` must succeed and handled +//! errors must still go out when no data directory is usable. +//! +//! One test function on purpose: the SDK is a process-wide singleton and the +//! scenarios steer it through process environment variables. + +#![cfg(unix)] + +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use bugsee::core::MockTransport; +use bugsee::{Bugsee, DataDirSource, LaunchOptions, LogLevel, StorageMode, DATA_DIR_ENV}; + +fn scratch() -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "bugsee-fallback-{}", + bugsee::core::util::random_hex(8) + )); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +#[test] +fn launch_degrades_through_the_resolution_order() { + let s = scratch(); + // A path nothing can be created at. + let file = s.join("blocker"); + std::fs::write(&file, b"x").unwrap(); + let blocked = file.join("data"); + + // Keep every default away from the real home directory and temp dir. + std::env::remove_var("HOME"); + std::env::remove_var("XDG_DATA_HOME"); + std::env::set_var("TMPDIR", &file); + std::env::remove_var(DATA_DIR_ENV); + + assert!(Bugsee::storage_status().is_none(), "nothing before launch"); + + // 1. A writable explicit directory is used as given. + let good = s.join("explicit"); + let transport = Arc::new(MockTransport::default()); + let guard = Bugsee::launch_with( + LaunchOptions::new("TOKEN_A") + .data_dir(&good) + .with_transport(transport.clone()), + ) + .expect("launch"); + let status = Bugsee::storage_status().expect("status"); + assert_eq!(status.mode, StorageMode::Disk); + assert_eq!(status.source, DataDirSource::Explicit); + assert_eq!(status.data_dir.as_deref(), Some(good.as_path())); + assert!(status.crash_recovery_available()); + assert!(!status.may_be_ephemeral()); + drop(guard); + + // 2. An unusable explicit directory falls through to the environment variable. + let from_env = s.join("from-env"); + std::env::set_var(DATA_DIR_ENV, &from_env); + let guard = Bugsee::launch_with( + LaunchOptions::new("TOKEN_A") + .data_dir(&blocked) + .with_transport(transport.clone()), + ) + .expect("launch"); + let status = Bugsee::storage_status().expect("status"); + assert_eq!(status.source, DataDirSource::Environment); + assert_eq!(status.data_dir.as_deref(), Some(from_env.as_path())); + assert_eq!(status.skipped.len(), 1); + drop(guard); + std::env::remove_var(DATA_DIR_ENV); + + // 3. Nothing usable anywhere: memory mode, and reports still go out. + let transport = Arc::new(MockTransport::default()); + let guard = Bugsee::launch_with( + LaunchOptions::new("TOKEN_A") + .data_dir(&blocked) + .with_transport(transport.clone()), + ) + .expect("an unwritable environment must not fail launch"); + let status = Bugsee::storage_status().expect("status"); + assert_eq!(status.mode, StorageMode::Memory); + assert_eq!(status.source, DataDirSource::None); + assert!(status.data_dir.is_none()); + assert!(!status.crash_recovery_available()); + assert!(!status.skipped.is_empty()); + + Bugsee::event("ignored_without_a_window"); + Bugsee::capture_message(LogLevel::Error, "handled error with no disk"); + Bugsee::create_report().upload(); + assert!(Bugsee::flush(Duration::from_secs(10))); + assert_eq!( + transport.uploaded_bundles.lock().unwrap().len(), + 2, + "both reports must be delivered from memory" + ); + drop(guard); + + let _ = std::fs::remove_dir_all(&s); +} From 184d102976d76c186b1b636d1359ca3d31589bfe Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 14:25:12 +0500 Subject: [PATCH 02/15] fix: address review of data-directory fallback (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - memory backlog: entries stay queued while in flight so the cap is real; cache the presigned endpoint (no duplicate issue on a retried PUT); honour the flush deadline; clear the backlog in fork children - a requested data_dir / BUGSEE_DATA_DIR is final: unusable means memory mode (explicit_rejected), never another disk location - temp fallback is bugsee-/, 0700, symlink/owner checked; probe uses create_new with a random name - recover and deliver leftovers from the previous default location once - mid-run degradation is visible (StorageStatus::degraded); fork child falls back to memory instead of failing; warn_once per message, never panics - #[non_exhaustive] on the new public types; ffi test no longer writes to the real data directory πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 18 +- crates/bugsee-core/src/reporting.rs | 1 + crates/bugsee-core/src/runtime.rs | 294 +++++++++++---- crates/bugsee-core/src/util.rs | 17 +- crates/bugsee-core/tests/storage_failures.rs | 224 ++++++++++-- crates/bugsee-ffi/tests/lifecycle.rs | 7 +- crates/bugsee/Cargo.toml | 3 + crates/bugsee/src/api.rs | 65 ++-- crates/bugsee/src/options.rs | 365 ++++++++++++++++--- crates/bugsee/tests/fork_safety.rs | 78 ++++ crates/bugsee/tests/storage_fallback.rs | 27 +- 11 files changed, 894 insertions(+), 205 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 3029573..f2f03f6 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -181,17 +181,25 @@ Global base scope behind `RwLock`; `with_scope` pushes a thread-local overlay. * **Durability envelope (be honest):** buffered append survives **process death** (panic, `abort`, signal, `kill -9`) because completed `write()`s live in the kernel page cache independent of the crashing process β€” this is what the `recovery_e2e` / `kill -9` tests exercise. It does **not** guarantee survival of **power loss / kernel panic**, since nothing is `fsync`'d; a hardening tier (periodic `fsync`, or the originally-envisioned `mmap(MAP_SHARED)` + `msync`) is **future work**, not current behavior. -**Data directory resolution and failure modes (read-only / ephemeral hosts).** The facade picks the first **writable** candidate (create + write-probe, not mere existence), skipping and recording any that fail: (1) `LaunchOptions::data_dir`; (2) `BUGSEE_DATA_DIR`; (3) platform default `bugsee/` under `$XDG_DATA_HOME`/`~/.local/share` (Linux), `~/Library/Application Support` (macOS), `%LOCALAPPDATA%` (Windows), skipped if the base is unset (container with no home); (4) `/bugsee/`. Launch never fails for storage reasons. Outcomes: +**Data directory resolution and failure modes (read-only / ephemeral hosts).** The facade picks the data directory in this order: (1) `LaunchOptions::data_dir`; (2) `BUGSEE_DATA_DIR`; (3) platform default `bugsee/` under `$XDG_DATA_HOME`/`~/.local/share` (Linux), `~/Library/Application Support` (macOS), `%LOCALAPPDATA%` (Windows), skipped if the base is unset (container with no home); (4) `/bugsee-/` (`bugsee/` on Windows). A candidate must pass a create + `create_new` write-probe (not mere existence). **A directory the host asked for (1 or 2) is final**: if unusable the SDK does not use another disk location, it runs in memory and reports why in `StorageStatus::explicit_rejected`; only the defaults (3, 4) fall through to each other. The temp fallback is created `0700` per component and refused if a component is a symlink or owned by another user (the unix temp dir is world-writable). Launch never fails for storage reasons. Outcomes: | Failure | Outcome | |---|---| -| Preferred candidate unusable, a later one works | falls through; one stderr line; `StorageStatus::skipped` lists why | -| No candidate usable, or `Session::begin`/`PartStore::new` fails | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from a bounded in-memory backlog (16; overflow drops the newest with `DropReason::StorageUnavailable`); an unsent report dies with the process | -| Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and silently dropped; a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out | +| Default candidate unusable, a later default works | falls through; one stderr line; `StorageStatus::skipped` lists why | +| Requested directory unusable, or no default usable, or `Session::begin`/`PartStore::new` fails (also in a fork child) | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from an in-memory backlog capped at 16 (an entry counts until its delivery settles; overflow drops the newest with `DropReason::StorageUnavailable`); an unsent report dies with the process; a fork child starts with an empty backlog (the parent's is the parent's to send) | +| Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and dropped; a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out; `StorageStatus::degraded` turns true and `crash_recovery_available()` false | | Disk full / permission denied at enqueue, snapshot, marker | same as above; the `reports/` snapshot dir is reclaimed by `SnapshotHandle::drop` | | Crash handler cannot write its marker | the raw `open`/`write` return values are ignored; the process still dies with the original signal (watchdog bounds the handler) | -Crash recovery needs the data directory to **outlive the process**: on a container, mount a volume and point `data_dir`/`BUGSEE_DATA_DIR` at it. The temp fallback is commonly wiped on restart. `Bugsee::storage_status()` reports `mode`, `data_dir`, `source`, `skipped`, `crash_recovery_available()` and `may_be_ephemeral()` (the SDK cannot tell a volume from a scratch disk; the temp fallback is the only case it flags). Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised; failures are injected by blocking path components and removing the directory mid-run. +The in-memory backlog retries like the disk queue: it caches the presigned endpoint (a failed PUT resumes at the PUT, no duplicate issue), backs off on the scheduled path, and a flush honours its deadline. + +Crash recovery needs the data directory to **outlive the process**: on a container, mount a volume and point `data_dir`/`BUGSEE_DATA_DIR` at it. The temp fallback is commonly wiped on restart. `Bugsee::storage_status()` reports `mode`, `data_dir`, `source`, `skipped`, `explicit_rejected`, `degraded`, `crash_recovery_available()` and `may_be_ephemeral()` (the SDK cannot tell a volume from a scratch disk; the temp fallback is the only case it flags). No FFI accessor yet (follow-up). + +**Release note, default location moved.** Earlier builds defaulted to `/bugsee/`; the default is now the platform location (3) or the private temp fallback (4). When no directory was requested and the old location exists, is a real directory and is owned by the current user, the launch recovers its crashed sessions and delivers its queue once, best effort (delivery is asynchronous and `flush` does not wait for it). It is not claimed to be lossless: a legacy session whose process is still alive is skipped, and a legacy directory that fails the ownership check is left alone. Hosts that set `data_dir` were never using it. + +**Unsafe.** The facade makes two `geteuid()` calls (unix) to name and vet the private temp directory, alongside core's single `kill(pid, 0)`. + +Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, a memory-mode case in `bugsee/tests/fork_safety.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised, since mounting one needs root; failures are injected by blocking path components and removing the directory mid-run. **Generations:** each launch bumps `gen`; next-launch recovery reads the **previous** gen's parts before advancing. diff --git a/crates/bugsee-core/src/reporting.rs b/crates/bugsee-core/src/reporting.rs index 3116578..a91844c 100644 --- a/crates/bugsee-core/src/reporting.rs +++ b/crates/bugsee-core/src/reporting.rs @@ -54,6 +54,7 @@ pub struct ExtraFile { } /// A finished, ready-to-deliver report. +#[derive(Clone)] pub struct AssembledReport { /// `.bundle.zip` file name. pub bundle_name: String, diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index faf62cc..21b16a5 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -26,7 +26,7 @@ //! limit. use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender}; use std::sync::Arc; use std::thread::JoinHandle; @@ -62,6 +62,7 @@ const DEFAULT_MAX_QUEUED: usize = 8192; const DEFAULT_MAX_REPORT_QUEUED: usize = 256; /// Where a recorder keeps its state. +#[non_exhaustive] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum StorageMode { /// The data directory is usable: capture parts, the liveness marker and the @@ -80,9 +81,14 @@ const MAX_MEMORY_REPORTS: usize = 16; /// A report waiting in memory for delivery, because it could not be queued on disk. struct MemReport { + /// Identifies the entry across the unlocked delivery attempt. + id: u64, report: AssembledReport, retry: u32, next_attempt_ms: i64, + /// Presigned upload endpoint cached after a failed PUT, so a retry resumes at + /// the PUT instead of minting a duplicate issue (mirrors the disk queue). + endpoint: Option, } /// A callback run on every report before delivery. Mutate the metadata in place; @@ -164,9 +170,14 @@ pub struct RecorderConfig { pub sample_rate: f64, /// Max in-flight capture entries before new ones are dropped (back-pressure). pub max_queued_entries: usize, - /// Skip the disk entirely and run in [`StorageMode::Memory`]. For a host - /// layer that already established no data directory is usable. - pub memory_only: bool, + /// `Some(reason)` skips the disk entirely and runs in [`StorageMode::Memory`]. + /// For a host layer that already established no data directory is usable; + /// the reason is logged and reported by [`Recorder::storage_reason`]. + pub memory_only: Option, + /// A previous default data directory to drain once at launch: recover its + /// crashed sessions and deliver its queue. For a host layer that moved its + /// default location and must not strand reports left at the old one. + pub legacy_data_dir: Option, } impl RecorderConfig { @@ -187,7 +198,8 @@ impl RecorderConfig { before_breadcrumb: None, sample_rate: 1.0, max_queued_entries: DEFAULT_MAX_QUEUED, - memory_only: false, + memory_only: None, + legacy_data_dir: None, } } } @@ -231,8 +243,13 @@ struct Shared { max_report_queued: usize, /// Whether telemetry sampling is paused (mirrors the facade's pause state). paused: AtomicBool, - /// Reports that could not be queued on disk, delivered from memory. + /// Reports that could not be queued on disk, delivered from memory. An entry + /// stays here until its delivery is settled, so the length is the real backlog. mem_queue: ForkLock>, + next_mem_id: AtomicU64, + /// Set the first time a disk write fails while running in disk mode: the + /// recorder is no longer fully durable (see [`Recorder::is_degraded`]). + degraded: AtomicBool, } /// The recorder's own locks join the `fork()` lock protocol, so a forked child @@ -334,6 +351,8 @@ impl Recorder { max_report_queued: DEFAULT_MAX_REPORT_QUEUED, paused: AtomicBool::new(false), mem_queue: ForkLock::new(Vec::new()), + next_mem_id: AtomicU64::new(0), + degraded: AtomicBool::new(false), }); crate::fork::install_fork_hook(); @@ -343,8 +362,8 @@ impl Recorder { // An unusable data directory degrades to memory mode instead of failing // the launch: the host must not lose error reporting (or its startup) // because a volume is read-only or full. - let (disk, storage_reason) = if config.memory_only { - (None, Some("no usable data directory".to_string())) + let (disk, storage_reason) = if let Some(reason) = config.memory_only.clone() { + (None, Some(reason)) } else { match open_disk(&data_dir, config.caps) { Ok(disk) => (Some(disk), None), @@ -352,11 +371,15 @@ impl Recorder { } }; if let Some(reason) = &storage_reason { - crate::util::warn_once(&format!( - "data directory unusable ({reason}); running in memory only: handled errors \ - and manual reports are still sent, but crashes cannot be recovered and \ - nothing survives this process" - )); + static WARNED: AtomicBool = AtomicBool::new(false); + crate::util::warn_once( + &WARNED, + &format!( + "data directory unusable ({reason}); running in memory only: handled \ + errors and manual reports are still sent, but crashes cannot be \ + recovered and nothing survives this process" + ), + ); } let (session, store) = match disk { Some((session, store)) => (Some(session), Some(store)), @@ -372,6 +395,7 @@ impl Recorder { config.upload_backoff_base, config.sampler, true, + config.legacy_data_dir, )?; Ok(Recorder { @@ -427,8 +451,21 @@ impl Recorder { self.shared.queued.store(0, Ordering::SeqCst); self.shared.report_queued.store(0, Ordering::SeqCst); + // The parent's unsent in-memory reports are the PARENT's to send; the child + // inheriting them would deliver every one a second time. + self.shared.mem_queue.lock().clear(); + + // A child that cannot open its own session degrades to memory rather than + // failing: the parent's disk may be gone or read-only by now. let disk = match self.storage { - StorageMode::Disk => Some(open_disk(&self.data_dir, self.caps)?), + StorageMode::Disk => match open_disk(&self.data_dir, self.caps) { + Ok(disk) => Some(disk), + Err(e) => { + self.storage = StorageMode::Memory; + self.storage_reason = Some(format!("{}: {e}", self.data_dir.display())); + None + } + }, StorageMode::Memory => None, }; let (session, store) = match disk { @@ -445,6 +482,7 @@ impl Recorder { self.backoff_base, sampler, false, + None, )?; // The parent's session is still the parent's: leave its marker alone. @@ -545,6 +583,13 @@ impl Recorder { self.storage } + /// Whether a disk write has failed since launch (disk full, permission + /// denied, directory removed). Reports still go out, rebuilt without capture + /// context or held in memory, but crash recovery can no longer be relied on. + pub fn is_degraded(&self) -> bool { + self.shared.degraded.load(Ordering::Relaxed) + } + /// Why the recorder fell back to [`StorageMode::Memory`], if it did. pub fn storage_reason(&self) -> Option<&str> { self.storage_reason.as_deref() @@ -782,11 +827,13 @@ fn spawn_pipeline( backoff_base: Duration, sampler: Option>, fresh_launch: bool, + legacy_data_dir: Option, ) -> std::io::Result { // Uploader thread: drains the durable queue with retry. let (upload_tx, upload_rx) = channel(); let uploader_shared = Arc::clone(shared); let uploader_data_dir = data_dir.map(Path::to_path_buf); + let uploader_legacy = legacy_data_dir.clone(); let uploader = std::thread::Builder::new() .name("bugsee-uploader".into()) .spawn(move || { @@ -794,6 +841,7 @@ fn spawn_pipeline( upload_rx, uploader_shared, uploader_data_dir, + uploader_legacy, backoff_base, fresh_launch, ) @@ -827,6 +875,11 @@ fn spawn_pipeline( if let Some(dir) = &worker_data_dir { run_recovery(&worker_shared, dir, generation); } + // Best effort, once: sessions a previous default location left + // behind. No generation of ours lives there, so none is skipped. + if let Some(dir) = &legacy_data_dir { + run_recovery(&worker_shared, dir, u64::MAX); + } })); let _ = worker_upload_tx.send(UploadMsg::Wake); worker_loop( @@ -962,7 +1015,12 @@ fn worker_loop( let Some(store) = store.as_mut() else { return }; if let Ok(value) = entry.to_json() { if let Ok(bytes) = serde_json::to_vec(&value) { - let _ = store.append(entry.wire_type(), entry.timestamp(), &bytes); + if store + .append(entry.wire_type(), entry.timestamp(), &bytes) + .is_err() + { + shared.degraded.store(true, Ordering::Relaxed); + } } } })); @@ -1114,7 +1172,10 @@ fn deliver_snapshot( handle.window.end, ) // The snapshot became unreadable (disk gone): still send what we know. - .or_else(|_| assemble_in_memory(shared, &meta, handle.window, None)); + .or_else(|_| { + shared.degraded.store(true, Ordering::Relaxed); + assemble_in_memory(shared, &meta, handle.window, None) + }); match assembled { Ok(assembled) => persist_or_hold(shared, data_dir, assembled), Err(_) => false, @@ -1129,6 +1190,7 @@ fn persist_or_hold(shared: &Shared, data_dir: Option<&Path>, assembled: Assemble if queue::enqueue(dir, &assembled).is_ok() { return true; } + shared.degraded.store(true, Ordering::Relaxed); // A failed enqueue can leave a partial sidecar set; the next launch's // orphan sweep reaps it. Do not retry the disk for this report. } @@ -1164,9 +1226,11 @@ fn hold_in_memory(shared: &Shared, report: AssembledReport) { true } else { queue.push(MemReport { + id: shared.next_mem_id.fetch_add(1, Ordering::Relaxed), report, retry: 0, next_attempt_ms: 0, + endpoint: None, }); false } @@ -1218,6 +1282,7 @@ fn process_report( // With a usable disk, snapshot + assemble from it. Any failure along the way // (most plausibly disk-full or a vanished directory) falls back to a bundle // without capture context, so the report still goes out. + let has_disk = store.is_some() && data_dir.is_some(); let from_disk = match (store, data_dir) { (Some(store), Some(data_dir)) => { assemble_from_disk(store, shared, data_dir, &meta, window, crash_json.clone()) @@ -1226,7 +1291,12 @@ fn process_report( }; let assembled = match from_disk { Some(assembled) => assembled, - None => assemble_in_memory(shared, &meta, window, crash_json)?, + None => { + if has_disk { + shared.degraded.store(true, Ordering::Relaxed); + } + assemble_in_memory(shared, &meta, window, crash_json)? + } }; // Persist to the durable queue; the uploader delivers it. Failing that, hold @@ -1277,24 +1347,38 @@ fn uploader_loop( rx: Receiver, shared: Arc, data_dir: Option, + legacy_data_dir: Option, backoff_base: Duration, fresh_launch: bool, ) { let poll = Duration::from_secs(5); + let legacy = legacy_data_dir; // A fresh launch retries any queued report immediately (force), ignoring // backoff scheduled by the prior session. A fork child skips this: the // parent's uploader is already working the shared queue. Guarded so a // panicking Transport impl can't kill the uploader. if fresh_launch { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain(&shared, data_dir.as_deref(), backoff_base, true) + drain_all( + &shared, + data_dir.as_deref(), + legacy.as_deref(), + backoff_base, + true, + ) })); } loop { match rx.recv_timeout(poll) { Ok(UploadMsg::Wake) => { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain(&shared, data_dir.as_deref(), backoff_base, false) + drain_all( + &shared, + data_dir.as_deref(), + legacy.as_deref(), + backoff_base, + false, + ) })); } Ok(UploadMsg::Drain { deadline, ack }) => { @@ -1310,7 +1394,13 @@ fn uploader_loop( } Err(RecvTimeoutError::Timeout) => { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain(&shared, data_dir.as_deref(), backoff_base, false) + drain_all( + &shared, + data_dir.as_deref(), + legacy.as_deref(), + backoff_base, + false, + ) })); } Err(RecvTimeoutError::Disconnected) => break, @@ -1318,15 +1408,36 @@ fn uploader_loop( } } +/// [`drain`] the data directory, then the legacy one a launch was asked to adopt. +fn drain_all( + shared: &Shared, + data_dir: Option<&Path>, + legacy: Option<&Path>, + backoff_base: Duration, + force: bool, +) -> bool { + let attempted = drain(shared, data_dir, backoff_base, force); + match legacy { + Some(dir) => drain_disk(shared, dir, backoff_base, force) | attempted, + None => attempted, + } +} + /// Attempt delivery of every due queued report once. On the flush path `force` /// is set so backing-off reports are retried immediately. Returns whether any /// report was attempted (used to pace `drain_until_empty`). fn drain(shared: &Shared, data_dir: Option<&Path>, backoff_base: Duration, force: bool) -> bool { + let attempted = drain_memory(shared, backoff_base, force, true, None); + match data_dir { + Some(dir) => drain_disk(shared, dir, backoff_base, force) | attempted, + None => attempted, + } +} + +/// The on-disk half of [`drain`]. +fn drain_disk(shared: &Shared, data_dir: &Path, backoff_base: Duration, force: bool) -> bool { let now = epoch_ms(); - let mut attempted = drain_memory(shared, backoff_base, force, true); - let Some(data_dir) = data_dir else { - return attempted; - }; + let mut attempted = false; for report in queue::list_pending(data_dir) { // Another process sharing the queue may be delivering this one. let Some(_claim) = queue::try_claim(&report) else { @@ -1459,55 +1570,86 @@ fn notify_dropped(shared: &Shared, reason: DropReason) { /// Attempt delivery of the in-memory backlog. Mirrors [`deliver_queued`]: a /// transient failure keeps the report (counting toward the retry cap only on the -/// backoff-paced path), anything terminal drops it and tells the host. The queue -/// is taken out for the duration so the lock is never held across the network. -fn drain_memory(shared: &Shared, backoff_base: Duration, force: bool, count_failure: bool) -> bool { - let pending = std::mem::take(&mut *shared.mem_queue.lock()); - if pending.is_empty() { - return false; - } - let now = epoch_ms(); +/// backoff-paced path), anything terminal drops it and tells the host. +/// +/// An entry STAYS in the queue while it is being delivered (the lock is held only +/// to pick it and to settle it, never across the network), so the queue length is +/// the true backlog and [`MAX_MEMORY_REPORTS`] holds. Stops at `deadline`, like +/// the disk loop in [`drain_until_empty`]. +fn drain_memory( + shared: &Shared, + backoff_base: Duration, + force: bool, + count_failure: bool, + deadline: Option, +) -> bool { let mut attempted = false; - let mut keep = Vec::new(); - for mut item in pending { - if !force && now < item.next_attempt_ms { - keep.push(item); - continue; + // Each entry is tried once per call, whatever its outcome. + let mut tried: Vec = Vec::new(); + loop { + if deadline.is_some_and(|d| Instant::now() >= d) { + break; } + let now = epoch_ms(); + let next = shared + .mem_queue + .lock() + .iter() + .find(|m| !tried.contains(&m.id) && (force || now >= m.next_attempt_ms)) + .map(|m| (m.id, m.report.clone(), m.endpoint.clone())); + let Some((id, report, cached_endpoint)) = next else { + break; + }; + tried.push(id); attempted = true; + let mut endpoint_out = None; - match transport::deliver( + let result = transport::deliver( &*shared.transport, &shared.app_token, &shared.environment_json, &shared.session, - &item.report, - None, + &report, + cached_endpoint.as_deref(), &mut endpoint_out, - ) { - Ok(()) => {} + ); + + // Settle: `None` keeps the (updated) entry, `Some(reason)` ends it. + let mut dropped = None; + let mut queue = shared.mem_queue.lock(); + let Some(pos) = queue.iter().position(|m| m.id == id) else { + continue; // cleared under us (fork child); nothing to settle + }; + match result { + Ok(()) => { + queue.remove(pos); + } Err(TransportError::Transient(_)) | Err(TransportError::SessionExpired) => { + let item = &mut queue[pos]; + item.endpoint = endpoint_out; if count_failure { item.retry += 1; if item.retry >= UPLOAD_RETRY_CAP { - notify_dropped(shared, DropReason::RetriesExhausted); - continue; + queue.remove(pos); + dropped = Some(DropReason::RetriesExhausted); + } else { + item.next_attempt_ms = now + backoff_delay(item.retry, backoff_base); } - item.next_attempt_ms = now + backoff_delay(item.retry, backoff_base); } - keep.push(item); } Err(TransportError::TooManySimilar { .. }) => { - notify_dropped(shared, DropReason::TooManySimilar); + queue.remove(pos); + dropped = Some(DropReason::TooManySimilar); + } + Err(e) => { + queue.remove(pos); + dropped = Some(drop_reason_for(e)); } - Err(e) => notify_dropped(shared, drop_reason_for(e)), } - } - if !keep.is_empty() { - // Reports held while we were delivering stay behind the retained ones. - let mut queue = shared.mem_queue.lock(); - keep.append(&mut queue); - *queue = keep; + drop(queue); + if let Some(reason) = dropped { + notify_dropped(shared, reason); + } } attempted } @@ -1537,27 +1679,29 @@ fn drain_until_empty( let start = Instant::now(); loop { let now = epoch_ms(); - drain_memory(shared, backoff_base, true, false); - for report in data_dir.map(queue::list_pending).unwrap_or_default() { - let Some(_claim) = queue::try_claim(&report) else { - continue; // a peer is delivering it; the loop below waits it out - }; - let Ok((zip, request_json, retry, _next)) = queue::load(&report) else { - continue; - }; - deliver_queued( - shared, - data_dir.unwrap_or(Path::new("")), - backoff_base, - &report, - zip, - request_json, - retry, - now, - false, - ); - if start.elapsed() >= deadline { - break; + drain_memory(shared, backoff_base, true, false, Some(start + deadline)); + if let Some(data_dir) = data_dir { + for report in queue::list_pending(data_dir) { + let Some(_claim) = queue::try_claim(&report) else { + continue; // a peer is delivering it; the loop below waits it out + }; + let Ok((zip, request_json, retry, _next)) = queue::load(&report) else { + continue; + }; + deliver_queued( + shared, + data_dir, + backoff_base, + &report, + zip, + request_json, + retry, + now, + false, + ); + if start.elapsed() >= deadline { + break; + } } } let disk_empty = data_dir.is_none_or(|d| queue::list_pending(d).is_empty()); diff --git a/crates/bugsee-core/src/util.rs b/crates/bugsee-core/src/util.rs index ed27ce0..0c0f3c8 100644 --- a/crates/bugsee-core/src/util.rs +++ b/crates/bugsee-core/src/util.rs @@ -10,14 +10,17 @@ use std::time::{SystemTime, UNIX_EPOCH}; -/// Print `message` to stderr the first time it is called in this process and do +/// Print `message` to stderr the first time it is called with `flag` and do /// nothing after. For conditions the SDK must announce (it otherwise does no -/// logging) but that would be noise if repeated, such as a degraded data dir. -pub fn warn_once(message: &str) { - use std::sync::atomic::{AtomicBool, Ordering}; - static WARNED: AtomicBool = AtomicBool::new(false); - if !WARNED.swap(true, Ordering::Relaxed) { - eprintln!("bugsee: {message}"); +/// logging) but that would be noise if repeated, such as a degraded data dir; +/// each kind of message owns a `static` flag so one does not mute another. +/// +/// Never panics: a closed or broken stderr is ignored (`eprintln!` would panic). +pub fn warn_once(flag: &std::sync::atomic::AtomicBool, message: &str) { + use std::io::Write; + use std::sync::atomic::Ordering; + if !flag.swap(true, Ordering::Relaxed) { + let _ = writeln!(std::io::stderr(), "bugsee: {message}"); } } diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index 94f6b1b..eb021de 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -106,7 +106,7 @@ fn memory_only_config_skips_the_disk_entirely() { let transport = Arc::new(MockTransport::default()); let data_dir = dir.path.join("must-stay-absent"); let mut config = RecorderConfig::new(&data_dir, "T"); - config.memory_only = true; + config.memory_only = Some("test".to_string()); let recorder = Recorder::launch(config, transport.clone()).expect("launch"); @@ -140,6 +140,9 @@ fn read_only_directory_runs_in_memory_and_still_reports() { #[test] fn directory_removed_mid_run_still_reports() { + // The SDK recreates a vanished directory (`create_dir_all`) and open part + // files keep accepting writes to their unlinked inodes, so this report takes + // the disk queue again (the earlier capture window is gone with the files). let dir = TempDir::new(); let data_dir = dir.path.join("data"); let transport = Arc::new(MockTransport::default()); @@ -171,7 +174,12 @@ fn directory_replaced_by_a_file_mid_run_falls_back_to_memory_delivery() { std::fs::remove_dir_all(&data_dir).unwrap(); std::fs::write(&data_dir, b"a file now").unwrap(); + assert!(!recorder.is_degraded()); assert_eq!(report_and_flush(&recorder, &transport), 1); + assert!( + recorder.is_degraded(), + "mid-run degradation must be visible" + ); drop(recorder); } @@ -191,49 +199,215 @@ fn queue_write_failure_holds_the_report_in_memory() { assert_eq!(recorder.storage_mode(), StorageMode::Disk); assert_eq!(report_and_flush(&recorder, &transport), 1); + assert!(recorder.is_degraded()); + drop(recorder); + // The failed enqueue must not leak its hard-linked snapshot. + let leftovers = std::fs::read_dir(data_dir.join("reports")) + .map(|rd| rd.flatten().count()) + .unwrap_or(0); + assert_eq!(leftovers, 0, "reports/ snapshot dirs must be reclaimed"); } #[test] -fn memory_backlog_is_bounded_and_overflow_is_reported() { - use bugsee_core::runtime::DropReason; - use bugsee_core::transport::{Transport, TransportError}; - use std::sync::Mutex; +fn legacy_directory_leftovers_are_delivered_once() { + use bugsee_core::model::report::TimeWindow; + use bugsee_core::reporting::assemble_docs; - // Never delivers, so everything stays in the memory backlog. - struct Offline; - impl Transport for Offline { - fn register_session(&self, _: &str, _: &[u8]) -> Result { - Err(TransportError::Transient("offline".into())) - } - fn create_issue( - &self, - _: &str, - _: Option<&str>, - _: &[u8], - ) -> Result { - Err(TransportError::Transient("offline".into())) - } - fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), TransportError> { - Err(TransportError::Transient("offline".into())) + let dir = TempDir::new(); + let legacy = dir.path.join("legacy"); + let data_dir = dir.path.join("data"); + let meta = manual_upload_meta(); + let env = bugsee_core::model::environment::Environment::detect_with( + "t", + &Default::default(), + &Default::default(), + ); + let window = TimeWindow { start: 0, end: 1 }; + let stranded = assemble_docs(Vec::new(), window, &meta, &env, None, &[], "T", 1).unwrap(); + bugsee_core::queue::enqueue(&legacy, &stranded).unwrap(); + + let transport = Arc::new(MockTransport::default()); + let mut config = RecorderConfig::new(&data_dir, "T"); + config.legacy_data_dir = Some(legacy.clone()); + let recorder = Recorder::launch(config, transport.clone()).expect("launch"); + assert!(recorder.flush(Duration::from_secs(10))); + // The startup drain is asynchronous; flush covers only the data dir queue. + let start = std::time::Instant::now(); + while transport.uploaded_bundles.lock().unwrap().is_empty() + && start.elapsed() < Duration::from_secs(10) + { + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 1); + assert!( + bugsee_core::queue::list_pending(&legacy).is_empty(), + "a delivered legacy report must be removed" + ); +} + +/// Blocks every delivery until released, so the backlog cannot drain. +struct Gate { + open: std::sync::Mutex, + cv: std::sync::Condvar, +} +impl Gate { + fn release(&self) { + *self.open.lock().unwrap() = true; + self.cv.notify_all(); + } +} +struct Gated(Arc); +impl bugsee_core::Transport for Gated { + fn register_session(&self, _: &str, _: &[u8]) -> Result { + let mut open = self.0.open.lock().unwrap(); + while !*open { + open = self.0.cv.wait(open).unwrap(); } + Err(bugsee_core::TransportError::Transient("offline".into())) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + Err(bugsee_core::TransportError::Transient("offline".into())) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), bugsee_core::TransportError> { + Err(bugsee_core::TransportError::Transient("offline".into())) } +} + +#[test] +fn memory_backlog_is_bounded_even_while_a_delivery_is_in_flight() { + use bugsee_core::runtime::DropReason; + use std::sync::Mutex; let dir = TempDir::new(); + let gate = Arc::new(Gate { + open: Mutex::new(false), + cv: std::sync::Condvar::new(), + }); let dropped = Arc::new(Mutex::new(Vec::new())); let sink = dropped.clone(); let mut config = RecorderConfig::new(blocked_path(&dir.path), "T"); config.on_report_dropped = Some(Box::new(move |r| sink.lock().unwrap().push(r))); - let recorder = Recorder::launch(config, Arc::new(Offline)).expect("launch"); + let recorder = Recorder::launch(config, Arc::new(Gated(gate.clone()))).expect("launch"); for _ in 0..40 { recorder.report(manual_upload_meta(), None); } - // Flush cannot succeed (offline) but must return promptly, not hang. + // The worker has handled every report once a Flush message round-trips + // (the flush itself cannot finish: the uploader is held in the gate). assert!(!recorder.flush(Duration::from_millis(500))); let seen = dropped.lock().unwrap().clone(); + gate.release(); + assert_eq!( + seen.len(), + 40 - 16, + "exactly the reports over the cap are dropped, got {seen:?}" + ); + assert!(seen.iter().all(|r| *r == DropReason::StorageUnavailable)); +} + +#[test] +fn memory_flush_respects_its_deadline_with_a_full_backlog() { + use std::time::Instant; + + // Every attempt costs 100 ms and fails transiently. + struct Slow; + impl bugsee_core::Transport for Slow { + fn register_session( + &self, + _: &str, + _: &[u8], + ) -> Result { + std::thread::sleep(Duration::from_millis(100)); + Err(bugsee_core::TransportError::Transient("slow".into())) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + Err(bugsee_core::TransportError::Transient("slow".into())) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), bugsee_core::TransportError> { + Err(bugsee_core::TransportError::Transient("slow".into())) + } + } + + let dir = TempDir::new(); + let recorder = Recorder::launch( + RecorderConfig::new(blocked_path(&dir.path), "T"), + Arc::new(Slow), + ) + .expect("launch"); + for _ in 0..16 { + recorder.report(manual_upload_meta(), None); + } + let start = Instant::now(); + assert!(!recorder.flush(Duration::from_millis(400))); + // 16 attempts at 100 ms would be 1.6 s per pass if the deadline were ignored. assert!( - !seen.is_empty() && seen.iter().all(|r| *r == DropReason::StorageUnavailable), - "overflow must be surfaced, got {seen:?}" + start.elapsed() < Duration::from_millis(1200), + "flush overran its deadline: {:?}", + start.elapsed() + ); +} + +#[test] +fn a_failed_put_retries_at_the_put_and_does_not_mint_a_duplicate_issue() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + // The issue is created, then the first PUT fails transiently. + #[derive(Default)] + struct FlakyPut { + issues: AtomicUsize, + puts: AtomicUsize, + } + struct Handle(Arc); + impl bugsee_core::Transport for Handle { + fn register_session( + &self, + _: &str, + _: &[u8], + ) -> Result { + Ok("token".into()) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + self.0.issues.fetch_add(1, Ordering::SeqCst); + Ok("https://uploads.example/x".into()) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), bugsee_core::TransportError> { + if self.0.puts.fetch_add(1, Ordering::SeqCst) == 0 { + Err(bugsee_core::TransportError::Transient("reset".into())) + } else { + Ok(()) + } + } + } + + let dir = TempDir::new(); + let flaky = Arc::new(FlakyPut::default()); + let recorder = Recorder::launch( + RecorderConfig::new(blocked_path(&dir.path), "T"), + Arc::new(Handle(flaky.clone())), + ) + .expect("launch"); + recorder.report(manual_upload_meta(), None); + assert!(recorder.flush(Duration::from_secs(10))); + assert_eq!(flaky.puts.load(Ordering::SeqCst), 2); + assert_eq!( + flaky.issues.load(Ordering::SeqCst), + 1, + "the retry must resume at the PUT, not create the issue again" ); } diff --git a/crates/bugsee-ffi/tests/lifecycle.rs b/crates/bugsee-ffi/tests/lifecycle.rs index 702199f..1eaf113 100644 --- a/crates/bugsee-ffi/tests/lifecycle.rs +++ b/crates/bugsee-ffi/tests/lifecycle.rs @@ -21,6 +21,10 @@ fn full_c_abi_lifecycle() { let token = c(&format!("ffiLifecycle{}", std::process::id())); let endpoint = c("http://127.0.0.1:1"); // connection refused; offline-safe + // Keep the SDK's state out of the developer's real data directory. + let data = std::env::temp_dir().join(format!("bugsee-ffi-{}", std::process::id())); + std::env::set_var(bugsee::DATA_DIR_ENV, &data); + assert_eq!( unsafe { bugsee_launch_with_endpoint(token.as_ptr(), endpoint.as_ptr()) }, BugseeStatus::Ok @@ -85,8 +89,5 @@ fn full_c_abi_lifecycle() { assert_eq!(bugsee_stop(), BugseeStatus::Ok); assert_eq!(bugsee_is_active(), 0, "stopped is not active"); - // Cleanup the default per-token data dir. - let mut data = std::env::temp_dir(); - data.push("bugsee"); let _ = std::fs::remove_dir_all(data); } diff --git a/crates/bugsee/Cargo.toml b/crates/bugsee/Cargo.toml index bc971a1..3051090 100644 --- a/crates/bugsee/Cargo.toml +++ b/crates/bugsee/Cargo.toml @@ -17,6 +17,9 @@ backtrace = "0.3" ureq = { version = "2", optional = true, features = ["json"] } sysinfo = { version = "0.33", optional = true, default-features = false, features = ["system", "disk"] } +[target.'cfg(unix)'.dependencies] +libc = "0.2" + [dev-dependencies] zip = { version = "2", default-features = false, features = ["zstd"] } diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 02b552d..54caa8e 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -50,6 +50,7 @@ struct Resolved { dir: Option, source: DataDirSource, skipped: Vec, + explicit_rejected: Option, } static RESOLUTION: ForkMutex> = ForkMutex::new(|| None); @@ -86,15 +87,26 @@ impl Bugsee { dir, source, skipped, + explicit_rejected, + legacy_dir, } = options.resolve_data_dir(); - let memory_only = dir.is_none(); + let memory_only = dir.is_none().then(|| { + explicit_rejected + .clone() + .or_else(|| (!skipped.is_empty()).then(|| skipped.join("; "))) + .unwrap_or_else(|| "no usable data directory".to_string()) + }); let data_dir = dir.unwrap_or_else(|| std::env::temp_dir().join("bugsee")); - if !memory_only && !skipped.is_empty() { - bugsee_core::util::warn_once(&format!( - "data directory unusable ({}); using {}", - skipped.join("; "), - data_dir.display() - )); + if memory_only.is_none() && !skipped.is_empty() { + static WARNED: AtomicBool = AtomicBool::new(false); + bugsee_core::util::warn_once( + &WARNED, + &format!( + "default data directory unusable ({}); using {}", + skipped.join("; "), + data_dir.display() + ), + ); } let transport = match options.transport.clone() { @@ -118,6 +130,7 @@ impl Bugsee { let mut config = RecorderConfig::new(data_dir, options.app_token.clone()); config.memory_only = memory_only; + config.legacy_data_dir = legacy_dir; config.caps = options.caps(); config.rotate_interval = options.rotate_interval; config.before_send = options.before_send; @@ -148,7 +161,7 @@ impl Bugsee { // Nothing can be recovered without a disk, so crash markers and panic // snapshots have nowhere to go. let persistent = recorder.storage_mode() == StorageMode::Disk; - *RESOLUTION.lock() = Some(Resolved { + let resolved = Resolved { dir: persistent.then(|| recorder.data_dir().to_path_buf()), source: if persistent { source @@ -156,7 +169,8 @@ impl Bugsee { DataDirSource::None }, skipped, - }); + explicit_rejected, + }; #[cfg(feature = "panic")] let panic_info_path = recorder.panic_info_path(); @@ -183,7 +197,13 @@ impl Bugsee { // `Recorder::drop` joins the worker/uploader threads (blocking up to a few // seconds), and doing that under the global lock would stall every // concurrent capture call β€” the same hazard `stop()` guards against. - let old = RECORDER.lock().replace(recorder); + // The resolution is published under the same lock as the recorder, so + // `storage_status` never pairs one launch's recorder with another's. + let old = { + let mut slot = RECORDER.lock(); + *RESOLUTION.lock() = Some(resolved); + slot.replace(recorder) + }; drop(old); PAUSED.store(false, Ordering::SeqCst); *REVIVE_INFO.lock() = Some(ReviveInfo { @@ -217,22 +237,20 @@ impl Bugsee { /// ([`StorageStatus::may_be_ephemeral`]). pub fn storage_status() -> Option { check_fork(); - let (mode, reason) = { - let guard = RECORDER.lock(); - let recorder = guard.as_ref()?; - ( - recorder.storage_mode(), - recorder.storage_reason().map(String::from), - ) - }; + // Both locks, always in this order (as in `launch_with`), so the pair is + // one launch's. + let recorder = RECORDER.lock(); + let recorder = recorder.as_ref()?; let resolved = RESOLUTION.lock(); let resolved = resolved.as_ref()?; Some(StorageStatus { - mode, + mode: recorder.storage_mode(), data_dir: resolved.dir.clone(), source: resolved.source, skipped: resolved.skipped.clone(), - reason, + explicit_rejected: resolved.explicit_rejected.clone(), + reason: recorder.storage_reason().map(String::from), + degraded: recorder.is_degraded(), }) } @@ -558,8 +576,11 @@ fn revive_after_fork(epoch: u64) { #[cfg(feature = "native")] if old_native.is_some() { drop(old_native); - if let Ok(handler) = bugsee_native::install(recorder.crash_info_path()) { - *NATIVE.lock() = Some(handler); + // A child that fell back to memory has nowhere to keep a crash marker. + if recorder.storage_mode() == StorageMode::Disk { + if let Ok(handler) = bugsee_native::install(recorder.crash_info_path()) { + *NATIVE.lock() = Some(handler); + } } } #[cfg(feature = "panic")] diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index acd48e3..757d084 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -231,8 +231,8 @@ impl LaunchOptions { } } - /// Pick the data directory: the first candidate in [`DataDirSource`] order - /// that can actually be created and written to. + /// Pick the data directory; see [`DataDirSource`] for the order and what an + /// unusable choice means. pub(crate) fn resolve_data_dir(&self) -> DataDirResolution { resolve_data_dir( self.data_dir.as_deref(), @@ -249,25 +249,30 @@ pub const DATA_DIR_ENV: &str = "BUGSEE_DATA_DIR"; /// Which step of the resolution order produced the data directory. /// -/// Candidates are tried in this order and the first one that is writable wins; -/// an unusable one is skipped (and recorded in -/// [`StorageStatus::skipped`](crate::StorageStatus::skipped)), never fatal: -/// /// 1. [`Explicit`](DataDirSource::Explicit): [`LaunchOptions::data_dir`]. /// 2. [`Environment`](DataDirSource::Environment): `BUGSEE_DATA_DIR`. /// 3. [`PlatformDefault`](DataDirSource::PlatformDefault): `bugsee/` under /// `$XDG_DATA_HOME` (else `~/.local/share`) on Linux, `~/Library/Application /// Support` on macOS, `%LOCALAPPDATA%` on Windows. Skipped when the base is /// unset, as in a container with no home directory. -/// 4. [`TempFallback`](DataDirSource::TempFallback): `bugsee/` under the -/// system temp directory. Often wiped on restart, so crash recovery may not -/// survive it; [`StorageStatus::may_be_ephemeral`](crate::StorageStatus::may_be_ephemeral) +/// 4. [`TempFallback`](DataDirSource::TempFallback): `bugsee-/` under +/// the system temp directory (`bugsee/` on Windows), created private +/// (`0700`) and refused if it is a symlink or owned by another user. Often +/// wiped on restart, so crash recovery may not survive it; +/// [`StorageStatus::may_be_ephemeral`](crate::StorageStatus::may_be_ephemeral) /// reports it. /// -/// If none is writable the SDK runs +/// **A directory you asked for is final.** If 1 or 2 is set and not writable the +/// SDK does NOT quietly use another location on disk (that would scatter +/// reports somewhere the operator never chose); it runs in memory and reports +/// why in [`StorageStatus::explicit_rejected`](crate::StorageStatus::explicit_rejected). +/// Only the defaults (3, then 4) fall through to each other. +/// +/// With no usable directory the SDK runs /// [in memory](bugsee_core::runtime::StorageMode::Memory): handled errors and /// manual reports are still sent, but there is no capture window and no crash /// recovery. It logs one line to stderr when it degrades. +#[non_exhaustive] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DataDirSource { Explicit, @@ -283,8 +288,14 @@ pub(crate) struct DataDirResolution { /// The chosen directory, or `None` if no candidate was writable. pub dir: Option, pub source: DataDirSource, - /// Candidates tried and rejected before the winner, with the reason. + /// Default candidates tried and rejected before the winner, with the reason. pub skipped: Vec, + /// Why an explicit / environment directory was rejected (it is final). + pub explicit_rejected: Option, + /// The location earlier SDK versions used by default + /// (`/bugsee/`), if it exists, is trustworthy and is not `dir`. + /// Its leftovers are recovered once at launch. + pub legacy_dir: Option, } /// Pure resolution core: `env` and `temp` are injected so tests need not touch @@ -295,37 +306,80 @@ pub(crate) fn resolve_data_dir( env: &dyn Fn(&str) -> Option, temp: &Path, ) -> DataDirResolution { - let mut candidates: Vec<(DataDirSource, PathBuf)> = Vec::new(); - if let Some(dir) = explicit { - candidates.push((DataDirSource::Explicit, dir.to_path_buf())); - } - if let Some(dir) = env(DATA_DIR_ENV).filter(|v| !v.is_empty()) { - candidates.push((DataDirSource::Environment, PathBuf::from(dir))); - } - // Isolate by app token so multiple apps don't collide. - let app = |base: PathBuf| base.join("bugsee").join(short_token(app_token)); + let token = short_token(app_token); + + let chosen = explicit + .map(|d| (DataDirSource::Explicit, d.to_path_buf())) + .or_else(|| { + env(DATA_DIR_ENV) + .filter(|v| !v.is_empty()) + .map(|v| (DataDirSource::Environment, PathBuf::from(v))) + }); + if let Some((source, dir)) = chosen { + return match prepare_dir(&dir, false) { + Ok(()) => DataDirResolution { + dir: Some(dir), + source, + skipped: Vec::new(), + explicit_rejected: None, + legacy_dir: None, + }, + Err(e) => DataDirResolution { + dir: None, + source: DataDirSource::None, + skipped: Vec::new(), + explicit_rejected: Some(format!("{}: {e}", dir.display())), + legacy_dir: None, + }, + }; + } + + let mut candidates: Vec<(DataDirSource, PathBuf, bool)> = Vec::new(); if let Some(base) = platform_data_base(env) { - candidates.push((DataDirSource::PlatformDefault, app(base))); + candidates.push(( + DataDirSource::PlatformDefault, + base.join("bugsee").join(&token), + false, + )); } - candidates.push((DataDirSource::TempFallback, app(temp.to_path_buf()))); + candidates.push(( + DataDirSource::TempFallback, + temp.join(private_temp_name()).join(&token), + true, + )); let mut skipped = Vec::new(); - for (source, dir) in candidates { - match probe_writable(&dir) { + let mut found = None; + for (source, dir, private) in candidates { + match prepare_dir(&dir, private) { Ok(()) => { - return DataDirResolution { - dir: Some(dir), - source, - skipped, - } + found = Some((source, dir)); + break; } Err(e) => skipped.push(format!("{}: {e}", dir.display())), } } - DataDirResolution { - dir: None, - source: DataDirSource::None, - skipped, + + let legacy = temp.join("bugsee").join(&token); + let legacy_dir = found + .as_ref() + .filter(|(_, dir)| *dir != legacy) + .and_then(|_| trusted_existing_dir(&legacy).then_some(legacy)); + match found { + Some((source, dir)) => DataDirResolution { + dir: Some(dir), + source, + skipped, + explicit_rejected: None, + legacy_dir, + }, + None => DataDirResolution { + dir: None, + source: DataDirSource::None, + skipped, + explicit_rejected: None, + legacy_dir: None, + }, } } @@ -341,14 +395,113 @@ fn platform_data_base(env: &dyn Fn(&str) -> Option) -> Option } } +/// The per-user directory name under the shared temp dir. The system temp +/// directory is world-writable on unix, so a fixed name would let another user +/// pre-create (or symlink) it and read or plant our reports. +#[cfg(unix)] +fn private_temp_name() -> String { + // SAFETY: `geteuid` has no preconditions and cannot fail. + format!("bugsee-{}", unsafe { libc::geteuid() }) +} +#[cfg(not(unix))] +fn private_temp_name() -> String { + "bugsee".to_string() +} + /// Create `dir` and prove a file can be written in it. Checking writability (not /// just existence) is what catches a read-only root file system or a volume -/// mounted read-only. +/// mounted read-only. With `private` (the shared temp area) each created +/// component is `0700` and an existing one is refused unless it is a real +/// directory owned by us. +fn prepare_dir(dir: &Path, private: bool) -> std::io::Result<()> { + if private { + make_private(dir)?; + } else { + std::fs::create_dir_all(dir)?; + } + probe_writable(dir) +} + +#[cfg(unix)] +fn make_private(dir: &Path) -> std::io::Result<()> { + use std::os::unix::fs::{DirBuilderExt, MetadataExt, PermissionsExt}; + // The shared temp dir itself is the system's; only the two components below + // it are ours to vet. + let leaf = dir; + let top = leaf.parent().unwrap_or(leaf); + if let Some(shared) = top.parent() { + std::fs::create_dir_all(shared)?; + } + // SAFETY: `geteuid` has no preconditions and cannot fail. + let uid = unsafe { libc::geteuid() }; + for component in [top, leaf] { + match std::fs::DirBuilder::new().mode(0o700).create(component) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(e), + } + // `symlink_metadata`: a symlink planted at our name must not be followed. + let meta = std::fs::symlink_metadata(component)?; + if !meta.file_type().is_dir() { + return Err(std::io::Error::other( + "not a real directory (symlink or file at our private name)", + )); + } + if meta.uid() != uid { + return Err(std::io::Error::other("owned by another user")); + } + if meta.mode() & 0o077 != 0 { + std::fs::set_permissions(component, std::fs::Permissions::from_mode(0o700))?; + } + } + Ok(()) +} +#[cfg(not(unix))] +fn make_private(dir: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(dir) +} + +/// Whether `dir` exists and may be trusted to hold OUR leftovers: a real +/// directory (not a symlink) owned by the current user. The legacy location is +/// in the shared temp area, where anyone could have planted files. +fn trusted_existing_dir(dir: &Path) -> bool { + let Ok(meta) = std::fs::symlink_metadata(dir) else { + return false; + }; + if !meta.file_type().is_dir() { + return false; + } + owned_by_us(&meta) +} + +#[cfg(unix)] +fn owned_by_us(meta: &std::fs::Metadata) -> bool { + use std::os::unix::fs::MetadataExt; + // SAFETY: `geteuid` has no preconditions and cannot fail. + meta.uid() == unsafe { libc::geteuid() } +} +#[cfg(not(unix))] +fn owned_by_us(_meta: &std::fs::Metadata) -> bool { + true +} + +/// Prove a file can be created in `dir`. `create_new` with an unpredictable name +/// refuses to follow or reuse anything pre-planted at that path. fn probe_writable(dir: &Path) -> std::io::Result<()> { use std::io::Write; - std::fs::create_dir_all(dir)?; - let probe = dir.join(format!(".bugsee-probe-{}", std::process::id())); - let result = std::fs::File::create(&probe).and_then(|mut f| f.write_all(b"1")); + let probe = dir.join(format!( + ".bugsee-probe-{}-{}", + std::process::id(), + bugsee_core::util::random_hex(8) + )); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let result = options.open(&probe).and_then(|mut f| f.write_all(b"1")); let _ = std::fs::remove_file(&probe); result } @@ -360,6 +513,7 @@ fn probe_writable(dir: &Path) -> std::io::Result<()> { /// the process (a volume mount). The SDK cannot tell a volume from a scratch /// disk, so check [`may_be_ephemeral`](StorageStatus::may_be_ephemeral) and /// your own deployment. +#[non_exhaustive] #[derive(Debug, Clone)] pub struct StorageStatus { /// `Disk` when the data directory is usable; `Memory` when it is not. @@ -368,18 +522,26 @@ pub struct StorageStatus { pub data_dir: Option, /// Which resolution step produced `data_dir`. pub source: DataDirSource, - /// Candidates rejected before the one in use, with the error for each. + /// Default candidates rejected before the one in use, with the error for each. pub skipped: Vec, + /// Set when the directory the host asked for (`data_dir` or + /// `BUGSEE_DATA_DIR`) was unusable. It is final: the SDK is in memory mode, + /// not on some other disk location. + pub explicit_rejected: Option, /// Why the SDK is in memory mode, if it is. pub reason: Option, + /// A disk write has failed since launch (full, read-only, removed). Reports + /// still go out, but crash recovery can no longer be relied on. + pub degraded: bool, } impl StorageStatus { /// Whether a fatal crash can be recovered on the next launch: true only with - /// a usable data directory. (It also needs that directory to survive the - /// process; see [`may_be_ephemeral`](Self::may_be_ephemeral).) + /// a usable data directory that has not failed since launch. (It also needs + /// that directory to survive the process; see + /// [`may_be_ephemeral`](Self::may_be_ephemeral).) pub fn crash_recovery_available(&self) -> bool { - self.mode == bugsee_core::runtime::StorageMode::Disk + self.mode == bugsee_core::runtime::StorageMode::Disk && !self.degraded } /// Whether the directory is the temp-dir fallback, which is commonly wiped @@ -426,6 +588,14 @@ mod tests { } } + fn home_env(home: &Path) -> Vec<(&'static str, OsString)> { + vec![ + ("HOME", home.as_os_str().to_owned()), + ("XDG_DATA_HOME", home.as_os_str().to_owned()), + ("LOCALAPPDATA", home.as_os_str().to_owned()), + ] + } + #[test] fn explicit_directory_wins_when_writable() { let s = scratch(); @@ -434,20 +604,53 @@ mod tests { let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&env), &s.join("tmp")); assert_eq!(r.source, DataDirSource::Explicit); assert_eq!(r.dir.as_deref(), Some(explicit.as_path())); - assert!(r.skipped.is_empty()); + assert!(r.skipped.is_empty() && r.explicit_rejected.is_none()); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn unwritable_explicit_is_final_and_never_falls_to_another_location() { + let s = scratch(); + let good_env = s.join("from-env"); + let env = [(DATA_DIR_ENV, good_env.clone().into_os_string())]; + let mut full_env = env.to_vec(); + full_env.extend(home_env(&s.join("home"))); + let r = resolve_data_dir( + Some(&blocked(&s, "f1")), + "tok", + &env_of(&full_env), + &s.join("tmp"), + ); + assert!(r.dir.is_none()); + assert_eq!(r.source, DataDirSource::None); + assert!(r.explicit_rejected.is_some()); + assert!( + !good_env.exists(), + "the environment variable must not be tried" + ); + assert!(!s.join("home").exists() && !s.join("tmp").exists()); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn unwritable_environment_directory_is_final_too() { + let s = scratch(); + let mut env = vec![(DATA_DIR_ENV, blocked(&s, "f2").into_os_string())]; + env.extend(home_env(&s.join("home"))); + let r = resolve_data_dir(None, "tok", &env_of(&env), &s.join("tmp")); + assert!(r.dir.is_none() && r.explicit_rejected.is_some()); + assert!(!s.join("home").exists() && !s.join("tmp").exists()); let _ = std::fs::remove_dir_all(&s); } #[test] - fn unwritable_explicit_falls_through_to_the_environment_variable() { + fn environment_variable_is_used_without_an_explicit_directory() { let s = scratch(); - let bad = blocked(&s, "f1"); let good = s.join("from-env"); let env = [(DATA_DIR_ENV, good.clone().into_os_string())]; - let r = resolve_data_dir(Some(&bad), "tok", &env_of(&env), &s.join("tmp")); + let r = resolve_data_dir(None, "tok", &env_of(&env), &s.join("tmp")); assert_eq!(r.source, DataDirSource::Environment); assert_eq!(r.dir, Some(good)); - assert_eq!(r.skipped.len(), 1, "the rejected candidate is recorded"); let _ = std::fs::remove_dir_all(&s); } @@ -455,40 +658,63 @@ mod tests { fn platform_default_is_used_when_home_is_set() { let s = scratch(); let home = s.join("home"); - let env = [ - ("HOME", home.clone().into_os_string()), - ("XDG_DATA_HOME", home.clone().into_os_string()), - ("LOCALAPPDATA", home.clone().into_os_string()), - ]; - let r = resolve_data_dir(None, "tok-1", &env_of(&env), &s.join("tmp")); + let r = resolve_data_dir(None, "tok-1", &env_of(&home_env(&home)), &s.join("tmp")); assert_eq!(r.source, DataDirSource::PlatformDefault); assert!(r.dir.unwrap().starts_with(&home)); let _ = std::fs::remove_dir_all(&s); } #[test] - fn no_home_falls_back_to_the_temp_directory() { + fn no_home_falls_back_to_a_private_temp_directory() { let s = scratch(); let tmp = s.join("tmp"); let r = resolve_data_dir(None, "tok-1", &env_of(&[]), &tmp); assert_eq!(r.source, DataDirSource::TempFallback); - assert_eq!(r.dir, Some(tmp.join("bugsee").join("tok1"))); + let dir = r.dir.unwrap(); + assert_eq!(dir, tmp.join(private_temp_name()).join("tok1")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for d in [dir.as_path(), dir.parent().unwrap()] { + let mode = std::fs::metadata(d).unwrap().permissions().mode(); + assert_eq!(mode & 0o077, 0, "{} must be private", d.display()); + } + } let _ = std::fs::remove_dir_all(&s); } + #[cfg(unix)] #[test] - fn nothing_writable_resolves_to_none_and_lists_every_attempt() { + fn temp_fallback_refuses_a_symlink_planted_at_the_private_name() { let s = scratch(); - let env = [(DATA_DIR_ENV, blocked(&s, "f2").into_os_string())]; + let tmp = s.join("tmp"); + let elsewhere = s.join("elsewhere"); + std::fs::create_dir_all(&tmp).unwrap(); + std::fs::create_dir_all(&elsewhere).unwrap(); + std::os::unix::fs::symlink(&elsewhere, tmp.join(private_temp_name())).unwrap(); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &tmp); + assert!(r.dir.is_none(), "must not follow the symlink"); + assert_eq!(r.skipped.len(), 1); + assert!( + std::fs::read_dir(&elsewhere).unwrap().next().is_none(), + "nothing may be created through the symlink" + ); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn nothing_writable_resolves_to_none_and_lists_every_default_attempt() { + let s = scratch(); + let blocked_home = blocked(&s, "f3"); let r = resolve_data_dir( - Some(&blocked(&s, "f3")), + None, "tok", - &env_of(&env), + &env_of(&home_env(&blocked_home)), &blocked(&s, "f4"), ); assert_eq!(r.source, DataDirSource::None); assert!(r.dir.is_none()); - assert_eq!(r.skipped.len(), 3); + assert_eq!(r.skipped.len(), 2); let _ = std::fs::remove_dir_all(&s); } @@ -500,4 +726,25 @@ mod tests { assert_eq!(r.source, DataDirSource::TempFallback); let _ = std::fs::remove_dir_all(&s); } + + // Unix only: elsewhere the temp fallback IS the old location. + #[cfg(unix)] + #[test] + fn the_previous_default_location_is_offered_for_recovery_only_if_it_exists() { + let s = scratch(); + let tmp = s.join("tmp"); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &tmp); + assert!(r.legacy_dir.is_none(), "nothing to adopt yet"); + + let legacy = tmp.join("bugsee").join("tok"); + std::fs::create_dir_all(&legacy).unwrap(); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &tmp); + assert_eq!(r.legacy_dir, Some(legacy.clone())); + + // A host that chose its directory never used the legacy one. + let explicit = s.join("explicit"); + let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&[]), &tmp); + assert!(r.legacy_dir.is_none()); + let _ = std::fs::remove_dir_all(&s); + } } diff --git a/crates/bugsee/tests/fork_safety.rs b/crates/bugsee/tests/fork_safety.rs index 14c77eb..64365f0 100644 --- a/crates/bugsee/tests/fork_safety.rs +++ b/crates/bugsee/tests/fork_safety.rs @@ -518,3 +518,81 @@ fn a_panic_in_the_child_before_touching_the_sdk_does_not_write_into_the_parents_ let _ = std::fs::remove_dir_all(&data); let _ = std::fs::remove_dir_all(&out); } + +/// Fails transiently until `allow` exists, then behaves like [`FileTransport`]. +struct AllowTransport { + allow: PathBuf, + inner: Arc, +} + +impl Transport for AllowTransport { + fn register_session(&self, a: &str, b: &[u8]) -> Result { + self.inner.register_session(a, b) + } + fn create_issue(&self, a: &str, b: Option<&str>, c: &[u8]) -> Result { + if self.allow.exists() { + self.inner.create_issue(a, b, c) + } else { + Err(TransportError::Transient("offline".into())) + } + } + fn upload_bundle(&self, e: &str, zip: &[u8]) -> Result<(), TransportError> { + self.inner.upload_bundle(e, zip) + } +} + +#[test] +fn a_child_in_memory_mode_does_not_resend_the_parents_unsent_reports() { + let _serial = serial(); + let root = scratch("mem"); + let out = root.join("out"); + let allow = root.join("allow"); + // A directory that can never be created: the SDK runs in memory. + let blocker = root.join("blocker"); + std::fs::write(&blocker, b"x").unwrap(); + let guard = Bugsee::launch_with( + LaunchOptions::new("APP_TOKEN") + .data_dir(blocker.join("data")) + .native_crash_capture(false) + .with_transport(Arc::new(AllowTransport { + allow: allow.clone(), + inner: FileTransport::new(&out), + })), + ) + .expect("launch"); + + // Offline: the parent's report stays in its in-memory backlog. + Bugsee::capture_message(LogLevel::Error, "parent-held-report"); + // (A sleep, not a timed-out `flush`: a main thread that has just waited on a + // timed channel recv carries its parker state across `fork` and on macOS + // libdispatch traps the child.) + std::thread::sleep(Duration::from_millis(500)); + + let child = fork_child(|| { + Bugsee::capture_message(LogLevel::Error, "child-report"); + std::fs::write(&allow, b"1").unwrap(); + let ok = Bugsee::flush(Duration::from_secs(10)); + i32::from(!ok) + }); + assert!(exited_with( + wait_with_deadline(child, Duration::from_secs(30)), + 0 + )); + assert!(Bugsee::flush(Duration::from_secs(10))); + + let reports = delivered(&out); + assert_eq!( + mentions(&reports, "child-report"), + 1, + "{}", + summarize(&reports) + ); + assert_eq!( + mentions(&reports, "parent-held-report"), + 1, + "the child must not resend the parent's backlog: {}", + summarize(&reports) + ); + drop(guard); + let _ = std::fs::remove_dir_all(&root); +} diff --git a/crates/bugsee/tests/storage_fallback.rs b/crates/bugsee/tests/storage_fallback.rs index ba82295..32128d8 100644 --- a/crates/bugsee/tests/storage_fallback.rs +++ b/crates/bugsee/tests/storage_fallback.rs @@ -61,7 +61,9 @@ fn launch_degrades_through_the_resolution_order() { assert!(!status.may_be_ephemeral()); drop(guard); - // 2. An unusable explicit directory falls through to the environment variable. + // 2. A directory the host asked for is FINAL: unusable means memory mode, not + // some other location on disk (here the environment variable is writable + // and must stay untouched). let from_env = s.join("from-env"); std::env::set_var(DATA_DIR_ENV, &from_env); let guard = Bugsee::launch_with( @@ -71,20 +73,27 @@ fn launch_degrades_through_the_resolution_order() { ) .expect("launch"); let status = Bugsee::storage_status().expect("status"); + assert_eq!(status.mode, StorageMode::Memory); + assert!(status.explicit_rejected.is_some()); + assert!(!from_env.exists(), "must not fall back to another location"); + drop(guard); + + // 3. Without an explicit directory the environment variable is used. + let guard = + Bugsee::launch_with(LaunchOptions::new("TOKEN_A").with_transport(transport.clone())) + .expect("launch"); + let status = Bugsee::storage_status().expect("status"); assert_eq!(status.source, DataDirSource::Environment); assert_eq!(status.data_dir.as_deref(), Some(from_env.as_path())); - assert_eq!(status.skipped.len(), 1); drop(guard); std::env::remove_var(DATA_DIR_ENV); - // 3. Nothing usable anywhere: memory mode, and reports still go out. + // 4. Nothing usable anywhere (no home, temp dir unusable): memory mode, and + // reports still go out. let transport = Arc::new(MockTransport::default()); - let guard = Bugsee::launch_with( - LaunchOptions::new("TOKEN_A") - .data_dir(&blocked) - .with_transport(transport.clone()), - ) - .expect("an unwritable environment must not fail launch"); + let guard = + Bugsee::launch_with(LaunchOptions::new("TOKEN_A").with_transport(transport.clone())) + .expect("an unwritable environment must not fail launch"); let status = Bugsee::storage_status().expect("status"); assert_eq!(status.mode, StorageMode::Memory); assert_eq!(status.source, DataDirSource::None); From 01e337cf08ab960f57e0fcb331d17f39dbad3abf Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 14:34:47 +0500 Subject: [PATCH 03/15] fix: second review round for data-directory fallback (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fork child that falls back to memory clears the panic snapshot path and updates the published storage resolution (dir None, source None) - legacy location is adopted only if /bugsee and the token dir are real directories owned by us and not our own directory (canonicalised compare) - legacy queue is included in flush, gc'd for orphans, and retired from polling once recovery has run and its queue is empty - do not resolve temp_dir() unless a default location is actually needed - tests: fork memory-fallback, symlinked/aliased legacy dir, end-to-end legacy crashed session + queued report delivered once and cleaned up πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/runtime.rs | 38 +++++- crates/bugsee-core/tests/storage_failures.rs | 43 +++++-- crates/bugsee/src/api.rs | 20 ++- crates/bugsee/src/options.rs | 121 +++++++++++++++---- crates/bugsee/tests/fork_safety.rs | 62 ++++++++++ 5 files changed, 240 insertions(+), 44 deletions(-) diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 21b16a5..232e8f8 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -834,6 +834,10 @@ fn spawn_pipeline( let uploader_shared = Arc::clone(shared); let uploader_data_dir = data_dir.map(Path::to_path_buf); let uploader_legacy = legacy_data_dir.clone(); + // Set once the worker has finished recovering the legacy directory, so the + // uploader does not retire it while recovery is still about to fill its queue. + let legacy_ready = Arc::new(AtomicBool::new(legacy_data_dir.is_none())); + let uploader_ready = Arc::clone(&legacy_ready); let uploader = std::thread::Builder::new() .name("bugsee-uploader".into()) .spawn(move || { @@ -842,6 +846,7 @@ fn spawn_pipeline( uploader_shared, uploader_data_dir, uploader_legacy, + uploader_ready, backoff_base, fresh_launch, ) @@ -878,9 +883,11 @@ fn spawn_pipeline( // Best effort, once: sessions a previous default location left // behind. No generation of ours lives there, so none is skipped. if let Some(dir) = &legacy_data_dir { + queue::gc_orphans(dir); run_recovery(&worker_shared, dir, u64::MAX); } })); + legacy_ready.store(true, Ordering::SeqCst); let _ = worker_upload_tx.send(UploadMsg::Wake); worker_loop( rx, @@ -1348,11 +1355,12 @@ fn uploader_loop( shared: Arc, data_dir: Option, legacy_data_dir: Option, + legacy_ready: Arc, backoff_base: Duration, fresh_launch: bool, ) { let poll = Duration::from_secs(5); - let legacy = legacy_data_dir; + let mut legacy = legacy_data_dir; // A fresh launch retries any queued report immediately (force), ignoring // backoff scheduled by the prior session. A fork child skips this: the // parent's uploader is already working the shared queue. Guarded so a @@ -1383,7 +1391,12 @@ fn uploader_loop( } Ok(UploadMsg::Drain { deadline, ack }) => { let empty = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_until_empty(&shared, data_dir.as_deref(), backoff_base, deadline) + let dirs: Vec<&Path> = data_dir + .as_deref() + .into_iter() + .chain(legacy.as_deref()) + .collect(); + drain_until_empty(&shared, &dirs, backoff_base, deadline) })) .unwrap_or(false); let _ = ack.send(empty); @@ -1405,6 +1418,21 @@ fn uploader_loop( } Err(RecvTimeoutError::Disconnected) => break, } + retire_legacy(&mut legacy, &legacy_ready); + } +} + +/// Stop polling the legacy directory once recovery has run (it runs once per +/// launch, so nothing new can appear) and its queue is empty. +fn retire_legacy(legacy: &mut Option, ready: &AtomicBool) { + if !ready.load(Ordering::SeqCst) { + return; + } + if legacy + .as_deref() + .is_some_and(|dir| queue::list_pending(dir).is_empty()) + { + *legacy = None; } } @@ -1672,7 +1700,7 @@ fn backoff_delay(retry: u32, base: Duration) -> i64 { /// whether the queue ended up empty. fn drain_until_empty( shared: &Shared, - data_dir: Option<&Path>, + dirs: &[&Path], backoff_base: Duration, deadline: Duration, ) -> bool { @@ -1680,7 +1708,7 @@ fn drain_until_empty( loop { let now = epoch_ms(); drain_memory(shared, backoff_base, true, false, Some(start + deadline)); - if let Some(data_dir) = data_dir { + for &data_dir in dirs { for report in queue::list_pending(data_dir) { let Some(_claim) = queue::try_claim(&report) else { continue; // a peer is delivering it; the loop below waits it out @@ -1704,7 +1732,7 @@ fn drain_until_empty( } } } - let disk_empty = data_dir.is_none_or(|d| queue::list_pending(d).is_empty()); + let disk_empty = dirs.iter().all(|d| queue::list_pending(d).is_empty()); if disk_empty && shared.mem_queue.lock().is_empty() { return true; } diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index eb021de..d8ab57a 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -209,13 +209,16 @@ fn queue_write_failure_holds_the_report_in_memory() { } #[test] -fn legacy_directory_leftovers_are_delivered_once() { +fn legacy_directory_leftovers_are_delivered_once_and_cleaned_up() { + use bugsee_core::capture::record; use bugsee_core::model::report::TimeWindow; use bugsee_core::reporting::assemble_docs; let dir = TempDir::new(); let legacy = dir.path.join("legacy"); let data_dir = dir.path.join("data"); + + // A report an old build queued but never delivered... let meta = manual_upload_meta(); let env = bugsee_core::model::environment::Environment::detect_with( "t", @@ -226,23 +229,41 @@ fn legacy_directory_leftovers_are_delivered_once() { let stranded = assemble_docs(Vec::new(), window, &meta, &env, None, &[], "T", 1).unwrap(); bugsee_core::queue::enqueue(&legacy, &stranded).unwrap(); + // ...and a session of that build that died (marker left behind, owner gone). + let part = legacy.join("parts").join("1").join("0"); + std::fs::create_dir_all(&part).unwrap(); + let mut bytes = Vec::new(); + let payload = br#"{"timestamp":100,"level":1,"source":2,"message":"legacy crash line"}"#; + record::frame(100, payload, &mut bytes); + std::fs::write(part.join("log.part"), &bytes).unwrap(); + std::fs::create_dir_all(legacy.join("sessions")).unwrap(); + std::fs::write(legacy.join("sessions").join("1.alive"), "0").unwrap(); + std::fs::write(legacy.join("gen"), "1").unwrap(); + let transport = Arc::new(MockTransport::default()); let mut config = RecorderConfig::new(&data_dir, "T"); config.legacy_data_dir = Some(legacy.clone()); let recorder = Recorder::launch(config, transport.clone()).expect("launch"); + + // Recovery runs on the worker before the flush is acknowledged, and a flush + // drains the legacy queue too. assert!(recorder.flush(Duration::from_secs(10))); - // The startup drain is asynchronous; flush covers only the data dir queue. - let start = std::time::Instant::now(); - while transport.uploaded_bundles.lock().unwrap().is_empty() - && start.elapsed() < Duration::from_secs(10) - { - std::thread::sleep(Duration::from_millis(20)); - } - assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 1); + assert_eq!( + transport.uploaded_bundles.lock().unwrap().len(), + 2, + "the queued report and the recovered crash" + ); + assert!(bugsee_core::queue::list_pending(&legacy).is_empty()); assert!( - bugsee_core::queue::list_pending(&legacy).is_empty(), - "a delivered legacy report must be removed" + !legacy.join("sessions").join("1.alive").exists(), + "the recovered session's marker must be gone" ); + assert!(!legacy.join("parts").join("1").exists()); + + // Nothing is delivered a second time, however long we wait. + assert!(recorder.flush(Duration::from_secs(10))); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 2); } /// Blocks every delivery until released, so the backlog cannot drain. diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 54caa8e..a693e7f 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -96,7 +96,9 @@ impl Bugsee { .or_else(|| (!skipped.is_empty()).then(|| skipped.join("; "))) .unwrap_or_else(|| "no usable data directory".to_string()) }); - let data_dir = dir.unwrap_or_else(|| std::env::temp_dir().join("bugsee")); + // Memory mode never touches the path; an empty one avoids resolving a + // temp directory just to ignore it. + let data_dir = dir.unwrap_or_default(); if memory_only.is_none() && !skipped.is_empty() { static WARNED: AtomicBool = AtomicBool::new(false); bugsee_core::util::warn_once( @@ -586,9 +588,23 @@ fn revive_after_fork(epoch: u64) { #[cfg(feature = "panic")] if recorder.storage_mode() == StorageMode::Disk { bugsee_panic::set_snapshot_path(recorder.panic_info_path()); + } else { + // The inherited path points into the PARENT's live session; a panic here + // must not write its snapshot there. + bugsee_panic::clear_snapshot_path(); } - *RECORDER.lock() = Some(recorder); + // Lock order RECORDER -> RESOLUTION, as in `launch_with`/`storage_status`. + { + let mut slot = RECORDER.lock(); + if recorder.storage_mode() == StorageMode::Memory { + if let Some(resolved) = RESOLUTION.lock().as_mut() { + resolved.dir = None; + resolved.source = DataDirSource::None; + } + } + *slot = Some(recorder); + } *failures = 0; SEEN_FORK_EPOCH.store(epoch, Ordering::SeqCst); } diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index 757d084..e4de340 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -238,7 +238,7 @@ impl LaunchOptions { self.data_dir.as_deref(), &self.app_token, &|name| std::env::var_os(name), - &std::env::temp_dir(), + &std::env::temp_dir, ) } } @@ -304,7 +304,7 @@ pub(crate) fn resolve_data_dir( explicit: Option<&Path>, app_token: &str, env: &dyn Fn(&str) -> Option, - temp: &Path, + temp: &dyn Fn() -> PathBuf, ) -> DataDirResolution { let token = short_token(app_token); @@ -334,6 +334,8 @@ pub(crate) fn resolve_data_dir( }; } + // Only now: an explicit directory never needs the temp location. + let temp = temp(); let mut candidates: Vec<(DataDirSource, PathBuf, bool)> = Vec::new(); if let Some(base) = platform_data_base(env) { candidates.push(( @@ -363,8 +365,8 @@ pub(crate) fn resolve_data_dir( let legacy = temp.join("bugsee").join(&token); let legacy_dir = found .as_ref() - .filter(|(_, dir)| *dir != legacy) - .and_then(|_| trusted_existing_dir(&legacy).then_some(legacy)); + .filter(|(_, dir)| legacy_is_adoptable(&legacy, dir)) + .map(|_| legacy); match found { Some((source, dir)) => DataDirResolution { dir: Some(dir), @@ -485,6 +487,24 @@ fn owned_by_us(_meta: &std::fs::Metadata) -> bool { true } +/// Whether `legacy` (`/bugsee/`) may be recovered from. Both it AND +/// `/bugsee` must be real directories owned by us: a symlink at either +/// would let someone else's tree, or our own live data directory, be processed +/// (recovering our live session as a crash). Paths are compared canonicalised, +/// not as strings, so another spelling of our own directory is rejected too. +fn legacy_is_adoptable(legacy: &Path, chosen: &Path) -> bool { + let Some(parent) = legacy.parent() else { + return false; + }; + if !trusted_existing_dir(parent) || !trusted_existing_dir(legacy) { + return false; + } + match (legacy.canonicalize(), chosen.canonicalize()) { + (Ok(a), Ok(b)) => a != b, + _ => false, + } +} + /// Prove a file can be created in `dir`. `create_new` with an unpredictable name /// refuses to follow or reuse anything pre-planted at that path. fn probe_writable(dir: &Path) -> std::io::Result<()> { @@ -601,7 +621,7 @@ mod tests { let s = scratch(); let explicit = s.join("explicit"); let env = [(DATA_DIR_ENV, s.join("from-env").into_os_string())]; - let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&env), &s.join("tmp")); + let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&env), &|| s.join("tmp")); assert_eq!(r.source, DataDirSource::Explicit); assert_eq!(r.dir.as_deref(), Some(explicit.as_path())); assert!(r.skipped.is_empty() && r.explicit_rejected.is_none()); @@ -615,12 +635,9 @@ mod tests { let env = [(DATA_DIR_ENV, good_env.clone().into_os_string())]; let mut full_env = env.to_vec(); full_env.extend(home_env(&s.join("home"))); - let r = resolve_data_dir( - Some(&blocked(&s, "f1")), - "tok", - &env_of(&full_env), - &s.join("tmp"), - ); + let r = resolve_data_dir(Some(&blocked(&s, "f1")), "tok", &env_of(&full_env), &|| { + s.join("tmp") + }); assert!(r.dir.is_none()); assert_eq!(r.source, DataDirSource::None); assert!(r.explicit_rejected.is_some()); @@ -637,7 +654,7 @@ mod tests { let s = scratch(); let mut env = vec![(DATA_DIR_ENV, blocked(&s, "f2").into_os_string())]; env.extend(home_env(&s.join("home"))); - let r = resolve_data_dir(None, "tok", &env_of(&env), &s.join("tmp")); + let r = resolve_data_dir(None, "tok", &env_of(&env), &|| s.join("tmp")); assert!(r.dir.is_none() && r.explicit_rejected.is_some()); assert!(!s.join("home").exists() && !s.join("tmp").exists()); let _ = std::fs::remove_dir_all(&s); @@ -648,7 +665,7 @@ mod tests { let s = scratch(); let good = s.join("from-env"); let env = [(DATA_DIR_ENV, good.clone().into_os_string())]; - let r = resolve_data_dir(None, "tok", &env_of(&env), &s.join("tmp")); + let r = resolve_data_dir(None, "tok", &env_of(&env), &|| s.join("tmp")); assert_eq!(r.source, DataDirSource::Environment); assert_eq!(r.dir, Some(good)); let _ = std::fs::remove_dir_all(&s); @@ -658,7 +675,7 @@ mod tests { fn platform_default_is_used_when_home_is_set() { let s = scratch(); let home = s.join("home"); - let r = resolve_data_dir(None, "tok-1", &env_of(&home_env(&home)), &s.join("tmp")); + let r = resolve_data_dir(None, "tok-1", &env_of(&home_env(&home)), &|| s.join("tmp")); assert_eq!(r.source, DataDirSource::PlatformDefault); assert!(r.dir.unwrap().starts_with(&home)); let _ = std::fs::remove_dir_all(&s); @@ -668,7 +685,7 @@ mod tests { fn no_home_falls_back_to_a_private_temp_directory() { let s = scratch(); let tmp = s.join("tmp"); - let r = resolve_data_dir(None, "tok-1", &env_of(&[]), &tmp); + let r = resolve_data_dir(None, "tok-1", &env_of(&[]), &|| tmp.clone()); assert_eq!(r.source, DataDirSource::TempFallback); let dir = r.dir.unwrap(); assert_eq!(dir, tmp.join(private_temp_name()).join("tok1")); @@ -692,7 +709,7 @@ mod tests { std::fs::create_dir_all(&tmp).unwrap(); std::fs::create_dir_all(&elsewhere).unwrap(); std::os::unix::fs::symlink(&elsewhere, tmp.join(private_temp_name())).unwrap(); - let r = resolve_data_dir(None, "tok", &env_of(&[]), &tmp); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); assert!(r.dir.is_none(), "must not follow the symlink"); assert_eq!(r.skipped.len(), 1); assert!( @@ -706,12 +723,9 @@ mod tests { fn nothing_writable_resolves_to_none_and_lists_every_default_attempt() { let s = scratch(); let blocked_home = blocked(&s, "f3"); - let r = resolve_data_dir( - None, - "tok", - &env_of(&home_env(&blocked_home)), - &blocked(&s, "f4"), - ); + let r = resolve_data_dir(None, "tok", &env_of(&home_env(&blocked_home)), &|| { + blocked(&s, "f4") + }); assert_eq!(r.source, DataDirSource::None); assert!(r.dir.is_none()); assert_eq!(r.skipped.len(), 2); @@ -722,7 +736,7 @@ mod tests { fn an_empty_environment_variable_is_ignored() { let s = scratch(); let env = [(DATA_DIR_ENV, OsString::new())]; - let r = resolve_data_dir(None, "tok", &env_of(&env), &s.join("tmp")); + let r = resolve_data_dir(None, "tok", &env_of(&env), &|| s.join("tmp")); assert_eq!(r.source, DataDirSource::TempFallback); let _ = std::fs::remove_dir_all(&s); } @@ -733,18 +747,73 @@ mod tests { fn the_previous_default_location_is_offered_for_recovery_only_if_it_exists() { let s = scratch(); let tmp = s.join("tmp"); - let r = resolve_data_dir(None, "tok", &env_of(&[]), &tmp); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); assert!(r.legacy_dir.is_none(), "nothing to adopt yet"); let legacy = tmp.join("bugsee").join("tok"); std::fs::create_dir_all(&legacy).unwrap(); - let r = resolve_data_dir(None, "tok", &env_of(&[]), &tmp); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); assert_eq!(r.legacy_dir, Some(legacy.clone())); // A host that chose its directory never used the legacy one. let explicit = s.join("explicit"); - let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&[]), &tmp); + let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&[]), &|| tmp.clone()); + assert!(r.legacy_dir.is_none()); + let _ = std::fs::remove_dir_all(&s); + } + #[cfg(unix)] + #[test] + fn a_symlinked_legacy_location_is_never_adopted() { + let s = scratch(); + let tmp = s.join("tmp"); + let other = s.join("other"); + std::fs::create_dir_all(other.join("tok")).unwrap(); + std::fs::create_dir_all(&tmp).unwrap(); + // `/bugsee` itself is a symlink to a tree that has our token dir. + std::os::unix::fs::symlink(&other, tmp.join("bugsee")).unwrap(); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); + assert!(r.dir.is_some() && r.legacy_dir.is_none()); + + // The token directory itself being a symlink is refused as well. + std::fs::remove_file(tmp.join("bugsee")).unwrap(); + std::fs::create_dir_all(tmp.join("bugsee")).unwrap(); + std::os::unix::fs::symlink(other.join("tok"), tmp.join("bugsee").join("tok")).unwrap(); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); assert!(r.legacy_dir.is_none()); let _ = std::fs::remove_dir_all(&s); } + + // The platform default must be `/bugsee/` to collide. + #[cfg(all(unix, not(target_vendor = "apple")))] + #[test] + fn the_legacy_location_is_skipped_when_it_is_our_own_directory_by_another_spelling() { + let s = scratch(); + let tmp = s.join("tmp"); + let legacy = tmp.join("bugsee").join("tok"); + std::fs::create_dir_all(&legacy).unwrap(); + // The platform default resolves to the same directory, spelled with `..`. + let base = tmp.join("elsewhere").join(".."); + std::fs::create_dir_all(tmp.join("elsewhere")).unwrap(); + let env = [ + ("HOME", base.clone().into_os_string()), + ("XDG_DATA_HOME", base.into_os_string()), + ]; + let r = resolve_data_dir(None, "tok", &env_of(&env), &|| tmp.clone()); + assert_eq!(r.source, DataDirSource::PlatformDefault); + assert!( + r.legacy_dir.is_none(), + "our live data directory must not be recovered as a legacy one" + ); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn an_explicit_directory_never_asks_for_the_temp_location() { + let s = scratch(); + let r = resolve_data_dir(Some(&s.join("explicit")), "tok", &env_of(&[]), &|| { + panic!("temp_dir must not be resolved eagerly") + }); + assert_eq!(r.source, DataDirSource::Explicit); + let _ = std::fs::remove_dir_all(&s); + } } diff --git a/crates/bugsee/tests/fork_safety.rs b/crates/bugsee/tests/fork_safety.rs index 64365f0..a27b0f5 100644 --- a/crates/bugsee/tests/fork_safety.rs +++ b/crates/bugsee/tests/fork_safety.rs @@ -519,6 +519,68 @@ fn a_panic_in_the_child_before_touching_the_sdk_does_not_write_into_the_parents_ let _ = std::fs::remove_dir_all(&out); } +/// A child that cannot open its own session falls back to memory. It must then +/// stop pointing the panic observer into the PARENT's live session, and its +/// storage status must say it has no directory. +#[test] +fn a_child_that_falls_back_to_memory_does_not_write_into_the_parents_session() { + let _serial = serial(); + let data = scratch("data"); + let out = scratch("out"); + let guard = Bugsee::launch_with( + LaunchOptions::new("APP_TOKEN") + .data_dir(&data) + .native_crash_capture(false) + .with_transport(FileTransport::new(&out)), + ) + .expect("launch"); + std::thread::sleep(Duration::from_millis(300)); + + // Make a NEW session impossible (the child's `Session::begin` needs + // `sessions/` to be a directory) while the parent's parts stay writable. + let sessions = data.join("sessions"); + let parked = data.join("sessions.parked"); + std::fs::rename(&sessions, &parked).unwrap(); + std::fs::write(&sessions, b"not a directory").unwrap(); + + let child = fork_child(|| { + Bugsee::capture_message(LogLevel::Warning, "child-starting"); // revives + let Some(status) = Bugsee::storage_status() else { + return 2; + }; + if status.mode != bugsee::StorageMode::Memory || status.data_dir.is_some() { + return 3; + } + let _ = std::thread::spawn(|| panic!("child-panic-in-memory-mode")).join(); + 0 + }); + assert!(exited_with( + wait_with_deadline(child, Duration::from_secs(30)), + 0 + )); + + let stray: Vec<_> = std::fs::read_dir(data.join("parts")) + .into_iter() + .flatten() + .flatten() + .map(|d| d.path().join("panic.info")) + .filter(|p| p.exists()) + .collect(); + assert!( + stray.is_empty(), + "the child wrote its panic into its parent's session: {stray:?}" + ); + // The parent is untouched. + let parent = Bugsee::storage_status().expect("status"); + assert_eq!(parent.mode, bugsee::StorageMode::Disk); + + let _ = std::fs::remove_file(&sessions); + let _ = std::fs::rename(&parked, &sessions); + drop(guard); + let _ = std::fs::remove_dir_all(&data); + let _ = std::fs::remove_dir_all(&out); +} + /// Fails transiently until `allow` exists, then behaves like [`FileTransport`]. struct AllowTransport { allow: PathBuf, From 11226fbf1e3ae0179c3eac338a067dcb4358e135 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:26:53 +0500 Subject: [PATCH 04/15] fix(core): exactly-once recovery and queue draining across processes sharing a data dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several processes on one data directory could double-deliver (or lose) reports: - A session was recovered twice: a peer that listed it before another finished claimed it after the winner had discarded it, producing a phantom report. The marker is now re-checked after the claim is won. - Stale-claim takeover was remove-then-create, so two takers could each remove the other's fresh claim and both hold it. Takeover is now serialised by a guard file and re-validated under it; release only removes the claim's own content. - Claims and session markers identified owners by bare pid: a recycled pid (or a restarted container's pid 1) kept a dead owner's work blocked forever. Owners now carry a process start time (Linux /proc, macOS proc_pidinfo, Windows GetProcessTimes), with a 30 minute lease as a backstop. - On Windows process liveness always answered "dead", so live peers' claims and markers were taken over. Implemented via OpenProcess/GetExitCodeProcess. - queue::gc_orphans at a peer's launch reaped another process's in-flight enqueue (sidecars before the bundle rename), stranding the bundle and losing the report. Only files untouched for 60 s are reaped now. - Leftover claims of discarded sessions are swept. Adds tests/multiprocess.rs (real child processes racing; SIGKILLed holder) plus unit tests for claims, process identity and queue gc. Closes #14 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 4 +- crates/bugsee-core/src/claim.rs | 297 +++++++++++++++++++---- crates/bugsee-core/src/lib.rs | 1 + crates/bugsee-core/src/process.rs | 232 ++++++++++++++++++ crates/bugsee-core/src/queue.rs | 49 +++- crates/bugsee-core/src/recovery.rs | 59 +++-- crates/bugsee-core/src/session.rs | 4 +- crates/bugsee-core/tests/multiprocess.rs | 245 +++++++++++++++++++ 8 files changed, 811 insertions(+), 80 deletions(-) create mode 100644 crates/bugsee-core/src/process.rs create mode 100644 crates/bugsee-core/tests/multiprocess.rs diff --git a/DESIGN.md b/DESIGN.md index f2f03f6..90bc74b 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -197,7 +197,9 @@ Crash recovery needs the data directory to **outlive the process**: on a contain **Release note, default location moved.** Earlier builds defaulted to `/bugsee/`; the default is now the platform location (3) or the private temp fallback (4). When no directory was requested and the old location exists, is a real directory and is owned by the current user, the launch recovers its crashed sessions and delivers its queue once, best effort (delivery is asynchronous and `flush` does not wait for it). It is not claimed to be lossless: a legacy session whose process is still alive is skipped, and a legacy directory that fails the ownership check is left alone. Hosts that set `data_dir` were never using it. -**Unsafe.** The facade makes two `geteuid()` calls (unix) to name and vet the private temp directory, alongside core's single `kill(pid, 0)`. +**Unsafe.** The facade makes two `geteuid()` calls (unix) to name and vet the private temp directory. Core's process probes (`process.rs`) are `kill(pid, 0)` and `proc_pidinfo` (macOS), plus `OpenProcess`/`GetExitCodeProcess`/`GetProcessTimes` on Windows. + +**Several processes on one directory (#14).** Every pending item is owned by exactly one process at a time: a crashed session by `sessions/.claim`, a queued report by `queue/.claim`. A claim is a file created with `O_EXCL`/`CREATE_NEW` holding `pid start-time nonce`; it is released on drop and, if the holder dies, **taken over** once stale. Stale means: the owner pid is gone, or alive with a different start time (pid reuse; a container's restarted app is pid 1 again), or the file has no owner and is older than 5 s, or it is older than a 30 minute lease (wedged holder, or a platform with no start time). Takeover is serialised by a short-lived `.takeover` guard file (re-checked under the guard), so two takers cannot both remove the other's fresh claim; a guard older than 10 s is abandoned. No OS file lock is used: it would be tied to the open file description, which `fork` duplicates. After winning a session claim the marker is re-checked, because a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s, since a peer's in-flight `enqueue` looks like an orphan (reaping it lost the report). The `.alive` marker also records the start time, so a recycled pid no longer keeps a dead session unrecovered. Delivery is at-least-once: a holder that dies after uploading but before removing the item makes its successor upload it again. Tests: `bugsee-core/tests/multiprocess.rs` (real child processes; racing launchers, SIGKILLed holder) and the unit tests in `claim.rs`, `process.rs`, `queue.rs`. Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, a memory-mode case in `bugsee/tests/fork_safety.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised, since mounting one needs root; failures are injected by blocking path components and removing the directory mid-run. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index 7e01144..df40239 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -10,75 +10,178 @@ //! and its workers above all β€” and the work items in it (a crashed session to //! recover, a queued report to upload) must each be handled by exactly one of //! them, or the backend receives duplicates. A claim is a small file created -//! with `O_EXCL` that records its owner's pid. +//! with `O_EXCL` (`CREATE_NEW` on Windows) that records its owner as +//! `" "`. //! -//! A claim must never block the work forever if its owner dies, so a claim whose -//! owner is no longer alive is *stale* and is taken over. (An OS file lock would -//! be released on death automatically, but it is per-open-file-description, not -//! per-process, which makes it unreliable across `fork`.) +//! A claim must never block the work forever if its owner dies, so a stale claim +//! is taken over. It is stale when +//! +//! * its owner process is gone, or the pid now belongs to a *different* process +//! (the recorded start time no longer matches: pid reuse, or a container's +//! restarted app being pid 1 again), +//! * it has no readable owner and is older than [`UNOWNED_GRACE`] (its creator +//! died between creating the file and writing it), or +//! * it is older than [`MAX_LEASE`], a backstop for a holder that is alive but +//! wedged, or on a platform that cannot tell processes apart. +//! +//! Taking over is serialised by a short-lived guard file, so two processes that +//! both find the same claim stale cannot each remove the other's fresh claim and +//! both end up holding it. (An OS file lock would release on death automatically, +//! but it is per-open-file-description, not per-process, which makes it unreliable +//! across `fork`.) +//! +//! None of this runs on the crash path; it allocates freely. use std::io::Write; use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; +use crate::process; + /// A claim that has no readable owner is given this long before it is treated as -/// stale: its creator writes the pid a moment after creating the file. +/// stale: its creator writes the owner a moment after creating the file. const UNOWNED_GRACE: Duration = Duration::from_secs(5); +/// No claim is honoured for longer than this, whatever its owner's state. Work +/// items are short (one upload, one recovery), so a claim this old is wedged. +const MAX_LEASE: Duration = Duration::from_secs(30 * 60); + +/// The takeover guard is held for a few syscalls; one older than this was left +/// by a process that died inside the critical section. +const GUARD_TTL: Duration = Duration::from_secs(10); + /// Holds a claim; dropping it releases the claim. #[derive(Debug)] pub struct Claim { path: PathBuf, + /// Exactly what this claim wrote, so release never removes a successor's claim. + content: String, } impl Drop for Claim { fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); + if std::fs::read_to_string(&self.path).is_ok_and(|c| c == self.content) { + let _ = std::fs::remove_file(&self.path); + } } } /// Try to claim `path`. `None` if another live process (or another thread of this -/// one) holds it. +/// one) holds it, or if it cannot be claimed. pub fn try_claim(path: &Path) -> Option { - for _ in 0..2 { - match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(path) - { - Ok(mut file) => { - let _ = write!(file, "{}", std::process::id()); - return Some(Claim { - path: path.to_path_buf(), - }); - } - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { - if !is_stale(path) { - return None; - } - // Take over a claim whose owner is gone, then retry the create. - // Losing the race to another taker just means `create_new` fails - // again and we report the claim as held. - let _ = std::fs::remove_file(path); + let content = owner_record(); + match create(path, &content) { + Create::Done => Some(Claim { + path: path.to_path_buf(), + content, + }), + Create::Exists => take_over(path, content), + // Directory missing / unwritable: nothing can be claimed, and the + // caller's own I/O will fail more usefully than we can here. + Create::Failed => None, + } +} + +enum Create { + Done, + Exists, + Failed, +} + +/// Atomically create `path` holding `content` (one `write`, so the owner is +/// visible as soon as the file is). +fn create(path: &Path, content: &str) -> Create { + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + { + Ok(mut file) => { + let _ = file.write_all(content.as_bytes()); + Create::Done + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Create::Exists, + Err(_) => Create::Failed, + } +} + +/// Replace a stale claim with ours. Only one process at a time may do this: the +/// guard file excludes a second taker between our staleness check and our create, +/// which is exactly the window in which it could otherwise remove the claim we +/// just made. +fn take_over(path: &Path, content: String) -> Option { + let observed = std::fs::read_to_string(path).ok()?; + if !is_stale(path, &observed) { + return None; + } + let guard_path = sibling(path, "takeover"); + match create(&guard_path, "") { + Create::Done => {} + Create::Exists => { + // Held by a peer mid-takeover (it finishes in microseconds, and the + // work is retried on the next drain), or left by one that died. + if age(&guard_path).is_some_and(|a| a > GUARD_TTL) { + let _ = std::fs::remove_file(&guard_path); } - // Directory missing / unwritable: nothing can be claimed, and the - // caller's own I/O will fail more usefully than we can here. - Err(_) => return None, + return None; } + Create::Failed => return None, + } + let _guard = Guard(guard_path); + // Re-check under the guard: a peer may have completed its own takeover since + // we looked, and what is there now is a live claim, not the stale one. + if std::fs::read_to_string(path).ok().as_deref() != Some(observed.as_str()) { + return None; + } + let _ = std::fs::remove_file(path); + match create(path, &content) { + Create::Done => Some(Claim { + path: path.to_path_buf(), + content, + }), + // A fresh creator slipped in between our remove and create: it holds it. + Create::Exists | Create::Failed => None, } - None } -fn is_stale(path: &Path) -> bool { - let owner = std::fs::read_to_string(path).unwrap_or_default(); - match owner.trim().parse::() { - // Our own pid is a live owner: another thread of this process holds it. - Ok(pid) => pid != std::process::id() && !crate::recovery::process_is_alive(pid), - Err(_) => std::fs::metadata(path) - .and_then(|m| m.modified()) - .ok() - .and_then(|t| SystemTime::now().duration_since(t).ok()) - .is_some_and(|age| age > UNOWNED_GRACE), +struct Guard(PathBuf); +impl Drop for Guard { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +fn sibling(path: &Path, ext: &str) -> PathBuf { + let mut s = path.as_os_str().to_os_string(); + s.push("."); + s.push(ext); + PathBuf::from(s) +} + +/// `" "` for this process; the nonce makes every claim's +/// content unique, so "is this still my claim" is a content comparison. +fn owner_record() -> String { + let (pid, start) = + process::parse_token(&process::self_token()).unwrap_or((std::process::id(), None)); + let start = start.map_or_else(|| "-".to_string(), |s| s.to_string()); + format!("{pid} {start} {}", crate::util::random_hex(8)) +} + +fn age(path: &Path) -> Option { + let modified = std::fs::metadata(path).and_then(|m| m.modified()).ok()?; + SystemTime::now().duration_since(modified).ok() +} + +fn is_stale(path: &Path, content: &str) -> bool { + let age = age(path); + if age.is_some_and(|a| a > MAX_LEASE) { + return true; + } + match process::parse_token(content) { + // A live owner with the recorded identity (including another thread of + // this very process) holds it. + Some((pid, start)) => !process::owner_is_alive(pid, start), + None => age.is_some_and(|a| a > UNOWNED_GRACE), } } @@ -92,6 +195,16 @@ mod tests { p } + fn backdate(path: &Path, by: Duration) { + let f = std::fs::OpenOptions::new().write(true).open(path).unwrap(); + f.set_modified(SystemTime::now() - by).unwrap(); + } + + /// This process's own live owner record, as a peer would have written it. + fn live_owner() -> String { + format!("{} nonce", process::self_token()) + } + #[test] fn a_claim_is_exclusive_and_released_on_drop() { let dir = tmp(); @@ -109,25 +222,115 @@ mod tests { // A pid that cannot be alive: far above any real pid range. std::fs::write(&path, "4294967290").unwrap(); assert!(try_claim(&path).is_some(), "stale claim must be reclaimed"); + assert!(!sibling(&path, "takeover").exists(), "guard released"); } #[test] fn a_claim_held_by_a_live_process_is_respected() { let dir = tmp(); let path = dir.join("x.claim"); - // pid 1 is always alive on unix; on other platforms liveness is "dead". - std::fs::write(&path, "1").unwrap(); + std::fs::write(&path, live_owner()).unwrap(); + assert!(try_claim(&path).is_none()); + // Old-format owner (pid only), pid 1: alive on unix. #[cfg(unix)] + { + std::fs::write(&path, "1").unwrap(); + assert!(try_claim(&path).is_none()); + } + } + + #[test] + fn a_recycled_pid_does_not_keep_a_dead_owners_claim_alive() { + let Some(start) = process::start_time(std::process::id()) else { + return; // platform cannot tell processes apart + }; + let dir = tmp(); + let path = dir.join("x.claim"); + // Our own (live) pid, but with a start time that is not ours: the claim + // was written by an earlier process that had this pid. + std::fs::write( + &path, + format!("{} {} stale", std::process::id(), start.wrapping_add(1)), + ) + .unwrap(); + assert!(try_claim(&path).is_some(), "recycled pid must not block"); + } + + #[test] + fn a_freshly_created_unowned_claim_is_not_stolen_but_an_old_one_is() { + let dir = tmp(); + let path = dir.join("x.claim"); + std::fs::write(&path, "").unwrap(); // creator has not written its owner yet assert!(try_claim(&path).is_none()); - #[cfg(not(unix))] + backdate(&path, UNOWNED_GRACE + Duration::from_secs(5)); + assert!(try_claim(&path).is_some(), "creator died before writing"); + } + + #[test] + fn a_wedged_holder_cannot_hold_a_claim_past_the_lease() { + let dir = tmp(); + let path = dir.join("x.claim"); + std::fs::write(&path, live_owner()).unwrap(); + assert!(try_claim(&path).is_none()); + backdate(&path, MAX_LEASE + Duration::from_secs(60)); assert!(try_claim(&path).is_some()); } #[test] - fn a_freshly_created_unowned_claim_is_not_stolen() { + fn releasing_a_claim_never_removes_its_successors() { + let dir = tmp(); + let path = dir.join("x.claim"); + let first = try_claim(&path).unwrap(); + // First is judged wedged and replaced behind its back. + std::fs::write(&path, live_owner()).unwrap(); + drop(first); + assert!(path.exists(), "the successor's claim must survive"); + } + + #[test] + fn an_abandoned_takeover_guard_does_not_block_forever() { let dir = tmp(); let path = dir.join("x.claim"); - std::fs::write(&path, "").unwrap(); // creator has not written its pid yet + std::fs::write(&path, "4294967290").unwrap(); + let guard = sibling(&path, "takeover"); + std::fs::write(&guard, "").unwrap(); + // A taker is (apparently) mid-takeover: back off. assert!(try_claim(&path).is_none()); + // It died there long ago: clear the guard, then succeed. + backdate(&guard, GUARD_TTL + Duration::from_secs(5)); + assert!( + try_claim(&path).is_none(), + "clearing the guard is its own step" + ); + assert!(try_claim(&path).is_some()); + } + + #[test] + fn racing_takers_of_one_stale_claim_yield_exactly_one_holder() { + for _ in 0..200 { + let dir = tmp(); + let path = dir.join("x.claim"); + std::fs::write(&path, "4294967290").unwrap(); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(8)); + let handles: Vec<_> = (0..8) + .map(|_| { + let (path, barrier) = (path.clone(), barrier.clone()); + std::thread::spawn(move || { + barrier.wait(); + let claim = try_claim(&path); + // Keep the claim held until every thread has tried. + barrier.wait(); + claim.is_some() + }) + }) + .collect(); + let holders = handles + .into_iter() + .map(|h| h.join().unwrap()) + .filter(|held| *held) + .count(); + assert_eq!(holders, 1, "exactly one taker may win"); + let _ = std::fs::remove_dir_all(&dir); + } } } diff --git a/crates/bugsee-core/src/lib.rs b/crates/bugsee-core/src/lib.rs index cdbc126..e187ece 100644 --- a/crates/bugsee-core/src/lib.rs +++ b/crates/bugsee-core/src/lib.rs @@ -23,6 +23,7 @@ pub mod errors; pub mod fork; pub mod model; pub mod panic_info; +pub(crate) mod process; pub mod queue; pub mod recovery; pub mod reporting; diff --git a/crates/bugsee-core/src/process.rs b/crates/bugsee-core/src/process.rs new file mode 100644 index 0000000..c37f583 --- /dev/null +++ b/crates/bugsee-core/src/process.rs @@ -0,0 +1,232 @@ +// +// process.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! Process identity and liveness, for deciding whether the owner recorded in a +//! claim or a liveness marker is still the process that wrote it. +//! +//! A bare pid is not an identity: pids are recycled, and in a container the +//! restarted app is routinely pid 1 again. An *owner token* therefore pairs the +//! pid with the process's **start time** (an opaque, stable per-process number), +//! so a recycled pid is told apart from the original owner. Where the platform +//! offers no start time the token carries the pid alone and liveness degrades to +//! "a process with that pid exists". + +use std::sync::OnceLock; + +/// Whether a process with `pid` currently exists. +/// +/// Unix: `kill(pid, 0)`; `0` (permitted) or `EPERM` (exists, not permitted) both +/// mean alive, `ESRCH` means gone. Windows: the process can be opened and has not +/// exited. A pid-reuse false positive is closed by [`owner_is_alive`]'s start +/// time comparison, not here. +#[cfg(unix)] +pub(crate) fn is_alive(pid: u32) -> bool { + if pid == 0 { + return false; + } + // SAFETY: `kill` with signal 0 performs only an error/permission check and + // has no side effects; it is safe to call with any pid value. + let ret = unsafe { libc::kill(pid as libc::pid_t, 0) }; + ret == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) +} + +#[cfg(windows)] +pub(crate) fn is_alive(pid: u32) -> bool { + win::with_process(pid, |handle| { + let mut code = 0u32; + // SAFETY: `handle` is a valid open process handle for the closure's duration. + let ok = unsafe { win::GetExitCodeProcess(handle, &mut code) }; + ok == 0 || code == win::STILL_ACTIVE + }) + // Access denied means the process exists but belongs to someone else. + .unwrap_or_else(|denied| denied) +} + +#[cfg(not(any(unix, windows)))] +pub(crate) fn is_alive(_pid: u32) -> bool { + false +} + +/// An opaque per-process start time for `pid`, or `None` if it cannot be read +/// (platform without support, process gone, or not permitted). Only ever compared +/// for equality against a value read the same way. +#[cfg(target_os = "linux")] +pub(crate) fn start_time(pid: u32) -> Option { + let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + // `pid (comm) state ...`: comm may contain spaces and parens, so split after + // the LAST ')'. `starttime` is field 22; `state` (field 3) is index 0 here. + let rest = &stat[stat.rfind(')')? + 1..]; + rest.split_whitespace().nth(19)?.parse().ok() +} + +#[cfg(target_os = "macos")] +pub(crate) fn start_time(pid: u32) -> Option { + // SAFETY: an all-zero `proc_bsdinfo` is a valid plain-data value, and + // `proc_pidinfo` writes at most `size_of` bytes into it. + let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; + let size = std::mem::size_of::() as libc::c_int; + let n = unsafe { + libc::proc_pidinfo( + pid as libc::c_int, + libc::PROC_PIDTBSDINFO, + 0, + (&mut info as *mut libc::proc_bsdinfo).cast(), + size, + ) + }; + (n == size).then(|| info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec) +} + +#[cfg(windows)] +pub(crate) fn start_time(pid: u32) -> Option { + win::with_process(pid, |handle| { + let mut ft = [win::FileTime::default(); 4]; + // SAFETY: `handle` is valid; the four out-pointers point into `ft`. + let [creation, exit, kernel, user] = &mut ft; + let ok = unsafe { win::GetProcessTimes(handle, creation, exit, kernel, user) }; + (ok != 0).then(|| (u64::from(ft[0].high) << 32) | u64::from(ft[0].low)) + }) + .ok() + .flatten() +} + +#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] +pub(crate) fn start_time(_pid: u32) -> Option { + None +} + +/// This process's owner token, `""` or `" "`. Cached, but keyed +/// by pid: a `fork` child inherits the cache and must not reuse the parent's token. +pub(crate) fn self_token() -> String { + static CACHE: OnceLock<(u32, String)> = OnceLock::new(); + let pid = std::process::id(); + let (cached_pid, token) = CACHE.get_or_init(|| (pid, make_token(pid))); + if *cached_pid == pid { + token.clone() + } else { + make_token(pid) + } +} + +fn make_token(pid: u32) -> String { + match start_time(pid) { + Some(start) => format!("{pid} {start}"), + None => pid.to_string(), + } +} + +/// Parse an owner token (`""` or `" "`; trailing fields, such as +/// a claim's nonce, are ignored). `None` if no pid can be read. +pub(crate) fn parse_token(text: &str) -> Option<(u32, Option)> { + let mut it = text.split_whitespace(); + let pid = it.next()?.parse().ok()?; + let start = it.next().and_then(|s| s.parse().ok()); + Some((pid, start)) +} + +/// Whether the process an owner token names is still alive *and is the same +/// process that wrote the token*. With a recorded start time, a live process +/// whose start time differs is a recycled pid, i.e. the owner is gone. +pub(crate) fn owner_is_alive(pid: u32, recorded_start: Option) -> bool { + if !is_alive(pid) { + return false; + } + match (recorded_start, start_time(pid)) { + (Some(recorded), Some(now)) => recorded == now, + // Cannot compare (old-format token, or start time unreadable): the pid + // exists, so give the owner the benefit of the doubt. + _ => true, + } +} + +#[cfg(windows)] +mod win { + //! The few kernel32 calls needed, declared directly: kernel32 is always + //! linked, so this avoids a dependency for four functions. + use std::ffi::c_void; + + pub type Handle = *mut c_void; + pub const STILL_ACTIVE: u32 = 259; + const PROCESS_QUERY_LIMITED_INFORMATION: u32 = 0x1000; + const ERROR_ACCESS_DENIED: i32 = 5; + + #[repr(C)] + #[derive(Clone, Copy, Default)] + pub struct FileTime { + pub low: u32, + pub high: u32, + } + + extern "system" { + fn OpenProcess(access: u32, inherit: i32, pid: u32) -> Handle; + fn CloseHandle(handle: Handle) -> i32; + pub fn GetExitCodeProcess(handle: Handle, code: *mut u32) -> i32; + pub fn GetProcessTimes( + handle: Handle, + creation: *mut FileTime, + exit: *mut FileTime, + kernel: *mut FileTime, + user: *mut FileTime, + ) -> i32; + } + + /// Run `f` with a query handle to `pid`. `Err(true)` if the process exists but + /// access was denied, `Err(false)` if it cannot be opened (gone). + pub fn with_process(pid: u32, f: impl FnOnce(Handle) -> T) -> Result { + if pid == 0 { + return Err(false); + } + // SAFETY: plain Win32 calls; the handle is closed on every path. + unsafe { + let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); + if handle.is_null() { + let denied = + std::io::Error::last_os_error().raw_os_error() == Some(ERROR_ACCESS_DENIED); + return Err(denied); + } + let out = f(handle); + CloseHandle(handle); + Ok(out) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn own_process_is_alive_with_a_stable_start_time() { + assert!(is_alive(std::process::id())); + let a = start_time(std::process::id()); + assert_eq!(a, start_time(std::process::id())); + #[cfg(any(target_os = "linux", target_os = "macos", windows))] + assert!(a.is_some(), "start time must be readable on this platform"); + } + + #[test] + fn a_pid_with_the_wrong_start_time_is_a_recycled_pid() { + let me = std::process::id(); + if let Some(start) = start_time(me) { + assert!(owner_is_alive(me, Some(start))); + assert!(!owner_is_alive(me, Some(start.wrapping_add(1)))); + } + assert!(owner_is_alive(me, None), "no recorded start: pid alone"); + assert!(!owner_is_alive(4_294_967_290, Some(1))); + } + + #[test] + fn tokens_round_trip() { + assert_eq!(parse_token("12 345"), Some((12, Some(345)))); + assert_eq!(parse_token("12"), Some((12, None))); + assert_eq!(parse_token("12 345 deadbeef"), Some((12, Some(345)))); + assert_eq!(parse_token("junk"), None); + assert_eq!(parse_token(""), None); + let (pid, start) = parse_token(&self_token()).unwrap(); + assert_eq!(pid, std::process::id()); + assert_eq!(start, start_time(pid)); + } +} diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index 2ff3efe..11e5254 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -226,6 +226,11 @@ pub fn blacklist_add(data_dir: &Path, sigs: &[String]) { /// crash-atomic across its multiple files). Without this they accumulate /// unbounded. Call once at startup on the worker thread, before any enqueue, so /// it never races this process's own queue writes. +/// +/// A peer sharing the directory may be mid-`enqueue` right now (sidecars written, +/// bundle not yet renamed into place), and its in-flight files look exactly like +/// orphans. Reaping them would strand the bundle without its `.req`, losing the +/// report, so only files untouched for [`ORPHAN_GRACE`] are reaped. pub fn gc_orphans(data_dir: &Path) { let dir = queue_dir(data_dir); let read = match std::fs::read_dir(&dir) { @@ -245,15 +250,30 @@ pub fn gc_orphans(data_dir: &Path) { .or_else(|| name.strip_suffix(".meta")) .or_else(|| name.strip_suffix(".endpoint")) .or_else(|| name.strip_suffix(".claim")) + .or_else(|| name.strip_suffix(".claim.takeover")) .or_else(|| name.strip_suffix(".tmp")); if let Some(bundle_name) = bundle_name { - if !dir.join(bundle_name).exists() { + if !dir.join(bundle_name).exists() && untouched_for(&path, ORPHAN_GRACE) { let _ = std::fs::remove_file(&path); } } } } +/// How long an orphan-looking queue file must sit untouched before it is reaped +/// (see [`gc_orphans`]); far longer than any `enqueue` takes. +const ORPHAN_GRACE: std::time::Duration = std::time::Duration::from_secs(60); + +/// Whether `path` was last modified at least `grace` ago. Unknown or future-dated +/// times count as recent: the conservative answer never disturbs a live peer. +fn untouched_for(path: &Path, grace: std::time::Duration) -> bool { + std::fs::metadata(path) + .and_then(|m| m.modified()) + .ok() + .and_then(|t| t.elapsed().ok()) + .is_some_and(|age| age >= grace) +} + /// Whether any of `sigs` is locally blacklisted (report should be suppressed). pub fn any_blacklisted(data_dir: &Path, sigs: &[String]) -> bool { if sigs.is_empty() { @@ -262,3 +282,30 @@ pub fn any_blacklisted(data_dir: &Path, sigs: &[String]) -> bool { let set = load_blacklist(data_dir); sigs.iter().any(|s| set.contains(s)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gc_spares_a_peers_in_flight_enqueue_and_reaps_old_orphans() { + let data = std::env::temp_dir().join(format!("bugsee-qgc-{}", crate::util::random_hex(8))); + let dir = queue_dir(&data); + std::fs::create_dir_all(&dir).unwrap(); + // A peer mid-enqueue: sidecars written, bundle not yet renamed into place. + let fresh = dir.join("a.bundle.zip.req"); + std::fs::write(&fresh, "{}").unwrap(); + let old = dir.join("b.bundle.zip.req"); + std::fs::write(&old, "{}").unwrap(); + let f = std::fs::OpenOptions::new().write(true).open(&old).unwrap(); + f.set_modified(std::time::SystemTime::now() - ORPHAN_GRACE * 2) + .unwrap(); + drop(f); + + gc_orphans(&data); + + assert!(fresh.exists(), "an in-flight enqueue must not be reaped"); + assert!(!old.exists(), "an old orphan must be reaped"); + let _ = std::fs::remove_dir_all(&data); + } +} diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index ca21356..0262252 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -97,11 +97,13 @@ pub fn find_pending(data_dir: &Path, current_generation: u64) -> Vec() { + } else if let Some((pid, start)) = crate::process::parse_token(owner) { // Skip a marker still owned by a live *other* process. Our own pid is // never skipped (a live process cannot share our pid, so a marker - // bearing it is a dead prior incarnation β€” recover it). - if pid != self_pid && process_is_alive(pid) { + // bearing it is a dead prior incarnation β€” recover it). The marker + // may carry the owner's start time, which tells a recycled pid (a + // dead owner whose pid something else now uses) from the owner. + if pid != self_pid && crate::process::owner_is_alive(pid, start) { continue; } } @@ -146,30 +148,6 @@ fn marker_is_fresh(marker: &Path) -> bool { .unwrap_or(true) } -/// Whether a process with `pid` currently exists. -/// -/// Uses `kill(pid, 0)`, which sends no signal and only probes existence: `0` -/// (permitted) or `EPERM` (exists, not permitted) both mean *alive*; `ESRCH` -/// means gone. A rare pid-reuse false-positive can defer recovering a real crash -/// to a later launch β€” acceptable vs. corrupting a running peer's live session. -#[cfg(unix)] -pub(crate) fn process_is_alive(pid: u32) -> bool { - if pid == 0 { - return false; - } - // SAFETY: `kill` with signal 0 performs only an error/permission check and - // has no side effects; it is safe to call with any pid value. - let ret = unsafe { libc::kill(pid as libc::pid_t, 0) }; - ret == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) -} - -/// No portable liveness probe off-unix: treat as dead so recovery still proceeds -/// (single-process is the common case; a live shared-dir peer is a unix concern). -#[cfg(not(unix))] -pub(crate) fn process_is_alive(_pid: u32) -> bool { - false -} - /// Build the crash report for a pending session. Precedence: a *fresh* aborting /// panic (`SIGABRT` + a recent panic snapshot) correlates into one managed event; /// any other native signal yields the thin native variant; a lone panic snapshot @@ -348,13 +326,24 @@ pub fn bump_attempts(data_dir: &Path, pending: &PendingSession) -> u32 { /// Claim a pending session for recovery, so that of several processes sharing the /// data directory exactly one recovers it. The claim lives beside the liveness -/// marker and is released when the returned guard drops. +/// marker and is released when the returned guard drops. `None` if a peer holds +/// it or has already recovered the session. pub fn claim_pending(data_dir: &Path, pending: &PendingSession) -> Option { - crate::claim::try_claim( + let claim = crate::claim::try_claim( &data_dir .join("sessions") .join(format!("{}.claim", pending.generation)), - ) + )?; + // The listing that produced `pending` may be stale: a peer that held this + // claim before us can have recovered and discarded the session in between + // (its claim released, the marker gone). Recovering it now would deliver a + // phantom second report for it. The marker is removed first by `discard`, so + // its absence, re-checked while holding the claim, is conclusive. + data_dir + .join("sessions") + .join(format!("{}.alive", pending.generation)) + .exists() + .then_some(claim) } /// Reap leftover on-disk state from generations that no longer have a liveness @@ -388,6 +377,16 @@ pub fn gc_orphans(data_dir: &Path, current_generation: u64) { } } } + // A claim left by a holder that died after the session was discarded. + // Taking it (which succeeds only if it is stale) and dropping it + // removes it; a live holder's claim is left alone. + if let Some(gen_str) = name.strip_suffix(".claim") { + if let Ok(generation) = gen_str.parse::() { + if generation < current_generation && !alive(generation) { + drop(crate::claim::try_claim(&entry.path())); + } + } + } } } } diff --git a/crates/bugsee-core/src/session.rs b/crates/bugsee-core/src/session.rs index b4751b0..5a81692 100644 --- a/crates/bugsee-core/src/session.rs +++ b/crates/bugsee-core/src/session.rs @@ -46,7 +46,9 @@ impl Session { .open(&marker) { Ok(mut f) => { - let _ = write!(f, "{}", std::process::id()); + // " ": the start time lets recovery tell this + // process from a later one that reuses the pid. + let _ = write!(f, "{}", crate::process::self_token()); break candidate; } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { diff --git a/crates/bugsee-core/tests/multiprocess.rs b/crates/bugsee-core/tests/multiprocess.rs new file mode 100644 index 0000000..348f50c --- /dev/null +++ b/crates/bugsee-core/tests/multiprocess.rs @@ -0,0 +1,245 @@ +// +// multiprocess.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! Several processes sharing one data directory (a prefork server's workers, a CLI +//! run beside a daemon): every pending report is delivered exactly once, and a +//! holder that dies mid-work never blocks the others (issue #14). +//! +//! The "processes" are real: this test binary re-executes itself in a child role +//! selected by environment variables. + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use bugsee_core::transport::{Transport, TransportError}; +use bugsee_core::{queue, recovery, AssembledReport, Recorder, RecorderConfig}; + +const ROLE: &str = "BUGSEE_MP_ROLE"; +const DIR: &str = "BUGSEE_MP_DIR"; +const LOG: &str = "BUGSEE_MP_LOG"; + +/// A transport that records each upload by appending a line to a shared log file +/// (one `write` under `O_APPEND`, atomic across processes). +struct LogTransport { + log: PathBuf, + lock: Mutex<()>, +} +impl Transport for LogTransport { + fn register_session(&self, _: &str, _: &[u8]) -> Result { + Ok("t".into()) + } + fn create_issue(&self, _: &str, _: Option<&str>, _: &[u8]) -> Result { + Ok("endpoint".into()) + } + fn upload_bundle(&self, _: &str, zip: &[u8]) -> Result<(), TransportError> { + let _g = self.lock.lock().unwrap(); + // Give a racing peer a real window to double-deliver. + std::thread::sleep(Duration::from_millis(20)); + let mut f = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&self.log) + .unwrap(); + // Queued test items carry a readable name; recovered crashes are binary + // ZIPs, logged as a tag (each recovered session yields one). + let id = match std::str::from_utf8(zip) { + Ok(text) if text.starts_with("queued-") => text.to_string(), + _ => "recovered-crash".to_string(), + }; + // One `write` call: `writeln!` would split text and newline. + f.write_all(format!("{id}\n").as_bytes()).unwrap(); + Ok(()) + } +} + +/// Child entry point. A no-op in a normal test run. +#[test] +fn child_entry() { + let Ok(role) = std::env::var(ROLE) else { + return; + }; + let dir = PathBuf::from(std::env::var(DIR).unwrap()); + match role.as_str() { + "worker" => { + // Spin until the parent releases every worker at once. + let go = dir.join("GO"); + while !go.exists() { + std::thread::sleep(Duration::from_millis(1)); + } + let transport = Arc::new(LogTransport { + log: PathBuf::from(std::env::var(LOG).unwrap()), + lock: Mutex::new(()), + }); + let rec = Recorder::launch(RecorderConfig::new(&dir, "tok"), transport).unwrap(); + rec.flush(Duration::from_secs(3)); + } + "holder" => { + // Hold every queue item and every recoverable session, then wait to + // be killed. + let mut held = Vec::new(); + for q in queue::list_pending(&dir) { + held.push(queue::try_claim(&q).expect("claim queue item")); + } + for p in recovery::find_pending(&dir, u64::MAX) { + held.push(recovery::claim_pending(&dir, &p).expect("claim session")); + } + // Signal readiness through the filesystem: libtest owns stdout. + std::fs::write(dir.join("READY.tmp"), held.len().to_string()).unwrap(); + std::fs::rename(dir.join("READY.tmp"), dir.join("READY")).unwrap(); + std::thread::sleep(Duration::from_secs(120)); + } + other => panic!("unknown role {other}"), + } +} + +struct TempDir(PathBuf); +impl TempDir { + fn new() -> Self { + let p = + std::env::temp_dir().join(format!("bugsee-mp-{}", bugsee_core::util::random_hex(8))); + std::fs::create_dir_all(&p).unwrap(); + TempDir(p) + } +} +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn spawn(role: &str, dir: &Path, log: &Path) -> Child { + Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "child_entry", "--nocapture"]) + .env(ROLE, role) + .env(DIR, dir) + .env(LOG, log) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap() +} + +/// `queued` reports in the queue and `crashed` dead sessions awaiting recovery. +fn seed(dir: &Path, queued: usize, crashed: u64) { + for i in 0..queued { + queue::enqueue( + dir, + &AssembledReport { + bundle_name: format!("{}.bundle.zip", bugsee_core::util::random_hex(10)), + zip: format!("queued-{i}").into_bytes(), + request_json: br#"{"signatures":[]}"#.to_vec(), + }, + ) + .unwrap(); + } + let sessions = dir.join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + for g in 1..=crashed { + // "0" is the portable "owner gone" sentinel. + std::fs::write(sessions.join(format!("{g}.alive")), "0").unwrap(); + std::fs::create_dir_all(dir.join("parts").join(g.to_string())).unwrap(); + } + std::fs::write(dir.join("gen"), crashed.to_string()).unwrap(); +} + +fn delivered(log: &Path) -> Vec { + std::fs::read_to_string(log) + .map(|s| s.lines().map(String::from).collect()) + .unwrap_or_default() +} + +/// Every file under `dir`, relative, for failure messages. +fn tree(dir: &Path) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + for e in std::fs::read_dir(&d).into_iter().flatten().flatten() { + let p = e.path(); + if p.is_dir() { + stack.push(p); + } else { + out.push(p.strip_prefix(dir).unwrap().display().to_string()); + } + } + } + out.sort(); + out +} + +fn run_workers(dir: &Path, log: &Path, n: usize) { + let kids: Vec = (0..n).map(|_| spawn("worker", dir, log)).collect(); + std::thread::sleep(Duration::from_millis(300)); // all spinning on GO + std::fs::write(dir.join("GO"), "").unwrap(); + for mut k in kids { + let _ = k.wait(); + } +} + +#[test] +fn racing_launchers_deliver_each_pending_report_exactly_once() { + // Repeated: the double-delivery windows are narrow, so one round can pass by luck. + for round in 0..8 { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + std::fs::create_dir_all(&dir).unwrap(); + let log = tmp.0.join("delivered.log"); + seed(&dir, 6, 4); + + run_workers(&dir, &log, 5); + + let got = delivered(&log); + // 6 queued reports + 4 recovered crashes, no more, no fewer. + assert_eq!( + got.len(), + 10, + "round {round}, delivery log: {got:?}, left on disk: {:?}", + tree(&dir) + ); + for i in 0..6 { + let n = got.iter().filter(|l| **l == format!("queued-{i}")).count(); + assert_eq!(n, 1, "round {round}: queued-{i} delivered {n} times"); + } + assert_eq!(got.iter().filter(|l| *l == "recovered-crash").count(), 4); + assert!(queue::list_pending(&dir).is_empty()); + } +} + +#[test] +fn a_holder_killed_mid_work_does_not_block_the_others() { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + std::fs::create_dir_all(&dir).unwrap(); + let log = tmp.0.join("delivered.log"); + seed(&dir, 4, 3); + + let mut holder = spawn("holder", &dir, &log); + let ready = dir.join("READY"); + let deadline = Instant::now() + Duration::from_secs(20); + while !ready.exists() { + assert!(Instant::now() < deadline, "holder never reported ready"); + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(std::fs::read_to_string(&ready).unwrap(), "7"); + + // While the holder lives, nothing it holds is delivered by anyone else. + run_workers(&dir, &log, 2); + assert!(delivered(&log).is_empty(), "live holder's work was taken"); + std::fs::remove_file(dir.join("GO")).unwrap(); + + // SIGKILL / TerminateProcess: no destructor runs, its claim files survive. + holder.kill().unwrap(); + let _ = holder.wait(); + + let deadline = Instant::now() + Duration::from_secs(20); + run_workers(&dir, &log, 3); + while delivered(&log).len() < 7 && Instant::now() < deadline { + run_workers(&dir, &log, 1); + } + assert_eq!(delivered(&log).len(), 7, "log: {:?}", delivered(&log)); +} From 16937c8190078388a7a0dcf9da1096d7c251a1c8 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:41:39 +0500 Subject: [PATCH 05/15] fix(core): arbitrate cross-process claims and session liveness with OS file locks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the pid/start-time claim scheme found it unsound where #14 matters most: containers with separate pid namespaces sharing a volume (both apps pid 1) cannot compare pids or start times, so a peer's live claim was judged stale and its live session reported as a crash. Pid-in-file schemes also needed a lease, a takeover guard with a timer, and identity checks, each with its own race. Replace them with kernel locks: fcntl(F_SETLK) on unix (per process, released on death, not inherited by fork) and LockFileEx on Windows, on dedicated *.lock files (queue/.lock, sessions/.recover.lock, and sessions/.lock held for a session's life). Nothing can go stale, so there is no lease, takeover or start-time logic. An in-process registry (pid-keyed, spin-locked with the holder's pid so a fork child cannot inherit a held lock) excludes threads and keeps a process from ever opening one lock file twice, which POSIX would punish by dropping the lock. - .alive stays a bare pid and lock files are new names, so older builds sharing a directory keep working (no exactly-once between old and new; documented). Markers with no .lock beside them still use the pid probe. - Liveness probe: zombies count as dead (Linux /proc state, macOS proc_pidinfo); Windows uses WaitForSingleObject. - Leftover lock files of finished items and dead sessions are swept when free. - Tests: a live peer whose marker names a colliding pid (parent's, init's) is not recovered until it really dies, then exactly once; concurrent claimers; probing never drops a held lock; zombie probe. Closes #14 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 14 +- crates/bugsee-core/src/claim.rs | 514 +++++++++++++---------- crates/bugsee-core/src/process.rs | 246 ++++------- crates/bugsee-core/src/queue.rs | 11 +- crates/bugsee-core/src/recovery.rs | 79 ++-- crates/bugsee-core/src/session.rs | 29 +- crates/bugsee-core/tests/multiprocess.rs | 46 ++ 7 files changed, 507 insertions(+), 432 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 90bc74b..a2d1009 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -197,9 +197,19 @@ Crash recovery needs the data directory to **outlive the process**: on a contain **Release note, default location moved.** Earlier builds defaulted to `/bugsee/`; the default is now the platform location (3) or the private temp fallback (4). When no directory was requested and the old location exists, is a real directory and is owned by the current user, the launch recovers its crashed sessions and delivers its queue once, best effort (delivery is asynchronous and `flush` does not wait for it). It is not claimed to be lossless: a legacy session whose process is still alive is skipped, and a legacy directory that fails the ownership check is left alone. Hosts that set `data_dir` were never using it. -**Unsafe.** The facade makes two `geteuid()` calls (unix) to name and vet the private temp directory. Core's process probes (`process.rs`) are `kill(pid, 0)` and `proc_pidinfo` (macOS), plus `OpenProcess`/`GetExitCodeProcess`/`GetProcessTimes` on Windows. +**Unsafe.** The facade makes two `geteuid()` calls (unix) to name and vet the private temp directory. Core adds `fcntl(F_SETLK)` (unix) / `LockFileEx` (Windows) for claims, a `kill(pid, 0)` probe (plus `proc_pidinfo` on macOS, `OpenProcess`/`WaitForSingleObject` on Windows) for legacy markers only, and one `UnsafeCell` registry guarded by a pid-stamped spin lock (below). -**Several processes on one directory (#14).** Every pending item is owned by exactly one process at a time: a crashed session by `sessions/.claim`, a queued report by `queue/.claim`. A claim is a file created with `O_EXCL`/`CREATE_NEW` holding `pid start-time nonce`; it is released on drop and, if the holder dies, **taken over** once stale. Stale means: the owner pid is gone, or alive with a different start time (pid reuse; a container's restarted app is pid 1 again), or the file has no owner and is older than 5 s, or it is older than a 30 minute lease (wedged holder, or a platform with no start time). Takeover is serialised by a short-lived `.takeover` guard file (re-checked under the guard), so two takers cannot both remove the other's fresh claim; a guard older than 10 s is abandoned. No OS file lock is used: it would be tied to the open file description, which `fork` duplicates. After winning a session claim the marker is re-checked, because a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s, since a peer's in-flight `enqueue` looks like an orphan (reaping it lost the report). The `.alive` marker also records the start time, so a recycled pid no longer keeps a dead session unrecovered. Delivery is at-least-once: a holder that dies after uploading but before removing the item makes its successor upload it again. Tests: `bugsee-core/tests/multiprocess.rs` (real child processes; racing launchers, SIGKILLed holder) and the unit tests in `claim.rs`, `process.rs`, `queue.rs`. +**Several processes on one directory (#14).** Every pending item is owned by exactly one process at a time, arbitrated by an **OS byte-range lock** on a `*.lock` file: a queued report by `queue/.lock`, a session being recovered by `sessions/.recover.lock`, and a *live session* by `sessions/.lock`, held for the session's whole life. The kernel releases the lock when the process dies however it dies, so nothing can go stale: there is no owner pid, start time, lease or takeover protocol to get wrong. Choices that follow from this: + +- **Pid namespaces.** Containers sharing a volume each see their app as pid 1; a pid (or pid plus start time) written in a file means nothing to the other. A lock does not depend on it, so a peer's live session is never mistaken for a crash and a dead one is never kept alive by pid collision. The `.alive` marker's pid is consulted only for a marker with no `.lock` beside it, i.e. one written by an older SDK build. +- **Why `fcntl`, not `flock`/OFD locks.** `fcntl` locks are per process and are not inherited across `fork`, so a prefork child never holds its parent's claim; `flock`/OFD locks follow the open file description and would. (MSRV 1.88 predates `File::lock` anyway, hence libc/FFI.) +- **Costs of per-process locks, handled in `claim.rs`.** They do not exclude threads, so a registry of held paths does (keyed by pid and spin-locked with the holder's pid stamped in, so a `fork` child that inherits it mid-update breaks the lock instead of deadlocking). POSIX drops a process's locks on a file when *any* descriptor for it closes, so a lock file is never opened twice by one process: the registry is checked before each open, and a claim closes its descriptor before leaving the registry. +- **Ordering at launch.** `Session::begin` creates the marker (`O_EXCL`), takes `.lock`, then writes the pid. Until the pid is written the marker is empty and fresh, which peers skip. +- **Stale scans.** After winning a session claim the marker is re-checked, since a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s: a peer's in-flight `enqueue` looks like an orphan, and reaping it lost the report. Lock files of finished items and dead sessions are swept when free. +- **Compatibility.** The `.alive` marker is still a bare pid and the lock files are new names, so older builds sharing the directory keep working. They do not take part in the locking, so a mixed-version directory (a rolling upgrade) gives no exactly-once guarantee between old and new processes, and a new process treats an old build's session by its pid as before. Run one SDK version per directory for the guarantee. +- **Limits.** Advisory locks on network filesystems are best effort (NFS needs lockd; some mounts do not support them, in which case a claim fails closed: the item is skipped, never double-handled). The only timestamps relied on are the 30 s grace for an empty marker and the 60 s orphan grace, which assume clocks within seconds. Delivery is at-least-once: a holder that dies after uploading but before removing the item (or after enqueueing a recovered report but before discarding the session) makes its successor do it again. Platforms with neither unix nor Windows locks exclude threads only. + +Tests: `bugsee-core/tests/multiprocess.rs` (real child processes: racing launchers, a SIGKILLed holder, a live peer whose marker names a colliding pid) and unit tests in `claim.rs`, `process.rs`, `queue.rs`. Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, a memory-mode case in `bugsee/tests/fork_safety.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised, since mounting one needs root; failures are injected by blocking path components and removing the directory mid-run. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index df40239..ea80538 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -4,184 +4,322 @@ // // Copyright Β© 2026 Bugsee. All rights reserved. // -//! Cross-process exclusive claims on on-disk work items. +//! Cross-process exclusive claims on on-disk work items, and session liveness. //! //! Several processes can share one data directory β€” a prefork server's master -//! and its workers above all β€” and the work items in it (a crashed session to -//! recover, a queued report to upload) must each be handled by exactly one of -//! them, or the backend receives duplicates. A claim is a small file created -//! with `O_EXCL` (`CREATE_NEW` on Windows) that records its owner as -//! `" "`. +//! and its workers, containers sharing a volume β€” and the work items in it (a +//! crashed session to recover, a queued report to upload) must each be handled +//! by exactly one of them, or the backend receives duplicates. //! -//! A claim must never block the work forever if its owner dies, so a stale claim -//! is taken over. It is stale when +//! A claim is an **OS byte-range lock** on a small `*.lock` file: //! -//! * its owner process is gone, or the pid now belongs to a *different* process -//! (the recorded start time no longer matches: pid reuse, or a container's -//! restarted app being pid 1 again), -//! * it has no readable owner and is older than [`UNOWNED_GRACE`] (its creator -//! died between creating the file and writing it), or -//! * it is older than [`MAX_LEASE`], a backstop for a holder that is alive but -//! wedged, or on a platform that cannot tell processes apart. +//! * unix: `fcntl(F_SETLK)`, a *process-scoped* lock. The kernel releases it the +//! instant the process dies, however it dies, and a `fork` child does not +//! inherit it. +//! * Windows: `LockFileEx` on the handle, released when the handle closes or the +//! process exits. //! -//! Taking over is serialised by a short-lived guard file, so two processes that -//! both find the same claim stale cannot each remove the other's fresh claim and -//! both end up holding it. (An OS file lock would release on death automatically, -//! but it is per-open-file-description, not per-process, which makes it unreliable -//! across `fork`.) +//! Because the kernel is the arbiter there is nothing to go stale: no owner pid +//! to probe, no start time, no lease to expire, no takeover protocol. That is +//! also why it is correct across **pid namespaces** (two containers sharing a +//! volume, both with a pid 1) and across pid reuse, where recording a pid in a +//! file is not. The lock file itself may be left behind by a dead holder; it is +//! just an unlocked file, and anyone may lock it. +//! +//! Two costs of process-scoped locks, both handled here: +//! +//! * They do not exclude *threads* of one process, so an in-process registry of +//! held paths does that. +//! * POSIX drops a process's locks on a file when it closes **any** descriptor +//! for that file. So a lock file is never opened twice by one process: the +//! registry is consulted before every open, and a claim closes its descriptor +//! *before* leaving the registry. +//! +//! Advisory locks on network filesystems are best effort (NFS needs lockd). The +//! SDK relies on no file timestamps for claims; the only ones it reads are the +//! grace windows for half-written files in `recovery` and `queue`, which assume +//! clocks that agree to within seconds. //! //! None of this runs on the crash path; it allocates freely. -use std::io::Write; +use std::cell::UnsafeCell; +use std::fs::{File, OpenOptions}; use std::path::{Path, PathBuf}; -use std::time::{Duration, SystemTime}; - -use crate::process; - -/// A claim that has no readable owner is given this long before it is treated as -/// stale: its creator writes the owner a moment after creating the file. -const UNOWNED_GRACE: Duration = Duration::from_secs(5); - -/// No claim is honoured for longer than this, whatever its owner's state. Work -/// items are short (one upload, one recovery), so a claim this old is wedged. -const MAX_LEASE: Duration = Duration::from_secs(30 * 60); - -/// The takeover guard is held for a few syscalls; one older than this was left -/// by a process that died inside the critical section. -const GUARD_TTL: Duration = Duration::from_secs(10); +use std::sync::atomic::{AtomicU32, Ordering}; /// Holds a claim; dropping it releases the claim. #[derive(Debug)] pub struct Claim { path: PathBuf, - /// Exactly what this claim wrote, so release never removes a successor's claim. - content: String, + file: Option, } impl Drop for Claim { fn drop(&mut self) { - if std::fs::read_to_string(&self.path).is_ok_and(|c| c == self.content) { - let _ = std::fs::remove_file(&self.path); - } + // Close (releasing the OS lock) strictly before leaving the registry, so + // a thread that acquires next never has its lock dropped by our close. + drop(self.file.take()); + registry().remove(&self.path); } } -/// Try to claim `path`. `None` if another live process (or another thread of this -/// one) holds it, or if it cannot be claimed. +/// Try to claim `path` (created if absent). `None` if another process, or another +/// thread of this one, holds it, or if it cannot be claimed. pub fn try_claim(path: &Path) -> Option { - let content = owner_record(); - match create(path, &content) { - Create::Done => Some(Claim { - path: path.to_path_buf(), - content, - }), - Create::Exists => take_over(path, content), - // Directory missing / unwritable: nothing can be claimed, and the - // caller's own I/O will fail more usefully than we can here. - Create::Failed => None, + match acquire(path, true) { + Acquire::Got(claim) => Some(claim), + Acquire::Held | Acquire::Unavailable => None, } } -enum Create { - Done, - Exists, - Failed, +/// Whether the lock file at `path` is held by a live process. +pub(crate) enum Probe { + Held, + Free, } -/// Atomically create `path` holding `content` (one `write`, so the owner is -/// visible as soon as the file is). -fn create(path: &Path, content: &str) -> Create { - match std::fs::OpenOptions::new() +/// Is `path` currently claimed? A missing file is free. A file that cannot be +/// inspected is reported held: the conservative answer never disturbs a peer. +pub(crate) fn probe(path: &Path) -> Probe { + if !path.exists() { + return Probe::Free; + } + match acquire(path, false) { + Acquire::Got(claim) => { + drop(claim); + Probe::Free + } + // Vanished since the existence check: nothing holds it. + Acquire::Unavailable if !path.exists() => Probe::Free, + Acquire::Held | Acquire::Unavailable => Probe::Held, + } +} + +/// Remove the lock file at `path` if nobody holds it (a dead holder's leftover). +pub(crate) fn remove_if_free(path: &Path) { + if let Some(claim) = try_claim(path) { + let _ = std::fs::remove_file(path); + drop(claim); + } +} + +enum Acquire { + Got(Claim), + Held, + Unavailable, +} + +fn acquire(path: &Path, create: bool) -> Acquire { + if !registry().insert(path) { + return Acquire::Held; // another thread of this process + } + let file = match OpenOptions::new() + .read(true) .write(true) - .create_new(true) + .create(create) + .truncate(false) .open(path) { - Ok(mut file) => { - let _ = file.write_all(content.as_bytes()); - Create::Done + Ok(f) => f, + Err(_) => { + registry().remove(path); + return Acquire::Unavailable; + } + }; + match os::try_lock(&file) { + Ok(true) => Acquire::Got(Claim { + path: path.to_path_buf(), + file: Some(file), + }), + Ok(false) => { + drop(file); + registry().remove(path); + Acquire::Held + } + Err(_) => { + drop(file); + registry().remove(path); + Acquire::Unavailable } - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Create::Exists, - Err(_) => Create::Failed, } } -/// Replace a stale claim with ours. Only one process at a time may do this: the -/// guard file excludes a second taker between our staleness check and our create, -/// which is exactly the window in which it could otherwise remove the claim we -/// just made. -fn take_over(path: &Path, content: String) -> Option { - let observed = std::fs::read_to_string(path).ok()?; - if !is_stale(path, &observed) { - return None; - } - let guard_path = sibling(path, "takeover"); - match create(&guard_path, "") { - Create::Done => {} - Create::Exists => { - // Held by a peer mid-takeover (it finishes in microseconds, and the - // work is retried on the next drain), or left by one that died. - if age(&guard_path).is_some_and(|a| a > GUARD_TTL) { - let _ = std::fs::remove_file(&guard_path); +/// The paths this process currently holds, keyed by pid so that a `fork` child, +/// which inherits a copy of this memory, ignores the parent's entries. +/// +/// A hand-rolled spin lock rather than a `Mutex`: if another thread held a +/// `Mutex` at the instant of `fork`, the child's copy would stay locked forever +/// and its first claim would deadlock. Here the lock word records the holder's +/// pid, so a child that finds it held by a different pid knows the holder does +/// not exist in this process and breaks it. +struct Registry { + owner: AtomicU32, + held: UnsafeCell>, +} + +// SAFETY: `held` is only touched while `owner` is held (see `with`). +unsafe impl Sync for Registry {} + +static REGISTRY: Registry = Registry { + owner: AtomicU32::new(0), + held: UnsafeCell::new(Vec::new()), +}; + +fn registry() -> &'static Registry { + ®ISTRY +} + +impl Registry { + fn with(&self, f: impl FnOnce(&mut Vec<(u32, PathBuf)>) -> R) -> R { + let pid = std::process::id(); + loop { + match self + .owner + .compare_exchange(0, pid, Ordering::Acquire, Ordering::Relaxed) + { + Ok(_) => break, + Err(holder) if holder != pid => { + // Held by a pid that is not us: a parent's thread at the time + // of `fork`. It does not exist here. Take the lock, and + // discard (leak, never drop) the vector it may have left + // half-modified. + if self + .owner + .compare_exchange(holder, pid, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + // SAFETY: we now hold the lock; the old value is leaked + // because its allocator state may be mid-update. + unsafe { std::ptr::write(self.held.get(), Vec::new()) }; + break; + } + } + Err(_) => std::thread::yield_now(), } - return None; } - Create::Failed => return None, + // SAFETY: exclusive while `owner == pid` (held above). + let out = f(unsafe { &mut *self.held.get() }); + self.owner.store(0, Ordering::Release); + out } - let _guard = Guard(guard_path); - // Re-check under the guard: a peer may have completed its own takeover since - // we looked, and what is there now is a live claim, not the stale one. - if std::fs::read_to_string(path).ok().as_deref() != Some(observed.as_str()) { - return None; + + /// Record `path` as held by this process; `false` if it already is. + fn insert(&self, path: &Path) -> bool { + let pid = std::process::id(); + self.with(|held| { + if held.iter().any(|(p, q)| *p == pid && q == path) { + return false; + } + // Entries of a different pid are a forked-from parent's; drop them. + held.retain(|(p, _)| *p == pid); + held.push((pid, path.to_path_buf())); + true + }) } - let _ = std::fs::remove_file(path); - match create(path, &content) { - Create::Done => Some(Claim { - path: path.to_path_buf(), - content, - }), - // A fresh creator slipped in between our remove and create: it holds it. - Create::Exists | Create::Failed => None, + + fn remove(&self, path: &Path) { + let pid = std::process::id(); + self.with(|held| held.retain(|(p, q)| !(*p == pid && q == path))); } } -struct Guard(PathBuf); -impl Drop for Guard { - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.0); +#[cfg(unix)] +mod os { + use std::fs::File; + use std::io; + use std::os::unix::io::AsRawFd; + + /// Take a non-blocking exclusive lock on byte 0. `Ok(false)` if a peer holds it. + pub fn try_lock(file: &File) -> io::Result { + // SAFETY: an all-zero `flock` is a valid plain-data value. + let mut fl: libc::flock = unsafe { std::mem::zeroed() }; + fl.l_type = libc::F_WRLCK as _; + fl.l_whence = libc::SEEK_SET as _; + fl.l_start = 0; + fl.l_len = 1; + loop { + // SAFETY: `fd` is a valid open descriptor and `fl` a valid `flock`. + let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETLK, &fl) }; + if ret == 0 { + return Ok(true); + } + let err = io::Error::last_os_error(); + match err.raw_os_error() { + Some(libc::EAGAIN) | Some(libc::EACCES) => return Ok(false), + Some(libc::EINTR) => continue, + _ => return Err(err), + } + } } } -fn sibling(path: &Path, ext: &str) -> PathBuf { - let mut s = path.as_os_str().to_os_string(); - s.push("."); - s.push(ext); - PathBuf::from(s) -} +#[cfg(windows)] +mod os { + use std::ffi::c_void; + use std::fs::File; + use std::io; + use std::os::windows::io::AsRawHandle; -/// `" "` for this process; the nonce makes every claim's -/// content unique, so "is this still my claim" is a content comparison. -fn owner_record() -> String { - let (pid, start) = - process::parse_token(&process::self_token()).unwrap_or((std::process::id(), None)); - let start = start.map_or_else(|| "-".to_string(), |s| s.to_string()); - format!("{pid} {start} {}", crate::util::random_hex(8)) -} + const LOCKFILE_FAIL_IMMEDIATELY: u32 = 1; + const LOCKFILE_EXCLUSIVE_LOCK: u32 = 2; + const ERROR_LOCK_VIOLATION: i32 = 33; -fn age(path: &Path) -> Option { - let modified = std::fs::metadata(path).and_then(|m| m.modified()).ok()?; - SystemTime::now().duration_since(modified).ok() -} + #[repr(C)] + struct Overlapped { + internal: usize, + internal_high: usize, + offset: u32, + offset_high: u32, + event: *mut c_void, + } -fn is_stale(path: &Path, content: &str) -> bool { - let age = age(path); - if age.is_some_and(|a| a > MAX_LEASE) { - return true; + extern "system" { + fn LockFileEx( + file: *mut c_void, + flags: u32, + reserved: u32, + len_low: u32, + len_high: u32, + overlapped: *mut Overlapped, + ) -> i32; + } + + /// Take a non-blocking exclusive lock on byte 0. `Ok(false)` if a peer holds it. + pub fn try_lock(file: &File) -> io::Result { + let mut ov = Overlapped { + internal: 0, + internal_high: 0, + offset: 0, + offset_high: 0, + event: std::ptr::null_mut(), + }; + // SAFETY: the handle is valid for the call and `ov` outlives it. + let ok = unsafe { + LockFileEx( + file.as_raw_handle(), + LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, + 0, + 1, + 0, + &mut ov, + ) + }; + if ok != 0 { + return Ok(true); + } + let err = io::Error::last_os_error(); + match err.raw_os_error() { + Some(ERROR_LOCK_VIOLATION) => Ok(false), + _ => Err(err), + } } - match process::parse_token(content) { - // A live owner with the recorded identity (including another thread of - // this very process) holds it. - Some((pid, start)) => !process::owner_is_alive(pid, start), - None => age.is_some_and(|a| a > UNOWNED_GRACE), +} + +/// No file locks: claims exclude threads of this process only. +#[cfg(not(any(unix, windows)))] +mod os { + pub fn try_lock(_file: &std::fs::File) -> std::io::Result { + Ok(true) } } @@ -195,122 +333,58 @@ mod tests { p } - fn backdate(path: &Path, by: Duration) { - let f = std::fs::OpenOptions::new().write(true).open(path).unwrap(); - f.set_modified(SystemTime::now() - by).unwrap(); - } - - /// This process's own live owner record, as a peer would have written it. - fn live_owner() -> String { - format!("{} nonce", process::self_token()) - } - #[test] fn a_claim_is_exclusive_and_released_on_drop() { let dir = tmp(); - let path = dir.join("x.claim"); + let path = dir.join("x.lock"); let first = try_claim(&path).expect("first claim"); assert!(try_claim(&path).is_none(), "already held"); + assert!(matches!(probe(&path), Probe::Held)); drop(first); + assert!(matches!(probe(&path), Probe::Free)); assert!(try_claim(&path).is_some(), "free again after release"); } #[test] - fn a_claim_whose_owner_is_dead_is_taken_over() { - let dir = tmp(); - let path = dir.join("x.claim"); - // A pid that cannot be alive: far above any real pid range. - std::fs::write(&path, "4294967290").unwrap(); - assert!(try_claim(&path).is_some(), "stale claim must be reclaimed"); - assert!(!sibling(&path, "takeover").exists(), "guard released"); - } - - #[test] - fn a_claim_held_by_a_live_process_is_respected() { - let dir = tmp(); - let path = dir.join("x.claim"); - std::fs::write(&path, live_owner()).unwrap(); - assert!(try_claim(&path).is_none()); - // Old-format owner (pid only), pid 1: alive on unix. - #[cfg(unix)] - { - std::fs::write(&path, "1").unwrap(); - assert!(try_claim(&path).is_none()); - } - } - - #[test] - fn a_recycled_pid_does_not_keep_a_dead_owners_claim_alive() { - let Some(start) = process::start_time(std::process::id()) else { - return; // platform cannot tell processes apart - }; - let dir = tmp(); - let path = dir.join("x.claim"); - // Our own (live) pid, but with a start time that is not ours: the claim - // was written by an earlier process that had this pid. - std::fs::write( - &path, - format!("{} {} stale", std::process::id(), start.wrapping_add(1)), - ) - .unwrap(); - assert!(try_claim(&path).is_some(), "recycled pid must not block"); - } - - #[test] - fn a_freshly_created_unowned_claim_is_not_stolen_but_an_old_one_is() { - let dir = tmp(); - let path = dir.join("x.claim"); - std::fs::write(&path, "").unwrap(); // creator has not written its owner yet - assert!(try_claim(&path).is_none()); - backdate(&path, UNOWNED_GRACE + Duration::from_secs(5)); - assert!(try_claim(&path).is_some(), "creator died before writing"); - } - - #[test] - fn a_wedged_holder_cannot_hold_a_claim_past_the_lease() { + fn a_leftover_lock_file_from_a_dead_holder_is_just_free() { let dir = tmp(); - let path = dir.join("x.claim"); - std::fs::write(&path, live_owner()).unwrap(); - assert!(try_claim(&path).is_none()); - backdate(&path, MAX_LEASE + Duration::from_secs(60)); + let path = dir.join("x.lock"); + // Contents are irrelevant: a dead holder's pid, even one that collides + // with a live process (here, our own), must not block anything. + std::fs::write(&path, std::process::id().to_string()).unwrap(); + assert!(matches!(probe(&path), Probe::Free)); assert!(try_claim(&path).is_some()); } #[test] - fn releasing_a_claim_never_removes_its_successors() { + fn a_missing_lock_file_probes_free_and_is_not_created() { let dir = tmp(); - let path = dir.join("x.claim"); - let first = try_claim(&path).unwrap(); - // First is judged wedged and replaced behind its back. - std::fs::write(&path, live_owner()).unwrap(); - drop(first); - assert!(path.exists(), "the successor's claim must survive"); + let path = dir.join("nope.lock"); + assert!(matches!(probe(&path), Probe::Free)); + assert!(!path.exists()); } #[test] - fn an_abandoned_takeover_guard_does_not_block_forever() { + fn dropping_one_claim_never_releases_another() { + // POSIX drops a process's locks on a file when ANY descriptor for it + // closes; the registry must keep a second open from ever happening. let dir = tmp(); - let path = dir.join("x.claim"); - std::fs::write(&path, "4294967290").unwrap(); - let guard = sibling(&path, "takeover"); - std::fs::write(&guard, "").unwrap(); - // A taker is (apparently) mid-takeover: back off. + let path = dir.join("x.lock"); + let held = try_claim(&path).unwrap(); assert!(try_claim(&path).is_none()); - // It died there long ago: clear the guard, then succeed. - backdate(&guard, GUARD_TTL + Duration::from_secs(5)); + assert!(matches!(probe(&path), Probe::Held)); assert!( try_claim(&path).is_none(), - "clearing the guard is its own step" + "probing must not have closed our lock" ); - assert!(try_claim(&path).is_some()); + drop(held); } #[test] - fn racing_takers_of_one_stale_claim_yield_exactly_one_holder() { - for _ in 0..200 { + fn racing_claimers_of_one_path_yield_exactly_one_holder() { + for _ in 0..100 { let dir = tmp(); - let path = dir.join("x.claim"); - std::fs::write(&path, "4294967290").unwrap(); + let path = dir.join("x.lock"); let barrier = std::sync::Arc::new(std::sync::Barrier::new(8)); let handles: Vec<_> = (0..8) .map(|_| { @@ -318,7 +392,7 @@ mod tests { std::thread::spawn(move || { barrier.wait(); let claim = try_claim(&path); - // Keep the claim held until every thread has tried. + // Hold until every thread has tried. barrier.wait(); claim.is_some() }) @@ -329,7 +403,7 @@ mod tests { .map(|h| h.join().unwrap()) .filter(|held| *held) .count(); - assert_eq!(holders, 1, "exactly one taker may win"); + assert_eq!(holders, 1, "exactly one claimer may win"); let _ = std::fs::remove_dir_all(&dir); } } diff --git a/crates/bugsee-core/src/process.rs b/crates/bugsee-core/src/process.rs index c37f583..c857e46 100644 --- a/crates/bugsee-core/src/process.rs +++ b/crates/bugsee-core/src/process.rs @@ -4,24 +4,19 @@ // // Copyright Β© 2026 Bugsee. All rights reserved. // -//! Process identity and liveness, for deciding whether the owner recorded in a -//! claim or a liveness marker is still the process that wrote it. +//! Process liveness probe, used only for **legacy** session markers. //! -//! A bare pid is not an identity: pids are recycled, and in a container the -//! restarted app is routinely pid 1 again. An *owner token* therefore pairs the -//! pid with the process's **start time** (an opaque, stable per-process number), -//! so a recycled pid is told apart from the original owner. Where the platform -//! offers no start time the token carries the pid alone and liveness degrades to -//! "a process with that pid exists". +//! Current sessions and claims are arbitrated by OS file locks (see `claim`), +//! which need no pid and are correct across pid namespaces. A session marker +//! written by an older SDK build has no lock beside it, so for those alone the +//! recorded pid is probed here. The probe is inherently single-namespace. -use std::sync::OnceLock; - -/// Whether a process with `pid` currently exists. +/// Whether a process with `pid` currently exists and has not terminated. A +/// zombie (exited, not yet reaped by its parent) counts as dead. /// /// Unix: `kill(pid, 0)`; `0` (permitted) or `EPERM` (exists, not permitted) both -/// mean alive, `ESRCH` means gone. Windows: the process can be opened and has not -/// exited. A pid-reuse false positive is closed by [`owner_is_alive`]'s start -/// time comparison, not here. +/// mean alive, `ESRCH` means gone. Windows: the process can be opened and its +/// handle is not yet signalled. #[cfg(unix)] pub(crate) fn is_alive(pid: u32) -> bool { if pid == 0 { @@ -30,167 +25,93 @@ pub(crate) fn is_alive(pid: u32) -> bool { // SAFETY: `kill` with signal 0 performs only an error/permission check and // has no side effects; it is safe to call with any pid value. let ret = unsafe { libc::kill(pid as libc::pid_t, 0) }; - ret == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) -} - -#[cfg(windows)] -pub(crate) fn is_alive(pid: u32) -> bool { - win::with_process(pid, |handle| { - let mut code = 0u32; - // SAFETY: `handle` is a valid open process handle for the closure's duration. - let ok = unsafe { win::GetExitCodeProcess(handle, &mut code) }; - ok == 0 || code == win::STILL_ACTIVE - }) - // Access denied means the process exists but belongs to someone else. - .unwrap_or_else(|denied| denied) -} - -#[cfg(not(any(unix, windows)))] -pub(crate) fn is_alive(_pid: u32) -> bool { - false + (ret == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)) + && !is_zombie(pid) } -/// An opaque per-process start time for `pid`, or `None` if it cannot be read -/// (platform without support, process gone, or not permitted). Only ever compared -/// for equality against a value read the same way. -#[cfg(target_os = "linux")] -pub(crate) fn start_time(pid: u32) -> Option { - let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; - // `pid (comm) state ...`: comm may contain spaces and parens, so split after - // the LAST ')'. `starttime` is field 22; `state` (field 3) is index 0 here. - let rest = &stat[stat.rfind(')')? + 1..]; - rest.split_whitespace().nth(19)?.parse().ok() +#[cfg(any(target_os = "linux", target_os = "android"))] +fn is_zombie(pid: u32) -> bool { + let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + // `pid (comm) state ...`: comm may contain spaces and parens, so look after + // the LAST ')'. + stat.rfind(')') + .and_then(|i| stat[i + 1..].split_whitespace().next()) + .is_some_and(|state| state == "Z" || state == "X") } #[cfg(target_os = "macos")] -pub(crate) fn start_time(pid: u32) -> Option { - // SAFETY: an all-zero `proc_bsdinfo` is a valid plain-data value, and +fn is_zombie(pid: u32) -> bool { + // The "short" BSD info, unlike the full one, is still available for a zombie. + // SAFETY: an all-zero `proc_bsdshortinfo` is valid plain data, and // `proc_pidinfo` writes at most `size_of` bytes into it. - let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; - let size = std::mem::size_of::() as libc::c_int; + let mut info: libc::proc_bsdshortinfo = unsafe { std::mem::zeroed() }; + let size = std::mem::size_of::() as libc::c_int; let n = unsafe { libc::proc_pidinfo( pid as libc::c_int, - libc::PROC_PIDTBSDINFO, + libc::PROC_PIDT_SHORTBSDINFO, 0, - (&mut info as *mut libc::proc_bsdinfo).cast(), + (&mut info as *mut libc::proc_bsdshortinfo).cast(), size, ) }; - (n == size).then(|| info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec) -} - -#[cfg(windows)] -pub(crate) fn start_time(pid: u32) -> Option { - win::with_process(pid, |handle| { - let mut ft = [win::FileTime::default(); 4]; - // SAFETY: `handle` is valid; the four out-pointers point into `ft`. - let [creation, exit, kernel, user] = &mut ft; - let ok = unsafe { win::GetProcessTimes(handle, creation, exit, kernel, user) }; - (ok != 0).then(|| (u64::from(ft[0].high) << 32) | u64::from(ft[0].low)) - }) - .ok() - .flatten() + const SZOMB: u32 = 5; + // Observed on macOS: for an exited-but-unreaped child `kill(pid, 0)` still + // succeeds while `proc_pidinfo` returns 0 (no info), whereas any live process, + // including other users', reports its info. So "no info" means zombie here. + n == 0 || (n == size && info.pbsi_status == SZOMB) } -#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] -pub(crate) fn start_time(_pid: u32) -> Option { - None +#[cfg(all( + unix, + not(any(target_os = "linux", target_os = "android", target_os = "macos")) +))] +fn is_zombie(_pid: u32) -> bool { + false } -/// This process's owner token, `""` or `" "`. Cached, but keyed -/// by pid: a `fork` child inherits the cache and must not reuse the parent's token. -pub(crate) fn self_token() -> String { - static CACHE: OnceLock<(u32, String)> = OnceLock::new(); - let pid = std::process::id(); - let (cached_pid, token) = CACHE.get_or_init(|| (pid, make_token(pid))); - if *cached_pid == pid { - token.clone() - } else { - make_token(pid) +#[cfg(windows)] +pub(crate) fn is_alive(pid: u32) -> bool { + if pid == 0 { + return false; } -} - -fn make_token(pid: u32) -> String { - match start_time(pid) { - Some(start) => format!("{pid} {start}"), - None => pid.to_string(), + // SAFETY: plain Win32 calls; the handle is closed on every path. + unsafe { + let handle = win::OpenProcess(win::SYNCHRONIZE, 0, pid); + if handle.is_null() { + // Access denied means the process exists but belongs to someone else. + return std::io::Error::last_os_error().raw_os_error() + == Some(win::ERROR_ACCESS_DENIED); + } + // A process handle is signalled once the process has terminated. + let alive = win::WaitForSingleObject(handle, 0) == win::WAIT_TIMEOUT; + win::CloseHandle(handle); + alive } } -/// Parse an owner token (`""` or `" "`; trailing fields, such as -/// a claim's nonce, are ignored). `None` if no pid can be read. -pub(crate) fn parse_token(text: &str) -> Option<(u32, Option)> { - let mut it = text.split_whitespace(); - let pid = it.next()?.parse().ok()?; - let start = it.next().and_then(|s| s.parse().ok()); - Some((pid, start)) -} - -/// Whether the process an owner token names is still alive *and is the same -/// process that wrote the token*. With a recorded start time, a live process -/// whose start time differs is a recycled pid, i.e. the owner is gone. -pub(crate) fn owner_is_alive(pid: u32, recorded_start: Option) -> bool { - if !is_alive(pid) { - return false; - } - match (recorded_start, start_time(pid)) { - (Some(recorded), Some(now)) => recorded == now, - // Cannot compare (old-format token, or start time unreadable): the pid - // exists, so give the owner the benefit of the doubt. - _ => true, - } +#[cfg(not(any(unix, windows)))] +pub(crate) fn is_alive(_pid: u32) -> bool { + false } #[cfg(windows)] mod win { //! The few kernel32 calls needed, declared directly: kernel32 is always - //! linked, so this avoids a dependency for four functions. + //! linked, so this avoids a dependency for three functions. use std::ffi::c_void; pub type Handle = *mut c_void; - pub const STILL_ACTIVE: u32 = 259; - const PROCESS_QUERY_LIMITED_INFORMATION: u32 = 0x1000; - const ERROR_ACCESS_DENIED: i32 = 5; - - #[repr(C)] - #[derive(Clone, Copy, Default)] - pub struct FileTime { - pub low: u32, - pub high: u32, - } + pub const SYNCHRONIZE: u32 = 0x0010_0000; + pub const WAIT_TIMEOUT: u32 = 258; + pub const ERROR_ACCESS_DENIED: i32 = 5; extern "system" { - fn OpenProcess(access: u32, inherit: i32, pid: u32) -> Handle; - fn CloseHandle(handle: Handle) -> i32; - pub fn GetExitCodeProcess(handle: Handle, code: *mut u32) -> i32; - pub fn GetProcessTimes( - handle: Handle, - creation: *mut FileTime, - exit: *mut FileTime, - kernel: *mut FileTime, - user: *mut FileTime, - ) -> i32; - } - - /// Run `f` with a query handle to `pid`. `Err(true)` if the process exists but - /// access was denied, `Err(false)` if it cannot be opened (gone). - pub fn with_process(pid: u32, f: impl FnOnce(Handle) -> T) -> Result { - if pid == 0 { - return Err(false); - } - // SAFETY: plain Win32 calls; the handle is closed on every path. - unsafe { - let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); - if handle.is_null() { - let denied = - std::io::Error::last_os_error().raw_os_error() == Some(ERROR_ACCESS_DENIED); - return Err(denied); - } - let out = f(handle); - CloseHandle(handle); - Ok(out) - } + pub fn OpenProcess(access: u32, inherit: i32, pid: u32) -> Handle; + pub fn CloseHandle(handle: Handle) -> i32; + pub fn WaitForSingleObject(handle: Handle, millis: u32) -> u32; } } @@ -199,34 +120,23 @@ mod tests { use super::*; #[test] - fn own_process_is_alive_with_a_stable_start_time() { + fn own_process_is_alive_and_an_impossible_pid_is_not() { assert!(is_alive(std::process::id())); - let a = start_time(std::process::id()); - assert_eq!(a, start_time(std::process::id())); - #[cfg(any(target_os = "linux", target_os = "macos", windows))] - assert!(a.is_some(), "start time must be readable on this platform"); + assert!(!is_alive(0)); + assert!(!is_alive(4_294_967_290)); } + #[cfg(unix)] #[test] - fn a_pid_with_the_wrong_start_time_is_a_recycled_pid() { - let me = std::process::id(); - if let Some(start) = start_time(me) { - assert!(owner_is_alive(me, Some(start))); - assert!(!owner_is_alive(me, Some(start.wrapping_add(1)))); + fn a_zombie_is_dead() { + let mut child = std::process::Command::new("true").spawn().unwrap(); + let pid = child.id(); + // Wait for it to exit without reaping it (a non-consuming probe). + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while is_alive(pid) && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(10)); } - assert!(owner_is_alive(me, None), "no recorded start: pid alone"); - assert!(!owner_is_alive(4_294_967_290, Some(1))); - } - - #[test] - fn tokens_round_trip() { - assert_eq!(parse_token("12 345"), Some((12, Some(345)))); - assert_eq!(parse_token("12"), Some((12, None))); - assert_eq!(parse_token("12 345 deadbeef"), Some((12, Some(345)))); - assert_eq!(parse_token("junk"), None); - assert_eq!(parse_token(""), None); - let (pid, start) = parse_token(&self_token()).unwrap(); - assert_eq!(pid, std::process::id()); - assert_eq!(start, start_time(pid)); + assert!(!is_alive(pid), "exited but unreaped child counts as dead"); + let _ = child.wait(); } } diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index 11e5254..804b95d 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -35,15 +35,15 @@ impl QueuedReport { fn endpoint_path(&self) -> PathBuf { sibling(&self.bundle, "endpoint") } - fn claim_path(&self) -> PathBuf { - sibling(&self.bundle, "claim") + fn lock_path(&self) -> PathBuf { + sibling(&self.bundle, "lock") } } /// Claim a queued report for delivery, so that of several processes draining one /// shared queue exactly one uploads it. Released when the guard drops. pub fn try_claim(report: &QueuedReport) -> Option { - crate::claim::try_claim(&report.claim_path()) + crate::claim::try_claim(&report.lock_path()) } fn sibling(bundle: &Path, ext: &str) -> PathBuf { @@ -151,6 +151,9 @@ pub fn remove(report: &QueuedReport) { let _ = std::fs::remove_file(report.req_path()); let _ = std::fs::remove_file(report.meta_path()); let _ = std::fs::remove_file(report.endpoint_path()); + // The delivering claimant's own lock file (removing a held lock file is fine: + // the item is gone, so no one will look for it). + let _ = std::fs::remove_file(report.lock_path()); } /// The bundle file's base name (for reconstructing an [`AssembledReport`]). @@ -249,6 +252,8 @@ pub fn gc_orphans(data_dir: &Path) { .strip_suffix(".req") .or_else(|| name.strip_suffix(".meta")) .or_else(|| name.strip_suffix(".endpoint")) + .or_else(|| name.strip_suffix(".lock")) + // Leftovers of an earlier claim scheme. .or_else(|| name.strip_suffix(".claim")) .or_else(|| name.strip_suffix(".claim.takeover")) .or_else(|| name.strip_suffix(".tmp")); diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 0262252..51c6a36 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -80,34 +80,18 @@ pub fn find_pending(data_dir: &Path, current_generation: u64) -> Vec= current_generation { continue; } - // Decide from the marker's recorded owner pid whether this is a real - // crash or a still-running shared-dir peer. - let owner = std::fs::read_to_string(&marker).unwrap_or_default(); - let owner = owner.trim(); - if owner.is_empty() { - // Present but not yet populated: `Session::begin` creates the marker - // (O_EXCL) and writes the pid a moment later, so a peer launching - // concurrently can observe this empty window. Skip only while the - // marker is FRESH β€” a peer writes its pid within microseconds. An empty - // marker older than the grace window is a session that died in the - // createβ†’write gap (or whose pid write silently failed on a full - // disk); recover it so its crash is surfaced and its generation - // reclaimed instead of being re-scanned and leaked on every future - // launch (F28). No pid β‡’ no liveness check; fall through to recover. - if marker_is_fresh(&marker) { - continue; - } - } else if let Some((pid, start)) = crate::process::parse_token(owner) { - // Skip a marker still owned by a live *other* process. Our own pid is - // never skipped (a live process cannot share our pid, so a marker - // bearing it is a dead prior incarnation β€” recover it). The marker - // may carry the owner's start time, which tells a recycled pid (a - // dead owner whose pid something else now uses) from the owner. - if pid != self_pid && crate::process::owner_is_alive(pid, start) { + // A session whose `.lock` exists is arbitrated by that OS lock: held + // means a live process (in any pid namespace), free means it died. The + // marker's pid is not consulted, since it is only meaningful inside the + // writer's own pid namespace and can be reused. + let lock = crate::session::Session::lock_path(data_dir, generation); + if lock.exists() { + if matches!(crate::claim::probe(&lock), crate::claim::Probe::Held) { continue; } + } else if !legacy_marker_is_dead(&marker, self_pid) { + continue; } - // Non-empty but unparseable content β‡’ the owner is gone; recover. let parts_dir = data_dir.join("parts").join(generation.to_string()); let minidump = exists_opt(parts_dir.join(MINIDUMP_NAME)); let crash_info = exists_opt(parts_dir.join(CRASH_INFO_NAME)); @@ -127,6 +111,31 @@ fn exists_opt(p: PathBuf) -> Option { p.exists().then_some(p) } +/// Whether a marker with no lock file beside it (written by an older SDK build, +/// or by a session that could not lock) is a crashed session rather than a live +/// peer, judged from its recorded pid. Only meaningful within one pid namespace. +fn legacy_marker_is_dead(marker: &Path, self_pid: u32) -> bool { + let owner = std::fs::read_to_string(marker).unwrap_or_default(); + let owner = owner.trim(); + if owner.is_empty() { + // Present but not yet populated: `Session::begin` creates the marker + // (O_EXCL) and writes the pid a moment later, so a peer launching + // concurrently can observe this empty window. Skip only while the + // marker is FRESH. An empty marker older than the grace window is a + // session that died in the create-to-write gap (or whose pid write failed + // on a full disk); recover it so its crash is surfaced and its + // generation reclaimed instead of being re-scanned on every launch. + return !marker_is_fresh(marker); + } + match owner.parse::() { + // Our own pid is never skipped (a live process cannot share our pid, so a + // marker bearing it is a dead prior incarnation). + Ok(pid) => pid == self_pid || !crate::process::is_alive(pid), + // Non-empty but unparseable content: the owner is gone. + Err(_) => true, + } +} + /// Grace window for an empty (pid-not-yet-written) liveness marker: a launching /// peer writes its pid within microseconds of the O_EXCL create, so a marker /// still empty after this is a dead session, not a live peer mid-launch. @@ -332,7 +341,7 @@ pub fn claim_pending(data_dir: &Path, pending: &PendingSession) -> Option() { - if generation < current_generation && !alive(generation) { - drop(crate::claim::try_claim(&entry.path())); - } + // Lock files of a generation with no marker: the session lock of a + // crashed process (its marker already recovered) and the recovery + // claim. Removed only if free, so a live holder's is left alone. + let lock_gen = name + .strip_suffix(".recover.lock") + .or_else(|| name.strip_suffix(".lock")) + .and_then(|g| g.parse::().ok()); + if let Some(generation) = lock_gen { + if generation < current_generation && !alive(generation) { + crate::claim::remove_if_free(&entry.path()); } } } diff --git a/crates/bugsee-core/src/session.rs b/crates/bugsee-core/src/session.rs index 5a81692..8d75c8c 100644 --- a/crates/bugsee-core/src/session.rs +++ b/crates/bugsee-core/src/session.rs @@ -27,6 +27,12 @@ const SESSIONS_DIR: &str = "sessions"; pub struct Session { data_dir: PathBuf, generation: u64, + /// The OS lock on `.lock`, held for the session's life. Peers sharing the + /// data dir (including from another pid namespace) probe it to tell a live + /// session from a crashed one; the kernel drops it when this process dies. + /// `None` if the lock file could not be taken (the bare pid in the marker is + /// then all a peer has). + _lock: Option, } impl Session { @@ -38,7 +44,7 @@ impl Session { std::fs::create_dir_all(&markers)?; let mut candidate = floor_generation(data_dir) + 1; - let generation = loop { + let (generation, lock) = loop { let marker = markers.join(format!("{candidate}.alive")); match std::fs::OpenOptions::new() .write(true) @@ -46,10 +52,13 @@ impl Session { .open(&marker) { Ok(mut f) => { - // " ": the start time lets recovery tell this - // process from a later one that reuses the pid. - let _ = write!(f, "{}", crate::process::self_token()); - break candidate; + // Lock BEFORE writing the pid: the marker is empty (and fresh, + // so peers skip it) until then, which closes the window in + // which it is neither locked nor attributed. + let lock = crate::claim::try_claim(&Session::lock_path(data_dir, candidate)); + // The marker stays a bare pid, readable by older builds. + let _ = write!(f, "{}", std::process::id()); + break (candidate, lock); } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { candidate += 1; @@ -64,6 +73,7 @@ impl Session { Ok(Session { data_dir: data_dir.to_path_buf(), generation, + _lock: lock, }) } @@ -79,9 +89,18 @@ impl Session { .join(format!("{generation}.alive")) } + /// The liveness lock-file path for `generation`. Its presence marks a session + /// whose liveness is arbitrated by the lock rather than the marker's pid. + pub(crate) fn lock_path(data_dir: &Path, generation: u64) -> PathBuf { + data_dir + .join(SESSIONS_DIR) + .join(format!("{generation}.lock")) + } + /// Mark a clean shutdown: remove this session's liveness marker. pub fn end(&self) { let _ = std::fs::remove_file(Session::marker_path(&self.data_dir, self.generation)); + let _ = std::fs::remove_file(Session::lock_path(&self.data_dir, self.generation)); } } diff --git a/crates/bugsee-core/tests/multiprocess.rs b/crates/bugsee-core/tests/multiprocess.rs index 348f50c..9b2ae1e 100644 --- a/crates/bugsee-core/tests/multiprocess.rs +++ b/crates/bugsee-core/tests/multiprocess.rs @@ -94,6 +94,13 @@ fn child_entry() { std::fs::rename(dir.join("READY.tmp"), dir.join("READY")).unwrap(); std::thread::sleep(Duration::from_secs(120)); } + "session" => { + // A live session of its own, as a peer's launch would create. + let session = bugsee_core::session::Session::begin(&dir).unwrap(); + std::fs::write(dir.join("READY.tmp"), session.generation().to_string()).unwrap(); + std::fs::rename(dir.join("READY.tmp"), dir.join("READY")).unwrap(); + std::thread::sleep(Duration::from_secs(120)); + } other => panic!("unknown role {other}"), } } @@ -243,3 +250,42 @@ fn a_holder_killed_mid_work_does_not_block_the_others() { } assert_eq!(delivered(&log).len(), 7, "log: {:?}", delivered(&log)); } + +/// Containers with separate pid namespaces share a volume, and each app is pid 1, +/// so a peer's recorded pid can name a live process HERE that is not the peer. +/// The marker's pid must therefore play no part in deciding the peer is dead. +#[test] +fn a_live_peers_session_is_not_mistaken_for_a_crash_whatever_pid_its_marker_names() { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + std::fs::create_dir_all(&dir).unwrap(); + let log = tmp.0.join("delivered.log"); + + let mut peer = spawn("session", &dir, &log); + let ready = dir.join("READY"); + let deadline = Instant::now() + Duration::from_secs(20); + while !ready.exists() { + assert!(Instant::now() < deadline, "peer never started"); + std::thread::sleep(Duration::from_millis(10)); + } + let generation: u64 = std::fs::read_to_string(&ready).unwrap().parse().unwrap(); + + // Make the marker name a pid that is alive in THIS namespace and is not the + // peer: the parent's own, then init's. Neither may make the peer look dead, + // nor make it look alive once it is gone. + let marker = dir.join("sessions").join(format!("{generation}.alive")); + for foreign in [std::process::id().to_string(), "1".to_string()] { + std::fs::write(&marker, &foreign).unwrap(); + run_workers(&dir, &log, 3); + let _ = std::fs::remove_file(dir.join("GO")); + assert!(delivered(&log).is_empty(), "live peer reported as a crash"); + assert!(marker.exists(), "live peer's marker was reclaimed"); + } + + // Once the peer really dies, its session is recovered exactly once, even + // though the marker still names a live pid. + peer.kill().unwrap(); + let _ = peer.wait(); + run_workers(&dir, &log, 3); + assert_eq!(delivered(&log), vec!["recovered-crash".to_string()]); +} From 31a293c8cfe74111ef76ff25fe73ca0ae9356184 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:44:09 +0500 Subject: [PATCH 06/15] fix: keep draining existing data dirs that cannot take new files (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - an existing directory that fails the write probe / session open is still recovered from and drained (read + delete only); claims are lenient there and a crash that cannot be re-queued on disk is delivered from memory; delivered but undeletable bundles are remembered so they are not resent every drain - in-memory TooManySimilar blacklist (cleared in fork children) - a clean stop gives the in-memory backlog a last bounded delivery attempt - legacy temp location adopted whenever it exists and is trusted, even if no new default directory was usable - rotation tick rewrites a missing liveness marker after a wipe and marks the recorder degraded - DESIGN.md: failure table, legacy/flush wording πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 7 +- crates/bugsee-core/src/claim.rs | 37 +++- crates/bugsee-core/src/queue.rs | 6 + crates/bugsee-core/src/recovery.rs | 11 +- crates/bugsee-core/src/runtime.rs | 154 +++++++++++-- crates/bugsee-core/tests/storage_failures.rs | 222 +++++++++++++++++++ crates/bugsee/src/api.rs | 7 +- crates/bugsee/src/options.rs | 86 ++++++- crates/bugsee/tests/storage_fallback.rs | 57 +++++ 9 files changed, 539 insertions(+), 48 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index f2f03f6..4d06151 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -186,8 +186,9 @@ Global base scope behind `RwLock`; `with_scope` pushes a thread-local overlay. * | Failure | Outcome | |---|---| | Default candidate unusable, a later default works | falls through; one stderr line; `StorageStatus::skipped` lists why | -| Requested directory unusable, or no default usable, or `Session::begin`/`PartStore::new` fails (also in a fork child) | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from an in-memory backlog capped at 16 (an entry counts until its delivery settles; overflow drops the newest with `DropReason::StorageUnavailable`); an unsent report dies with the process; a fork child starts with an empty backlog (the parent's is the parent's to send) | -| Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and dropped; a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out; `StorageStatus::degraded` turns true and `crash_recovery_available()` false | +| Requested directory unusable, or no default usable, or `Session::begin`/`PartStore::new` fails (also in a fork child) | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from an in-memory backlog capped at 16 (an entry counts until its delivery settles; overflow drops the newest with `DropReason::StorageUnavailable`); a `TooManySimilar` verdict is remembered in memory; a clean stop (guard drop) gives the backlog one last bounded (1.5 s) attempt, but a report still unsent when the process dies is lost; a fork child starts with an empty backlog (the parent's is the parent's to send) | +| Directory exists but cannot take NEW files (full, read-only remount) | memory mode for new reports, but the directory is still **drained and recovered from**: its queued bundles are delivered and its crashed sessions recovered (claims are best-effort there; a crash that cannot be re-queued on disk is delivered from memory). Bundles delivered but undeletable are remembered and not resent in this process; a later launch with a writable disk may resend them and the backend deduplicates. Only a missing or non-directory path is skipped | +| Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and dropped; the rotation tick rewrites a missing `sessions/.alive` (a wiped-and-recreated tree would otherwise make the next launch's orphan sweep drop this session's crash); a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out; `StorageStatus::degraded` turns true and `crash_recovery_available()` false | | Disk full / permission denied at enqueue, snapshot, marker | same as above; the `reports/` snapshot dir is reclaimed by `SnapshotHandle::drop` | | Crash handler cannot write its marker | the raw `open`/`write` return values are ignored; the process still dies with the original signal (watchdog bounds the handler) | @@ -195,7 +196,7 @@ The in-memory backlog retries like the disk queue: it caches the presigned endpo Crash recovery needs the data directory to **outlive the process**: on a container, mount a volume and point `data_dir`/`BUGSEE_DATA_DIR` at it. The temp fallback is commonly wiped on restart. `Bugsee::storage_status()` reports `mode`, `data_dir`, `source`, `skipped`, `explicit_rejected`, `degraded`, `crash_recovery_available()` and `may_be_ephemeral()` (the SDK cannot tell a volume from a scratch disk; the temp fallback is the only case it flags). No FFI accessor yet (follow-up). -**Release note, default location moved.** Earlier builds defaulted to `/bugsee/`; the default is now the platform location (3) or the private temp fallback (4). When no directory was requested and the old location exists, is a real directory and is owned by the current user, the launch recovers its crashed sessions and delivers its queue once, best effort (delivery is asynchronous and `flush` does not wait for it). It is not claimed to be lossless: a legacy session whose process is still alive is skipped, and a legacy directory that fails the ownership check is left alone. Hosts that set `data_dir` were never using it. +**Release note, default location moved.** Earlier builds defaulted to `/bugsee/`; the default is now the platform location (3) or the private temp fallback (4). When no directory was requested and the old location exists, is a real directory and is owned by the current user, the launch recovers its crashed sessions and delivers its queue once, best effort, including when no new directory could be created. Recovery runs on the worker before a `flush` is acknowledged and `flush` drains the legacy queue with the data directory's, so a flush covers it; the uploader stops polling it once its queue is empty. It is not claimed to be lossless: a legacy session whose process is still alive is skipped, and a legacy directory that fails the ownership check is left alone. Hosts that set `data_dir` were never using it. **Unsafe.** The facade makes two `geteuid()` calls (unix) to name and vet the private temp directory, alongside core's single `kill(pid, 0)`. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index 7e01144..12ab003 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -38,8 +38,35 @@ impl Drop for Claim { } /// Try to claim `path`. `None` if another live process (or another thread of this -/// one) holds it. +/// one) holds it, or if the claim file cannot be created at all. pub fn try_claim(path: &Path) -> Option { + match claim_state(path) { + ClaimState::Held(claim) => Some(claim), + ClaimState::Taken | ClaimState::Unavailable => None, + } +} + +/// Like [`try_claim`] for a directory that may be full or read-only, where the +/// claim file cannot be created but the work (recover a crash, deliver a queued +/// report) is still worth doing: `Some(None)` means "no claim could be made, +/// proceed unclaimed". The cost is that two processes sharing such a directory +/// may both do the work, and the backend deduplicates. `None` means another +/// process holds the claim. +pub fn try_claim_lenient(path: &Path) -> Option> { + match claim_state(path) { + ClaimState::Held(claim) => Some(Some(claim)), + ClaimState::Unavailable => Some(None), + ClaimState::Taken => None, + } +} + +enum ClaimState { + Held(Claim), + Taken, + Unavailable, +} + +fn claim_state(path: &Path) -> ClaimState { for _ in 0..2 { match std::fs::OpenOptions::new() .write(true) @@ -48,13 +75,13 @@ pub fn try_claim(path: &Path) -> Option { { Ok(mut file) => { let _ = write!(file, "{}", std::process::id()); - return Some(Claim { + return ClaimState::Held(Claim { path: path.to_path_buf(), }); } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { if !is_stale(path) { - return None; + return ClaimState::Taken; } // Take over a claim whose owner is gone, then retry the create. // Losing the race to another taker just means `create_new` fails @@ -63,10 +90,10 @@ pub fn try_claim(path: &Path) -> Option { } // Directory missing / unwritable: nothing can be claimed, and the // caller's own I/O will fail more usefully than we can here. - Err(_) => return None, + Err(_) => return ClaimState::Unavailable, } } - None + ClaimState::Taken } fn is_stale(path: &Path) -> bool { diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index 2ff3efe..ea6c033 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -46,6 +46,12 @@ pub fn try_claim(report: &QueuedReport) -> Option { crate::claim::try_claim(&report.claim_path()) } +/// [`try_claim`] that still lets delivery proceed when the queue directory cannot +/// accept a claim file (full or read-only); see [`crate::claim::try_claim_lenient`]. +pub fn try_claim_lenient(report: &QueuedReport) -> Option> { + crate::claim::try_claim_lenient(&report.claim_path()) +} + fn sibling(bundle: &Path, ext: &str) -> PathBuf { let mut s = bundle.as_os_str().to_os_string(); s.push("."); diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index ca21356..65f8fd0 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -349,8 +349,15 @@ pub fn bump_attempts(data_dir: &Path, pending: &PendingSession) -> u32 { /// Claim a pending session for recovery, so that of several processes sharing the /// data directory exactly one recovers it. The claim lives beside the liveness /// marker and is released when the returned guard drops. -pub fn claim_pending(data_dir: &Path, pending: &PendingSession) -> Option { - crate::claim::try_claim( +/// +/// Lenient (see [`crate::claim::try_claim_lenient`]): a full or read-only +/// directory cannot take a claim file, and that is exactly when a stranded crash +/// most needs recovering. +pub fn claim_pending( + data_dir: &Path, + pending: &PendingSession, +) -> Option> { + crate::claim::try_claim_lenient( &data_dir .join("sessions") .join(format!("{}.claim", pending.generation)), diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 232e8f8..57a9864 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -25,6 +25,7 @@ //! under back-pressure) so a burst or a stalled worker cannot grow RSS without //! limit. +use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender}; @@ -247,6 +248,12 @@ struct Shared { /// stays here until its delivery is settled, so the length is the real backlog. mem_queue: ForkLock>, next_mem_id: AtomicU64, + /// Crash signatures the server refused as too similar, remembered in memory + /// (the on-disk `.blacklist` is unavailable or unreadable with no usable disk). + mem_blacklist: ForkLock>, + /// Queued bundles that were delivered but could not be deleted (read-only or + /// full directory). Skipped from then on, or every drain would send them again. + delivered_stuck: ForkLock>, /// Set the first time a disk write fails while running in disk mode: the /// recorder is no longer fully durable (see [`Recorder::is_degraded`]). degraded: AtomicBool, @@ -260,8 +267,12 @@ impl ForkLockable for Shared { self.scope.fork_lock(); self.session.fork_lock(); self.mem_queue.fork_lock(); + self.mem_blacklist.fork_lock(); + self.delivered_stuck.fork_lock(); } fn fork_unlock(&self) { + self.delivered_stuck.fork_unlock(); + self.mem_blacklist.fork_unlock(); self.mem_queue.fork_unlock(); self.session.fork_unlock(); self.scope.fork_unlock(); @@ -352,6 +363,8 @@ impl Recorder { paused: AtomicBool::new(false), mem_queue: ForkLock::new(Vec::new()), next_mem_id: AtomicU64::new(0), + mem_blacklist: ForkLock::new(BTreeSet::new()), + delivered_stuck: ForkLock::new(BTreeSet::new()), degraded: AtomicBool::new(false), }); @@ -385,10 +398,16 @@ impl Recorder { Some((session, store)) => (Some(session), Some(store)), None => (None, None), }; + // A directory that cannot take NEW files may still hold reports and crashes + // a previous run persisted (that is what a full disk looks like), and + // recovering them only reads and deletes. So an existing directory is + // still drained and recovered from in memory mode. + let drain_dir = (session.is_some() || data_dir.is_dir()).then_some(data_dir.as_path()); let pipeline = spawn_pipeline( &shared, store.is_some().then_some(data_dir.as_path()), - session.as_ref().map_or(0, Session::generation), + drain_dir, + session.as_ref().map_or(u64::MAX, Session::generation), store, config.caps, config.rotate_interval, @@ -454,6 +473,8 @@ impl Recorder { // The parent's unsent in-memory reports are the PARENT's to send; the child // inheriting them would deliver every one a second time. self.shared.mem_queue.lock().clear(); + self.shared.mem_blacklist.lock().clear(); + self.shared.delivered_stuck.lock().clear(); // A child that cannot open its own session degrades to memory rather than // failing: the parent's disk may be gone or read-only by now. @@ -475,7 +496,8 @@ impl Recorder { let pipeline = spawn_pipeline( &self.shared, store.is_some().then_some(self.data_dir.as_path()), - session.as_ref().map_or(0, Session::generation), + store.is_some().then_some(self.data_dir.as_path()), + session.as_ref().map_or(u64::MAX, Session::generation), store, self.caps, self.rotate_interval, @@ -820,6 +842,7 @@ fn open_disk(data_dir: &Path, caps: WindowCaps) -> std::io::Result<(Session, Par fn spawn_pipeline( shared: &Arc, data_dir: Option<&Path>, + drain_dir: Option<&Path>, generation: u64, store: Option, caps: WindowCaps, @@ -832,7 +855,7 @@ fn spawn_pipeline( // Uploader thread: drains the durable queue with retry. let (upload_tx, upload_rx) = channel(); let uploader_shared = Arc::clone(shared); - let uploader_data_dir = data_dir.map(Path::to_path_buf); + let uploader_data_dir = drain_dir.map(Path::to_path_buf); let uploader_legacy = legacy_data_dir.clone(); // Set once the worker has finished recovering the legacy directory, so the // uploader does not retire it while recovery is still about to fill its queue. @@ -856,6 +879,8 @@ fn spawn_pipeline( let (tx, rx) = channel(); let worker_shared = Arc::clone(shared); let worker_data_dir = data_dir.map(Path::to_path_buf); + let worker_drain_dir = drain_dir.map(Path::to_path_buf); + let marker = data_dir.map(|d| Session::marker_path(d, generation)); let worker_upload_tx = upload_tx.clone(); let worker = std::thread::Builder::new() @@ -869,7 +894,7 @@ fn spawn_pipeline( // first (runs before this session enqueues anything). Only a fresh // launch: a fork child's parent may be mid-enqueue right now, and // its in-flight sidecars would look orphaned. - if let (true, Some(dir)) = (fresh_launch, &worker_data_dir) { + if let (true, Some(dir)) = (fresh_launch, &worker_drain_dir) { queue::gc_orphans(dir); } // A fork child recovers too. A prefork master recovers only at ITS @@ -877,7 +902,7 @@ fn spawn_pipeline( // recover a worker that died later; the next worker started from // it must, or that crash waits for the master to restart. Claims // keep siblings from recovering the same session twice. - if let Some(dir) = &worker_data_dir { + if let Some(dir) = &worker_drain_dir { run_recovery(&worker_shared, dir, generation); } // Best effort, once: sessions a previous default location left @@ -895,6 +920,7 @@ fn spawn_pipeline( worker_shared, rotate_interval, worker_data_dir, + marker, caps, sampler, worker_upload_tx, @@ -957,7 +983,7 @@ fn recover_one(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSess merge_scope(shared, &mut built.meta); let window = TimeWindow { start, end }; - let delivered = reporting::assemble_with_extras( + let assembled = reporting::assemble_with_extras( &pending.parts_dir, window, &built.meta, @@ -966,9 +992,22 @@ fn recover_one(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSess &built.extra_files, &shared.app_token, end, - ) - .and_then(|assembled| queue::enqueue(data_dir, &assembled)) - .is_ok(); + ); + // A full or read-only directory cannot take the queue entry; deliver from + // memory instead of losing the crash for this launch. The evidence stays on + // disk (nothing to discard it with), so a later launch with a healthy disk + // reports it again and the backend deduplicates. + let delivered = match assembled { + Ok(assembled) => { + if queue::enqueue(data_dir, &assembled).is_ok() { + true + } else { + hold_in_memory(shared, assembled); + false + } + } + Err(_) => false, + }; // Only destroy the crash evidence once it is safely queued. if delivered { @@ -999,6 +1038,7 @@ fn worker_loop( shared: Arc, rotate_interval: Duration, data_dir: Option, + marker: Option, caps: WindowCaps, mut sampler: Option>, upload_tx: Sender, @@ -1092,11 +1132,30 @@ fn worker_loop( // elapsed β€” via a message OR a timeout β€” so steady traffic can't starve it. if Instant::now() >= next_rotate { rotate_tick(store.as_mut(), &shared, &mut sampler); + if let Some(marker) = &marker { + ensure_marker(&shared, marker); + } next_rotate = Instant::now() + rotate_interval; } } } +/// Make sure this session's liveness marker still exists. If the data directory +/// was removed and recreated under us (`create_dir_all` heals the tree but not +/// the marker), the next launch would not see this session as live, and its +/// crash would be dropped as an orphan. Rewrite it and mark the recorder +/// degraded: the capture window it had is gone with the removed files. +fn ensure_marker(shared: &Shared, marker: &Path) { + if marker.exists() { + return; + } + shared.degraded.store(true, Ordering::Relaxed); + if let Some(parent) = marker.parent() { + let _ = std::fs::create_dir_all(parent); + } + let _ = std::fs::write(marker, std::process::id().to_string()); +} + /// One rotation tick: sample telemetry (unless paused) then roll + evict the /// window. Panic-contained so a host `TelemetrySampler` panic can't kill the /// worker. @@ -1224,6 +1283,14 @@ fn assemble_in_memory( ) } +fn mem_blacklisted(shared: &Shared, sigs: &[String]) -> bool { + if sigs.is_empty() { + return false; + } + let set = shared.mem_blacklist.lock(); + sigs.iter().any(|s| set.contains(s)) +} + /// Park a report in the bounded in-memory queue for the uploader. Over the cap /// the NEWEST report is dropped (and the host told), keeping the backlog bounded. fn hold_in_memory(shared: &Shared, report: AssembledReport) { @@ -1273,11 +1340,12 @@ fn process_report( return Ok(()); } - // Suppress reports the server blacklisted (12004 TooManySimilar). - if let Some(dir) = data_dir { - if queue::any_blacklisted(dir, &meta.signatures) { - return Ok(()); - } + // Suppress reports the server blacklisted (12004 TooManySimilar), from the + // in-memory set (the only one with no usable disk) or the on-disk one. + if mem_blacklisted(shared, &meta.signatures) + || data_dir.is_some_and(|dir| queue::any_blacklisted(dir, &meta.signatures)) + { + return Ok(()); } let window_start = window_end.saturating_sub(caps.max_window_ms); @@ -1402,6 +1470,18 @@ fn uploader_loop( let _ = ack.send(empty); } Ok(UploadMsg::Stop(ack)) => { + // A report held in memory has no next launch to be retried by, so + // give it one last bounded attempt (the stopper waits 2 s for the + // ack). Disk reports survive in `queue/` and need nothing here. + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + drain_memory( + &shared, + backoff_base, + true, + false, + Some(Instant::now() + Duration::from_millis(1500)), + ) + })); let _ = ack.send(()); break; } @@ -1418,19 +1498,30 @@ fn uploader_loop( } Err(RecvTimeoutError::Disconnected) => break, } - retire_legacy(&mut legacy, &legacy_ready); + retire_legacy(&shared, &mut legacy, &legacy_ready); } } +/// Queued reports in `dir` still awaiting delivery: everything on disk except +/// bundles already delivered whose deletion failed. +fn pending_reports(shared: &Shared, dir: &Path) -> Vec { + let mut all = queue::list_pending(dir); + let stuck = shared.delivered_stuck.lock(); + if !stuck.is_empty() { + all.retain(|r| !stuck.contains(&r.bundle)); + } + all +} + /// Stop polling the legacy directory once recovery has run (it runs once per /// launch, so nothing new can appear) and its queue is empty. -fn retire_legacy(legacy: &mut Option, ready: &AtomicBool) { +fn retire_legacy(shared: &Shared, legacy: &mut Option, ready: &AtomicBool) { if !ready.load(Ordering::SeqCst) { return; } if legacy .as_deref() - .is_some_and(|dir| queue::list_pending(dir).is_empty()) + .is_some_and(|dir| pending_reports(shared, dir).is_empty()) { *legacy = None; } @@ -1466,9 +1557,9 @@ fn drain(shared: &Shared, data_dir: Option<&Path>, backoff_base: Duration, force fn drain_disk(shared: &Shared, data_dir: &Path, backoff_base: Duration, force: bool) -> bool { let now = epoch_ms(); let mut attempted = false; - for report in queue::list_pending(data_dir) { + for report in pending_reports(shared, data_dir) { // Another process sharing the queue may be delivering this one. - let Some(_claim) = queue::try_claim(&report) else { + let Some(_claim) = queue::try_claim_lenient(&report) else { continue; }; let Ok((zip, request_json, retry, next_attempt)) = queue::load(&report) else { @@ -1537,7 +1628,13 @@ fn deliver_queued( cached_endpoint.as_deref(), &mut endpoint_out, ) { - Ok(()) => queue::remove(report), + Ok(()) => { + queue::remove(report); + if report.bundle.exists() { + // Delivered, but the directory will not let us delete it. + shared.delivered_stuck.lock().insert(report.bundle.clone()); + } + } // A transient failure OR a still-expired session are both retryable. Err(TransportError::Transient(_)) | Err(TransportError::SessionExpired) => { // Persist the presigned endpoint if the failure left one to resume @@ -1567,6 +1664,7 @@ fn deliver_queued( signatures }; queue::blacklist_add(data_dir, &sigs); + shared.mem_blacklist.lock().extend(sigs); queue::remove(report); notify_dropped(shared, DropReason::TooManySimilar); } @@ -1665,9 +1763,17 @@ fn drain_memory( } } } - Err(TransportError::TooManySimilar { .. }) => { + Err(TransportError::TooManySimilar { signatures }) => { queue.remove(pos); dropped = Some(DropReason::TooManySimilar); + // Stop the same fault from being sent again: no disk to blacklist on. + let sigs = if signatures.is_empty() { + queue::signatures_of(&report) + } else { + signatures + }; + // Lock order: the queue lock is held; the blacklist is a leaf. + shared.mem_blacklist.lock().extend(sigs); } Err(e) => { queue.remove(pos); @@ -1709,8 +1815,8 @@ fn drain_until_empty( let now = epoch_ms(); drain_memory(shared, backoff_base, true, false, Some(start + deadline)); for &data_dir in dirs { - for report in queue::list_pending(data_dir) { - let Some(_claim) = queue::try_claim(&report) else { + for report in pending_reports(shared, data_dir) { + let Some(_claim) = queue::try_claim_lenient(&report) else { continue; // a peer is delivering it; the loop below waits it out }; let Ok((zip, request_json, retry, _next)) = queue::load(&report) else { @@ -1732,7 +1838,7 @@ fn drain_until_empty( } } } - let disk_empty = dirs.iter().all(|d| queue::list_pending(d).is_empty()); + let disk_empty = dirs.iter().all(|d| pending_reports(shared, d).is_empty()); if disk_empty && shared.mem_queue.lock().is_empty() { return true; } diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index d8ab57a..175de28 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -432,3 +432,225 @@ fn a_failed_put_retries_at_the_put_and_does_not_mint_a_duplicate_issue() { "the retry must resume at the PUT, not create the issue again" ); } + +/// Lay down what a previous run left behind: a queued report nobody delivered and +/// a session that died (marker left, owner gone). +fn seed_leftovers(dir: &Path) { + use bugsee_core::capture::record; + use bugsee_core::model::report::TimeWindow; + use bugsee_core::reporting::assemble_docs; + + let env = bugsee_core::model::environment::Environment::detect_with( + "t", + &Default::default(), + &Default::default(), + ); + let window = TimeWindow { start: 0, end: 1 }; + let queued = assemble_docs( + Vec::new(), + window, + &manual_upload_meta(), + &env, + None, + &[], + "T", + 1, + ) + .unwrap(); + bugsee_core::queue::enqueue(dir, &queued).unwrap(); + + let part = dir.join("parts").join("1").join("0"); + std::fs::create_dir_all(&part).unwrap(); + let mut bytes = Vec::new(); + let payload = br#"{"timestamp":100,"level":1,"source":2,"message":"earlier crash"}"#; + record::frame(100, payload, &mut bytes); + std::fs::write(part.join("log.part"), &bytes).unwrap(); + std::fs::create_dir_all(dir.join("sessions")).unwrap(); + std::fs::write(dir.join("sessions").join("1.alive"), "0").unwrap(); + std::fs::write(dir.join("gen"), "1").unwrap(); +} + +fn wait_for_bundles(transport: &MockTransport, n: usize) -> usize { + let start = std::time::Instant::now(); + while transport.uploaded_bundles.lock().unwrap().len() < n + && start.elapsed() < Duration::from_secs(10) + { + std::thread::sleep(Duration::from_millis(20)); + } + // Settle: anything delivered again would show up now. + std::thread::sleep(Duration::from_millis(400)); + let got = transport.uploaded_bundles.lock().unwrap().len(); + got +} + +/// A directory that cannot take new files can still hold reports and crashes a +/// previous run persisted (that is what a full disk looks like). They must be +/// delivered, and not again on every drain. +#[cfg(unix)] +#[test] +fn leftovers_in_a_directory_that_cannot_take_new_files_are_still_delivered() { + use std::os::unix::fs::PermissionsExt; + for memory_only in [false, true] { + let dir = TempDir::new(); + let data = dir.path.join("data"); + std::fs::create_dir_all(&data).unwrap(); + seed_leftovers(&data); + for d in [data.join("queue"), data.join("sessions"), data.clone()] { + std::fs::set_permissions(d, std::fs::Permissions::from_mode(0o555)).unwrap(); + } + // root ignores permission bits, which would make this test vacuous. + if std::fs::write(data.join("probe"), b"x").is_ok() { + eprintln!("skipped: permission bits are not enforced for this user"); + return; + } + + let transport = Arc::new(MockTransport::default()); + let mut config = RecorderConfig::new(&data, "T"); + if memory_only { + // What the facade does once its write probe has failed. + config.memory_only = Some("write probe failed".into()); + } + let recorder = Recorder::launch(config, transport.clone()).expect("launch"); + assert_eq!(recorder.storage_mode(), StorageMode::Memory); + + assert_eq!( + wait_for_bundles(&transport, 2), + 2, + "the queued report and the recovered crash, each delivered once \ + (memory_only={memory_only})" + ); + drop(recorder); + restore_permissions(&data); + restore_permissions(&data.join("queue")); + restore_permissions(&data.join("sessions")); + } +} + +#[test] +fn a_too_similar_verdict_is_remembered_without_a_disk() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct Refuse(Arc); + impl bugsee_core::Transport for Refuse { + fn register_session( + &self, + _: &str, + _: &[u8], + ) -> Result { + Ok("token".into()) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Err(bugsee_core::TransportError::TooManySimilar { + signatures: vec!["sig-loop".into()], + }) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), bugsee_core::TransportError> { + Ok(()) + } + } + + let dir = TempDir::new(); + let issues = Arc::new(AtomicUsize::new(0)); + let recorder = Recorder::launch( + RecorderConfig::new(blocked_path(&dir.path), "T"), + Arc::new(Refuse(issues.clone())), + ) + .expect("launch"); + for _ in 0..3 { + let mut meta = manual_upload_meta(); + meta.signatures = vec!["sig-loop".into()]; + recorder.report(meta, None); + assert!(recorder.flush(Duration::from_secs(10))); + } + assert_eq!( + issues.load(Ordering::SeqCst), + 1, + "after the first refusal the same fault must not be sent again" + ); +} + +#[test] +fn a_clean_shutdown_gives_unsent_memory_reports_a_last_attempt() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + // The first attempt fails, so the report is still unsent when we stop. + struct FailOnce { + attempts: AtomicUsize, + delivered: AtomicUsize, + } + struct Handle(Arc); + impl bugsee_core::Transport for Handle { + fn register_session( + &self, + _: &str, + _: &[u8], + ) -> Result { + Ok("token".into()) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + if self.0.attempts.fetch_add(1, Ordering::SeqCst) == 0 { + Err(bugsee_core::TransportError::Transient("blip".into())) + } else { + Ok("https://uploads.example/x".into()) + } + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), bugsee_core::TransportError> { + self.0.delivered.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + + let dir = TempDir::new(); + let state = Arc::new(FailOnce { + attempts: AtomicUsize::new(0), + delivered: AtomicUsize::new(0), + }); + let recorder = Recorder::launch( + RecorderConfig::new(blocked_path(&dir.path), "T"), + Arc::new(Handle(state.clone())), + ) + .expect("launch"); + recorder.report(manual_upload_meta(), None); + // No flush: just drop, as a `LaunchGuard` does. + drop(recorder); + assert_eq!(state.delivered.load(Ordering::SeqCst), 1); +} + +#[test] +fn a_directory_wiped_and_recreated_mid_run_gets_its_liveness_marker_back() { + let dir = TempDir::new(); + let data = dir.path.join("data"); + let transport = Arc::new(MockTransport::default()); + let mut config = RecorderConfig::new(&data, "T"); + config.rotate_interval = Duration::from_millis(50); + let recorder = Recorder::launch(config, transport).expect("launch"); + + std::fs::remove_dir_all(&data).unwrap(); + // The rotation tick recreates the tree (`create_dir_all`) but the marker + // only comes back if the SDK rewrites it. + let start = std::time::Instant::now(); + let alive = || { + std::fs::read_dir(data.join("sessions")) + .map(|rd| { + rd.flatten() + .any(|e| e.file_name().to_string_lossy().ends_with(".alive")) + }) + .unwrap_or(false) + }; + while !alive() && start.elapsed() < Duration::from_secs(5) { + std::thread::sleep(Duration::from_millis(20)); + } + assert!(alive(), "a crash after the wipe must still be recoverable"); + assert!(recorder.is_degraded(), "the lost window is a degradation"); +} diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index a693e7f..985b567 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -88,6 +88,7 @@ impl Bugsee { source, skipped, explicit_rejected, + recovery_dir, legacy_dir, } = options.resolve_data_dir(); let memory_only = dir.is_none().then(|| { @@ -96,9 +97,9 @@ impl Bugsee { .or_else(|| (!skipped.is_empty()).then(|| skipped.join("; "))) .unwrap_or_else(|| "no usable data directory".to_string()) }); - // Memory mode never touches the path; an empty one avoids resolving a - // temp directory just to ignore it. - let data_dir = dir.unwrap_or_default(); + // In memory mode no new file is written there, but an existing directory + // is still drained and recovered from; with none, an empty path. + let data_dir = dir.or(recovery_dir).unwrap_or_default(); if memory_only.is_none() && !skipped.is_empty() { static WARNED: AtomicBool = AtomicBool::new(false); bugsee_core::util::warn_once( diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index e4de340..a77619c 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -292,6 +292,10 @@ pub(crate) struct DataDirResolution { pub skipped: Vec, /// Why an explicit / environment directory was rejected (it is final). pub explicit_rejected: Option, + /// A rejected candidate that nevertheless EXISTS as a directory (full or + /// read-only). It may hold reports and crashes a previous run persisted, so + /// it is still drained and recovered from; no new files are written to it. + pub recovery_dir: Option, /// The location earlier SDK versions used by default /// (`/bugsee/`), if it exists, is trustworthy and is not `dir`. /// Its leftovers are recovered once at launch. @@ -322,13 +326,15 @@ pub(crate) fn resolve_data_dir( source, skipped: Vec::new(), explicit_rejected: None, + recovery_dir: None, legacy_dir: None, }, Err(e) => DataDirResolution { + explicit_rejected: Some(format!("{}: {e}", dir.display())), + recovery_dir: dir.is_dir().then_some(dir), dir: None, source: DataDirSource::None, skipped: Vec::new(), - explicit_rejected: Some(format!("{}: {e}", dir.display())), legacy_dir: None, }, }; @@ -352,27 +358,36 @@ pub(crate) fn resolve_data_dir( let mut skipped = Vec::new(); let mut found = None; + let mut recovery_dir = None; for (source, dir, private) in candidates { match prepare_dir(&dir, private) { Ok(()) => { found = Some((source, dir)); break; } - Err(e) => skipped.push(format!("{}: {e}", dir.display())), + Err(e) => { + skipped.push(format!("{}: {e}", dir.display())); + // Only the user's own default location: the shared temp area is + // exactly where a planted directory would be refused. + if !private && recovery_dir.is_none() && dir.is_dir() { + recovery_dir = Some(dir); + } + } } } let legacy = temp.join("bugsee").join(&token); - let legacy_dir = found - .as_ref() - .filter(|(_, dir)| legacy_is_adoptable(&legacy, dir)) - .map(|_| legacy); + // Also with no usable directory: leftovers there are exactly what a launch + // that cannot write anywhere new should still deliver. + let chosen = found.as_ref().map(|(_, dir)| dir.as_path()); + let legacy_dir = legacy_is_adoptable(&legacy, chosen).then_some(legacy); match found { Some((source, dir)) => DataDirResolution { dir: Some(dir), source, skipped, explicit_rejected: None, + recovery_dir: None, legacy_dir, }, None => DataDirResolution { @@ -380,7 +395,8 @@ pub(crate) fn resolve_data_dir( source: DataDirSource::None, skipped, explicit_rejected: None, - legacy_dir: None, + recovery_dir, + legacy_dir, }, } } @@ -492,16 +508,20 @@ fn owned_by_us(_meta: &std::fs::Metadata) -> bool { /// would let someone else's tree, or our own live data directory, be processed /// (recovering our live session as a crash). Paths are compared canonicalised, /// not as strings, so another spelling of our own directory is rejected too. -fn legacy_is_adoptable(legacy: &Path, chosen: &Path) -> bool { +fn legacy_is_adoptable(legacy: &Path, chosen: Option<&Path>) -> bool { let Some(parent) = legacy.parent() else { return false; }; if !trusted_existing_dir(parent) || !trusted_existing_dir(legacy) { return false; } - match (legacy.canonicalize(), chosen.canonicalize()) { - (Ok(a), Ok(b)) => a != b, - _ => false, + let Ok(legacy) = legacy.canonicalize() else { + return false; + }; + match chosen { + // No live directory of ours to collide with. + None => true, + Some(chosen) => chosen.canonicalize().is_ok_and(|c| c != legacy), } } @@ -807,6 +827,50 @@ mod tests { let _ = std::fs::remove_dir_all(&s); } + #[cfg(unix)] + #[test] + fn a_rejected_directory_that_exists_is_kept_for_recovery() { + use std::os::unix::fs::PermissionsExt; + let s = scratch(); + let ro = s.join("ro"); + std::fs::create_dir_all(&ro).unwrap(); + std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o555)).unwrap(); + if std::fs::write(ro.join("p"), b"x").is_ok() { + let _ = std::fs::remove_dir_all(&s); + return; // permission bits not enforced (root) + } + let r = resolve_data_dir(Some(&ro), "tok", &env_of(&[]), &|| s.join("tmp")); + assert!(r.dir.is_none() && r.explicit_rejected.is_some()); + assert_eq!( + r.recovery_dir.as_deref(), + Some(ro.as_path()), + "its queue and crashed sessions must still be drained" + ); + + // A missing directory has nothing to recover. + let r = resolve_data_dir(Some(&blocked(&s, "f5")), "tok", &env_of(&[]), &|| { + s.join("tmp") + }); + assert!(r.recovery_dir.is_none()); + let _ = std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o755)); + let _ = std::fs::remove_dir_all(&s); + } + + #[cfg(unix)] + #[test] + fn the_legacy_location_is_adopted_even_when_no_new_default_is_usable() { + let s = scratch(); + let tmp = s.join("tmp"); + let legacy = tmp.join("bugsee").join("tok"); + std::fs::create_dir_all(&legacy).unwrap(); + // The private temp name is occupied by a symlink, and there is no home. + std::os::unix::fs::symlink(s.join("elsewhere"), tmp.join(private_temp_name())).unwrap(); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); + assert!(r.dir.is_none(), "no new directory is usable"); + assert_eq!(r.legacy_dir, Some(legacy)); + let _ = std::fs::remove_dir_all(&s); + } + #[test] fn an_explicit_directory_never_asks_for_the_temp_location() { let s = scratch(); diff --git a/crates/bugsee/tests/storage_fallback.rs b/crates/bugsee/tests/storage_fallback.rs index 32128d8..35fbefc 100644 --- a/crates/bugsee/tests/storage_fallback.rs +++ b/crates/bugsee/tests/storage_fallback.rs @@ -112,5 +112,62 @@ fn launch_degrades_through_the_resolution_order() { ); drop(guard); + // 5. A requested directory that cannot take new files but already holds a + // queued report (a full or read-only volume): memory mode, yet the + // report a previous run left there is still delivered. + { + use bugsee::core::model::report::TimeWindow; + use std::os::unix::fs::PermissionsExt; + + let ro = s.join("ro-volume"); + let env = bugsee::core::model::environment::Environment::detect_with( + "t", + &Default::default(), + &Default::default(), + ); + let queued = bugsee::core::reporting::assemble_docs( + Vec::new(), + TimeWindow { start: 0, end: 1 }, + &bugsee::core::reporting::manual_upload_meta(), + &env, + None, + &[], + "TOKEN_A", + 1, + ) + .unwrap(); + bugsee::core::queue::enqueue(&ro, &queued).unwrap(); + for d in [ro.join("queue"), ro.clone()] { + std::fs::set_permissions(d, std::fs::Permissions::from_mode(0o555)).unwrap(); + } + if std::fs::write(ro.join("probe"), b"x").is_err() { + let transport = Arc::new(MockTransport::default()); + let guard = Bugsee::launch_with( + LaunchOptions::new("TOKEN_A") + .data_dir(&ro) + .with_transport(transport.clone()), + ) + .expect("launch"); + let status = Bugsee::storage_status().expect("status"); + assert_eq!(status.mode, StorageMode::Memory); + assert!(status.explicit_rejected.is_some()); + let start = std::time::Instant::now(); + while transport.uploaded_bundles.lock().unwrap().is_empty() + && start.elapsed() < Duration::from_secs(10) + { + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!( + transport.uploaded_bundles.lock().unwrap().len(), + 1, + "the stranded report must be delivered" + ); + drop(guard); + } + for d in [ro.join("queue"), ro.clone()] { + let _ = std::fs::set_permissions(d, std::fs::Permissions::from_mode(0o755)); + } + } + let _ = std::fs::remove_dir_all(&s); } From 10271f3187f81b463900d5ed2be891271414b9ad Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:52:50 +0500 Subject: [PATCH 07/15] fix(core): harden the file-lock claims (second review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A launch that cannot hold .lock removes the file instead of leaving an unheld one, which every peer read as "that session died" and recovered (and deleted) the live session. Contended claims are retried briefly; an empty, fresh marker with a free lock is skipped (the pid is written after locking); on unix a probe uses F_GETLK and never takes the lock. - Filesystems that reject locks (ENOLCK, EOPNOTSUPP, ENOSYS, EINVAL) degrade to thread-only claims with a one-time message, and probes report unknown so recovery falls back to the marker pid, instead of blocking the queue forever. - An unclean shutdown (detached worker/uploader) keeps the session lock until the process exits so peers cannot recover a session still being written. - Registry keys are canonicalised; a fork child resets the registry from the fork hook; old-scheme .claim/.claim.takeover files are swept. - Tests: the real pid-namespace hazard (marker pid that does not exist here, reaped pid), crashed sessions with free lock files, the lock-race launch, unsupported-lock injection, and the probe/launch ordering cases. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 7 +- crates/bugsee-core/src/claim.rs | 293 +++++++++++++++++++---- crates/bugsee-core/src/queue.rs | 2 +- crates/bugsee-core/src/recovery.rs | 126 +++++++++- crates/bugsee-core/src/runtime.rs | 10 + crates/bugsee-core/src/session.rs | 100 +++++++- crates/bugsee-core/tests/multiprocess.rs | 60 ++++- 7 files changed, 545 insertions(+), 53 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index a2d1009..41ab698 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -204,10 +204,13 @@ Crash recovery needs the data directory to **outlive the process**: on a contain - **Pid namespaces.** Containers sharing a volume each see their app as pid 1; a pid (or pid plus start time) written in a file means nothing to the other. A lock does not depend on it, so a peer's live session is never mistaken for a crash and a dead one is never kept alive by pid collision. The `.alive` marker's pid is consulted only for a marker with no `.lock` beside it, i.e. one written by an older SDK build. - **Why `fcntl`, not `flock`/OFD locks.** `fcntl` locks are per process and are not inherited across `fork`, so a prefork child never holds its parent's claim; `flock`/OFD locks follow the open file description and would. (MSRV 1.88 predates `File::lock` anyway, hence libc/FFI.) - **Costs of per-process locks, handled in `claim.rs`.** They do not exclude threads, so a registry of held paths does (keyed by pid and spin-locked with the holder's pid stamped in, so a `fork` child that inherits it mid-update breaks the lock instead of deadlocking). POSIX drops a process's locks on a file when *any* descriptor for it closes, so a lock file is never opened twice by one process: the registry is checked before each open, and a claim closes its descriptor before leaving the registry. -- **Ordering at launch.** `Session::begin` creates the marker (`O_EXCL`), takes `.lock`, then writes the pid. Until the pid is written the marker is empty and fresh, which peers skip. +- **Ordering at launch.** `Session::begin` creates the marker (`O_EXCL`), takes `.lock`, then writes the pid. Until the pid is written the marker is empty and fresh, which peers skip even if the lock reads free. A lock that cannot be held with kernel enforcement (contended past a short retry, or the filesystem rejects locks) is *removed*: an existing but unheld lock file reads as "that session died", so peers fall back to the marker's pid instead. On unix a probe asks `F_GETLK` and never takes the lock, so it cannot steal one from a launching peer. +- **Unclean shutdown.** If the worker or uploader thread is detached at shutdown it may still be writing the session, so the session's lock is kept until the process exits (the kernel then drops it) rather than released on drop. +- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown so recovery uses the marker's pid. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. +- **Paths.** The in-process registry canonicalises the lock file's directory, so `/tmp` vs `/private/tmp` or relative vs absolute paths are one claim. A `fork` child clears the registry (`claim::reset_after_fork`, from the fork hook) before claiming anything. - **Stale scans.** After winning a session claim the marker is re-checked, since a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s: a peer's in-flight `enqueue` looks like an orphan, and reaping it lost the report. Lock files of finished items and dead sessions are swept when free. - **Compatibility.** The `.alive` marker is still a bare pid and the lock files are new names, so older builds sharing the directory keep working. They do not take part in the locking, so a mixed-version directory (a rolling upgrade) gives no exactly-once guarantee between old and new processes, and a new process treats an old build's session by its pid as before. Run one SDK version per directory for the guarantee. -- **Limits.** Advisory locks on network filesystems are best effort (NFS needs lockd; some mounts do not support them, in which case a claim fails closed: the item is skipped, never double-handled). The only timestamps relied on are the 30 s grace for an empty marker and the 60 s orphan grace, which assume clocks within seconds. Delivery is at-least-once: a holder that dies after uploading but before removing the item (or after enqueueing a recovered report but before discarding the session) makes its successor do it again. Platforms with neither unix nor Windows locks exclude threads only. +- **Limits.** Advisory locks on network filesystems are best effort (see above). The only timestamps relied on are the 30 s grace for an empty marker and the 60 s orphan grace, which assume clocks within seconds. Delivery is at-least-once: a holder that dies after uploading but before removing the item (or after enqueueing a recovered report but before discarding the session) makes its successor do it again. Platforms with neither unix nor Windows locks exclude threads only. Tests: `bugsee-core/tests/multiprocess.rs` (real child processes: racing launchers, a SIGKILLed holder, a live peer whose marker names a colliding pid) and unit tests in `claim.rs`, `process.rs`, `queue.rs`. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index ea80538..4911f0a 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -31,27 +31,48 @@ //! * They do not exclude *threads* of one process, so an in-process registry of //! held paths does that. //! * POSIX drops a process's locks on a file when it closes **any** descriptor -//! for that file. So a lock file is never opened twice by one process: the -//! registry is consulted before every open, and a claim closes its descriptor -//! *before* leaving the registry. +//! for that file. So a lock file is never opened twice by one process: every +//! open (claim or probe) registers the path first, and a claim closes its +//! descriptor *before* leaving the registry. Registry keys are canonicalised +//! (`/tmp` and `/private/tmp`, relative and absolute, are one file). //! -//! Advisory locks on network filesystems are best effort (NFS needs lockd). The -//! SDK relies on no file timestamps for claims; the only ones it reads are the -//! grace windows for half-written files in `recovery` and `queue`, which assume -//! clocks that agree to within seconds. +//! **Unsupported locks.** Some filesystems reject byte-range locks outright (NFS +//! without lockd: `ENOLCK`; FUSE/9p: `EOPNOTSUPP`/`ENOSYS`/`EINVAL`). The base +//! scheme (`O_EXCL` files) worked there, so refusing to claim would be a +//! regression: a queue that never drains. Instead such a claim degrades to +//! thread-only exclusion (announced once on stderr), and a probe reports +//! [`Probe::Unknown`] so the caller falls back to its pid check. +//! +//! **Locks that succeed but do not exclude** (Docker Desktop bind mounts, WSL2 +//! drvfs/9p, NFS mounted `nolock`) cannot be detected. Processes sharing a +//! directory over such a mount can deliver a report twice; run one SDK host per +//! directory there. +//! +//! The SDK relies on no file timestamps for claims; the only ones it reads are +//! the grace windows for half-written files in `recovery` and `queue`, which +//! assume clocks that agree to within seconds. //! //! None of this runs on the crash path; it allocates freely. use std::cell::UnsafeCell; use std::fs::{File, OpenOptions}; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; /// Holds a claim; dropping it releases the claim. #[derive(Debug)] pub struct Claim { - path: PathBuf, + key: PathBuf, file: Option, + os_locked: bool, +} + +impl Claim { + /// Whether the kernel enforces this claim against other processes. `false` + /// means the filesystem rejected locks and only threads are excluded. + pub(crate) fn is_os_locked(&self) -> bool { + self.os_locked + } } impl Drop for Claim { @@ -59,14 +80,14 @@ impl Drop for Claim { // Close (releasing the OS lock) strictly before leaving the registry, so // a thread that acquires next never has its lock dropped by our close. drop(self.file.take()); - registry().remove(&self.path); + registry().remove(&self.key); } } /// Try to claim `path` (created if absent). `None` if another process, or another /// thread of this one, holds it, or if it cannot be claimed. pub fn try_claim(path: &Path) -> Option { - match acquire(path, true) { + match acquire(path) { Acquire::Got(claim) => Some(claim), Acquire::Held | Acquire::Unavailable => None, } @@ -76,23 +97,40 @@ pub fn try_claim(path: &Path) -> Option { pub(crate) enum Probe { Held, Free, + /// The filesystem cannot say (it rejects locks): use another liveness signal. + Unknown, } /// Is `path` currently claimed? A missing file is free. A file that cannot be /// inspected is reported held: the conservative answer never disturbs a peer. +/// +/// On unix this asks (`F_GETLK`) without taking the lock, so a probe can never +/// steal the lock from a creator that has made the file but not yet locked it. pub(crate) fn probe(path: &Path) -> Probe { if !path.exists() { return Probe::Free; } - match acquire(path, false) { - Acquire::Got(claim) => { - drop(claim); - Probe::Free + let key = key_of(path); + if !registry().insert(&key) { + return Probe::Held; // held by a thread of this process, or being probed + } + let result = match OpenOptions::new().read(true).write(true).open(path) { + Ok(file) => { + let r = match test_is_locked(&file) { + Ok(Peek::Free) => Probe::Free, + Ok(Peek::Held) => Probe::Held, + Ok(Peek::Unsupported) => Probe::Unknown, + Err(_) => Probe::Held, + }; + drop(file); // close before leaving the registry + r } // Vanished since the existence check: nothing holds it. - Acquire::Unavailable if !path.exists() => Probe::Free, - Acquire::Held | Acquire::Unavailable => Probe::Held, - } + Err(_) if !path.exists() => Probe::Free, + Err(_) => Probe::Held, + }; + registry().remove(&key); + result } /// Remove the lock file at `path` if nobody holds it (a dead holder's leftover). @@ -103,55 +141,132 @@ pub(crate) fn remove_if_free(path: &Path) { } } +/// Forget every registry entry. Called in a `fork` child before it claims +/// anything: the child inherits the parent's registry (entries it does not own, +/// possibly mid-update under the parent's spin lock) and none of its locks. +pub fn reset_after_fork() { + registry().reset(); +} + enum Acquire { Got(Claim), Held, Unavailable, } -fn acquire(path: &Path, create: bool) -> Acquire { - if !registry().insert(path) { +fn acquire(path: &Path) -> Acquire { + let key = key_of(path); + if !registry().insert(&key) { return Acquire::Held; // another thread of this process } let file = match OpenOptions::new() .read(true) .write(true) - .create(create) + .create(true) .truncate(false) .open(path) { Ok(f) => f, Err(_) => { - registry().remove(path); + registry().remove(&key); return Acquire::Unavailable; } }; - match os::try_lock(&file) { - Ok(true) => Acquire::Got(Claim { - path: path.to_path_buf(), + match test_try_lock(&file) { + Ok(Lock::Got) => Acquire::Got(Claim { + key, file: Some(file), + os_locked: true, }), - Ok(false) => { + Ok(Lock::Contended) => { drop(file); - registry().remove(path); + registry().remove(&key); Acquire::Held } + Ok(Lock::Unsupported) => { + static ANNOUNCED: AtomicBool = AtomicBool::new(false); + crate::util::warn_once( + &ANNOUNCED, + "this filesystem does not support file locks; processes sharing the data \ + directory cannot be kept from delivering the same report twice", + ); + Acquire::Got(Claim { + key, + file: Some(file), + os_locked: false, + }) + } Err(_) => { drop(file); - registry().remove(path); + registry().remove(&key); Acquire::Unavailable } } } +/// The registry key for `path`: its directory canonicalised, so every spelling of +/// one file maps to one key. Falls back to the path as given. +fn key_of(path: &Path) -> PathBuf { + match (path.parent(), path.file_name()) { + (Some(parent), Some(name)) => { + let parent = if parent.as_os_str().is_empty() { + Path::new(".") + } else { + parent + }; + std::fs::canonicalize(parent) + .map(|p| p.join(name)) + .unwrap_or_else(|_| path.to_path_buf()) + } + _ => path.to_path_buf(), + } +} + +/// Outcome of an attempt to take a lock. +enum Lock { + Got, + Contended, + Unsupported, +} + +/// Outcome of asking whether a lock is held. +enum Peek { + Free, + Held, + Unsupported, +} + +#[cfg(test)] +thread_local! { + /// Test hook: make this thread's lock calls behave as on a filesystem that + /// rejects locks. + pub(crate) static FORCE_UNSUPPORTED: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +fn test_try_lock(file: &File) -> std::io::Result { + #[cfg(test)] + if FORCE_UNSUPPORTED.with(|f| f.get()) { + return Ok(Lock::Unsupported); + } + os::try_lock(file) +} + +fn test_is_locked(file: &File) -> std::io::Result { + #[cfg(test)] + if FORCE_UNSUPPORTED.with(|f| f.get()) { + return Ok(Peek::Unsupported); + } + os::is_locked(file) +} + /// The paths this process currently holds, keyed by pid so that a `fork` child, /// which inherits a copy of this memory, ignores the parent's entries. /// /// A hand-rolled spin lock rather than a `Mutex`: if another thread held a /// `Mutex` at the instant of `fork`, the child's copy would stay locked forever -/// and its first claim would deadlock. Here the lock word records the holder's -/// pid, so a child that finds it held by a different pid knows the holder does -/// not exist in this process and breaks it. +/// and its first claim would deadlock. The fork hook calls [`reset_after_fork`] +/// to clear it; as a second line the lock word records the holder's pid, so a +/// child that finds it held by a different pid breaks it. struct Registry { owner: AtomicU32, held: UnsafeCell>, @@ -221,32 +336,80 @@ impl Registry { let pid = std::process::id(); self.with(|held| held.retain(|(p, q)| !(*p == pid && q == path))); } + + /// Discard everything, including a lock word left held by a thread that does + /// not exist in this process. Only for a freshly forked, single-threaded child. + fn reset(&self) { + // SAFETY: the caller is the only thread in the process, so nothing else + // can be using the registry; the old vector is leaked, not dropped, as its + // allocator state may be mid-update from the parent's other threads. + unsafe { std::ptr::write(self.held.get(), Vec::new()) }; + self.owner.store(0, Ordering::Release); + } } #[cfg(unix)] mod os { + use super::{Lock, Peek}; use std::fs::File; use std::io; use std::os::unix::io::AsRawFd; - /// Take a non-blocking exclusive lock on byte 0. `Ok(false)` if a peer holds it. - pub fn try_lock(file: &File) -> io::Result { + fn whole() -> libc::flock { // SAFETY: an all-zero `flock` is a valid plain-data value. let mut fl: libc::flock = unsafe { std::mem::zeroed() }; fl.l_type = libc::F_WRLCK as _; fl.l_whence = libc::SEEK_SET as _; fl.l_start = 0; fl.l_len = 1; + fl + } + + /// Errors meaning "this filesystem does not do byte-range locks". + fn is_unsupported(errno: i32) -> bool { + errno == libc::ENOLCK + || errno == libc::ENOSYS + || errno == libc::EINVAL + || errno == libc::EOPNOTSUPP + || errno == libc::ENOTSUP + } + + /// Take a non-blocking exclusive lock on byte 0. + pub fn try_lock(file: &File) -> io::Result { + let fl = whole(); loop { // SAFETY: `fd` is a valid open descriptor and `fl` a valid `flock`. let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETLK, &fl) }; if ret == 0 { - return Ok(true); + return Ok(Lock::Got); } let err = io::Error::last_os_error(); match err.raw_os_error() { - Some(libc::EAGAIN) | Some(libc::EACCES) => return Ok(false), + Some(libc::EAGAIN) | Some(libc::EACCES) => return Ok(Lock::Contended), Some(libc::EINTR) => continue, + Some(e) if is_unsupported(e) => return Ok(Lock::Unsupported), + _ => return Err(err), + } + } + } + + /// Ask whether a peer holds the lock, without taking it. + pub fn is_locked(file: &File) -> io::Result { + let mut fl = whole(); + loop { + // SAFETY: as above; `F_GETLK` only writes the conflicting lock into `fl`. + let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETLK, &mut fl) }; + if ret == 0 { + return Ok(if fl.l_type as i32 == libc::F_UNLCK as i32 { + Peek::Free + } else { + Peek::Held + }); + } + let err = io::Error::last_os_error(); + match err.raw_os_error() { + Some(libc::EINTR) => continue, + Some(e) if is_unsupported(e) => return Ok(Peek::Unsupported), _ => return Err(err), } } @@ -255,6 +418,7 @@ mod os { #[cfg(windows)] mod os { + use super::{Lock, Peek}; use std::ffi::c_void; use std::fs::File; use std::io; @@ -284,8 +448,8 @@ mod os { ) -> i32; } - /// Take a non-blocking exclusive lock on byte 0. `Ok(false)` if a peer holds it. - pub fn try_lock(file: &File) -> io::Result { + /// Take a non-blocking exclusive lock on byte 0. + pub fn try_lock(file: &File) -> io::Result { let mut ov = Overlapped { internal: 0, internal_high: 0, @@ -305,21 +469,35 @@ mod os { ) }; if ok != 0 { - return Ok(true); + return Ok(Lock::Got); } let err = io::Error::last_os_error(); match err.raw_os_error() { - Some(ERROR_LOCK_VIOLATION) => Ok(false), + Some(ERROR_LOCK_VIOLATION) => Ok(Lock::Contended), _ => Err(err), } } + + /// Windows has no query-only call, so this takes the lock; the caller closes + /// the handle straight after, which releases it. + pub fn is_locked(file: &File) -> io::Result { + Ok(match try_lock(file)? { + Lock::Got => Peek::Free, + Lock::Contended => Peek::Held, + Lock::Unsupported => Peek::Unsupported, + }) + } } /// No file locks: claims exclude threads of this process only. #[cfg(not(any(unix, windows)))] mod os { - pub fn try_lock(_file: &std::fs::File) -> std::io::Result { - Ok(true) + use super::{Lock, Peek}; + pub fn try_lock(_file: &std::fs::File) -> std::io::Result { + Ok(Lock::Unsupported) + } + pub fn is_locked(_file: &std::fs::File) -> std::io::Result { + Ok(Peek::Unsupported) } } @@ -338,6 +516,7 @@ mod tests { let dir = tmp(); let path = dir.join("x.lock"); let first = try_claim(&path).expect("first claim"); + assert!(first.is_os_locked()); assert!(try_claim(&path).is_none(), "already held"); assert!(matches!(probe(&path), Probe::Held)); drop(first); @@ -365,7 +544,7 @@ mod tests { } #[test] - fn dropping_one_claim_never_releases_another() { + fn probing_never_takes_or_drops_a_lock() { // POSIX drops a process's locks on a file when ANY descriptor for it // closes; the registry must keep a second open from ever happening. let dir = tmp(); @@ -378,6 +557,21 @@ mod tests { "probing must not have closed our lock" ); drop(held); + // And a probe of a free file leaves it free for the next claimer. + assert!(matches!(probe(&path), Probe::Free)); + assert!(try_claim(&path).is_some()); + } + + #[test] + fn one_file_under_two_spellings_is_one_claim() { + let dir = tmp(); + let path = dir.join("x.lock"); + let held = try_claim(&path).unwrap(); + // The same file through a `..` detour. + let detour = dir.join("sub").join("..").join("x.lock"); + std::fs::create_dir_all(dir.join("sub")).unwrap(); + assert!(try_claim(&detour).is_none()); + drop(held); } #[test] @@ -407,4 +601,19 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } } + + #[test] + fn a_filesystem_that_rejects_locks_degrades_to_thread_only_claims() { + let dir = tmp(); + let path = dir.join("x.lock"); + FORCE_UNSUPPORTED.with(|f| f.set(true)); + let first = try_claim(&path).expect("must still claim"); + assert!(!first.is_os_locked()); + assert!(try_claim(&path).is_none(), "threads are still excluded"); + drop(first); + assert!(try_claim(&path).is_some(), "and it is released on drop"); + // A probe cannot say: the caller must use another signal. + assert!(matches!(probe(&path), Probe::Unknown)); + FORCE_UNSUPPORTED.with(|f| f.set(false)); + } } diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index 804b95d..b9620b9 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -271,7 +271,7 @@ const ORPHAN_GRACE: std::time::Duration = std::time::Duration::from_secs(60); /// Whether `path` was last modified at least `grace` ago. Unknown or future-dated /// times count as recent: the conservative answer never disturbs a live peer. -fn untouched_for(path: &Path, grace: std::time::Duration) -> bool { +pub(crate) fn untouched_for(path: &Path, grace: std::time::Duration) -> bool { std::fs::metadata(path) .and_then(|m| m.modified()) .ok() diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 51c6a36..c9d2fd6 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -86,8 +86,22 @@ pub fn find_pending(data_dir: &Path, current_generation: u64) -> Vec continue, + crate::claim::Probe::Free => { + // A launching session locks BEFORE it writes its pid, and a + // marker still empty and fresh may be one that is between + // creating the lock file and locking it. + if marker_is_empty(&marker) && marker_is_fresh(&marker) { + continue; + } + } + // The filesystem rejects locks: fall back to the pid. + crate::claim::Probe::Unknown => { + if !legacy_marker_is_dead(&marker, self_pid) { + continue; + } + } } } else if !legacy_marker_is_dead(&marker, self_pid) { continue; @@ -136,6 +150,10 @@ fn legacy_marker_is_dead(marker: &Path, self_pid: u32) -> bool { } } +fn marker_is_empty(marker: &Path) -> bool { + std::fs::read_to_string(marker).is_ok_and(|s| s.trim().is_empty()) +} + /// Grace window for an empty (pid-not-yet-written) liveness marker: a launching /// peer writes its pid within microseconds of the O_EXCL create, so a marker /// still empty after this is a dead session, not a live peer mid-launch. @@ -398,6 +416,22 @@ pub fn gc_orphans(data_dir: &Path, current_generation: u64) { crate::claim::remove_if_free(&entry.path()); } } + // Leftovers of an earlier claim scheme, once they are plainly dead. + let old_gen = name + .strip_suffix(".claim.takeover") + .or_else(|| name.strip_suffix(".claim")) + .and_then(|g| g.parse::().ok()); + if let Some(generation) = old_gen { + if generation < current_generation + && !alive(generation) + && crate::queue::untouched_for( + &entry.path(), + std::time::Duration::from_secs(60), + ) + { + let _ = std::fs::remove_file(entry.path()); + } + } } } } @@ -712,3 +746,91 @@ fn crash_meta(signatures: Vec, trigger: TriggerType) -> ReportMeta { attachments: Vec::new(), } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::claim::{try_claim, FORCE_UNSUPPORTED}; + use crate::session::Session; + + fn tmp() -> PathBuf { + let p = std::env::temp_dir().join(format!("bugsee-rec-{}", crate::util::random_hex(8))); + std::fs::create_dir_all(p.join("sessions")).unwrap(); + p + } + + fn marker(dir: &Path, generation: u64, content: &str) -> PathBuf { + let m = Session::marker_path(dir, generation); + std::fs::write(&m, content).unwrap(); + m + } + + fn gens(dir: &Path) -> Vec { + find_pending(dir, u64::MAX) + .iter() + .map(|p| p.generation) + .collect() + } + + #[test] + fn a_free_lock_means_the_session_died_whatever_pid_its_marker_names() { + let dir = tmp(); + marker(&dir, 1, "1"); // a pid that is alive here + std::fs::write(Session::lock_path(&dir, 1), "").unwrap(); + assert_eq!(gens(&dir), vec![1]); + } + + #[test] + fn a_held_lock_means_alive_whatever_pid_its_marker_names() { + let dir = tmp(); + marker(&dir, 1, "4294967290"); // not a pid in this namespace + let _held = try_claim(&Session::lock_path(&dir, 1)).unwrap(); + assert!(gens(&dir).is_empty()); + } + + #[test] + fn a_launching_session_between_lock_file_and_lock_is_not_recovered() { + // Created the lock file, not yet locked it, not yet written its pid. + let dir = tmp(); + let m = marker(&dir, 1, ""); + std::fs::write(Session::lock_path(&dir, 1), "").unwrap(); + assert!(gens(&dir).is_empty(), "fresh and empty: still launching"); + let f = std::fs::OpenOptions::new().write(true).open(&m).unwrap(); + f.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(120)) + .unwrap(); + drop(f); + assert_eq!(gens(&dir), vec![1], "an old empty marker is a dead launch"); + } + + #[test] + fn without_locks_the_marker_pid_decides() { + let dir = tmp(); + marker(&dir, 1, "4294967290"); + std::fs::write(Session::lock_path(&dir, 1), "").unwrap(); + #[cfg(unix)] + { + marker(&dir, 2, "1"); + std::fs::write(Session::lock_path(&dir, 2), "").unwrap(); + } + FORCE_UNSUPPORTED.with(|f| f.set(true)); + let found = gens(&dir); + FORCE_UNSUPPORTED.with(|f| f.set(false)); + // Dead pid: recovered. Live pid (unix): left alone. + assert_eq!(found, vec![1]); + } + + #[test] + fn old_scheme_claim_files_of_dead_sessions_are_swept() { + let dir = tmp(); + let claim = dir.join("sessions").join("1.claim"); + let guard = dir.join("sessions").join("1.claim.takeover"); + for p in [&claim, &guard] { + std::fs::write(p, "").unwrap(); + let f = std::fs::OpenOptions::new().write(true).open(p).unwrap(); + f.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(300)) + .unwrap(); + } + gc_orphans(&dir, 5); + assert!(!claim.exists() && !guard.exists()); + } +} diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 232e8f8..75bba1f 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -440,6 +440,9 @@ impl Recorder { &mut self, sampler: Option>, ) -> std::io::Result<()> { + // The registry of held claims is the parent's, and may be mid-update. + crate::claim::reset_after_fork(); + // The parent's threads are not ours to join; drop their handles without // running anything. std::mem::forget(self.worker.take()); @@ -664,6 +667,13 @@ impl Drop for Recorder { if let Some(session) = &self.session { session.end(); } + } else if !(worker_acked && uploader_acked) { + // A detached thread may still be writing this session's data: keep + // the liveness lock until the process exits so no peer recovers (and + // deletes) the session underneath it. + if let Some(session) = &mut self.session { + session.keep_lock_until_exit(); + } } } } diff --git a/crates/bugsee-core/src/session.rs b/crates/bugsee-core/src/session.rs index 8d75c8c..6a8218e 100644 --- a/crates/bugsee-core/src/session.rs +++ b/crates/bugsee-core/src/session.rs @@ -30,8 +30,9 @@ pub struct Session { /// The OS lock on `.lock`, held for the session's life. Peers sharing the /// data dir (including from another pid namespace) probe it to tell a live /// session from a crashed one; the kernel drops it when this process dies. - /// `None` if the lock file could not be taken (the bare pid in the marker is - /// then all a peer has). + /// `None` when the session has no lock file (the filesystem rejects locks, or + /// the file could not be locked), in which case the pid in the marker is all + /// a peer has and `.lock` is deliberately absent. _lock: Option, } @@ -53,9 +54,8 @@ impl Session { { Ok(mut f) => { // Lock BEFORE writing the pid: the marker is empty (and fresh, - // so peers skip it) until then, which closes the window in - // which it is neither locked nor attributed. - let lock = crate::claim::try_claim(&Session::lock_path(data_dir, candidate)); + // so peers skip it) until then. + let lock = Session::take_lock(data_dir, candidate); // The marker stays a bare pid, readable by older builds. let _ = write!(f, "{}", std::process::id()); break (candidate, lock); @@ -77,6 +77,38 @@ impl Session { }) } + /// Take `.lock`, or leave no lock file at all. + /// + /// A peer's probe never takes the lock (unix), but on Windows it momentarily + /// does, so a contended claim is retried briefly. If it still cannot be held + /// with kernel enforcement the file is removed: a lock file that exists but + /// is not held reads as "that session died" to every peer, which would have + /// them recover (and delete) this live session. Without the file peers fall + /// back to the pid in the marker. + fn take_lock(data_dir: &Path, generation: u64) -> Option { + let path = Session::lock_path(data_dir, generation); + for attempt in 0..20 { + match crate::claim::try_claim(&path) { + Some(claim) if claim.is_os_locked() => return Some(claim), + // Thread-only (the filesystem rejects locks): useless to peers. + Some(_) => break, + None if attempt < 19 => std::thread::sleep(std::time::Duration::from_millis(2)), + None => break, + } + } + let _ = std::fs::remove_file(&path); + None + } + + /// Keep this session's lock until the process exits instead of releasing it + /// when the session is dropped. For an unclean shutdown, where a detached + /// worker or uploader may still be writing this session's data: releasing the + /// lock would let a peer recover the session as a crash and delete its parts + /// underneath them. The kernel drops the lock when the process ends. + pub(crate) fn keep_lock_until_exit(&mut self) { + std::mem::forget(self._lock.take()); + } + /// This session's generation number. pub fn generation(&self) -> u64 { self.generation @@ -142,3 +174,61 @@ fn max_numeric_entry(dir: &Path, strip_suffix: Option<&str>) -> u64 { } max } + +#[cfg(test)] +mod tests { + use super::*; + use crate::claim::{probe, Probe}; + + fn tmp() -> PathBuf { + let p = std::env::temp_dir().join(format!("bugsee-sess-{}", crate::util::random_hex(8))); + std::fs::create_dir_all(&p).unwrap(); + p + } + + #[test] + fn a_live_session_holds_its_lock_and_a_clean_end_removes_it() { + let dir = tmp(); + let session = Session::begin(&dir).unwrap(); + let lock = Session::lock_path(&dir, session.generation()); + assert!(matches!(probe(&lock), Probe::Held)); + session.end(); + assert!(!lock.exists()); + } + + #[test] + fn a_session_that_cannot_hold_its_lock_leaves_no_lock_file_behind() { + // A lock file that exists but is not held reads as "that session died". + let dir = tmp(); + std::fs::create_dir_all(dir.join(SESSIONS_DIR)).unwrap(); + let peer = crate::claim::try_claim(&Session::lock_path(&dir, 1)).unwrap(); + let session = Session::begin(&dir).unwrap(); + assert_eq!(session.generation(), 1); + assert!(session._lock.is_none()); + drop(peer); + assert!(!Session::lock_path(&dir, 1).exists()); + } + + #[test] + fn a_session_on_a_filesystem_without_locks_leaves_no_lock_file() { + let dir = tmp(); + crate::claim::FORCE_UNSUPPORTED.with(|f| f.set(true)); + let session = Session::begin(&dir).unwrap(); + crate::claim::FORCE_UNSUPPORTED.with(|f| f.set(false)); + assert!(session._lock.is_none()); + assert!(!Session::lock_path(&dir, session.generation()).exists()); + } + + #[test] + fn keeping_the_lock_until_exit_survives_dropping_the_session() { + let dir = tmp(); + let mut session = Session::begin(&dir).unwrap(); + let lock = Session::lock_path(&dir, session.generation()); + session.keep_lock_until_exit(); + drop(session); + assert!( + matches!(probe(&lock), Probe::Held), + "a detached thread may still be writing: peers must not recover it" + ); + } +} diff --git a/crates/bugsee-core/tests/multiprocess.rs b/crates/bugsee-core/tests/multiprocess.rs index 9b2ae1e..0e2f700 100644 --- a/crates/bugsee-core/tests/multiprocess.rs +++ b/crates/bugsee-core/tests/multiprocess.rs @@ -101,6 +101,13 @@ fn child_entry() { std::fs::rename(dir.join("READY.tmp"), dir.join("READY")).unwrap(); std::thread::sleep(Duration::from_secs(120)); } + "lockholder" => { + // Hold `sessions/1.lock`, as a peer's probe or launch would for a moment. + let claim = bugsee_core::claim::try_claim(&dir.join("sessions").join("1.lock")); + std::fs::write(dir.join("READY.tmp"), claim.is_some().to_string()).unwrap(); + std::fs::rename(dir.join("READY.tmp"), dir.join("READY")).unwrap(); + std::thread::sleep(Duration::from_secs(120)); + } other => panic!("unknown role {other}"), } } @@ -150,6 +157,8 @@ fn seed(dir: &Path, queued: usize, crashed: u64) { for g in 1..=crashed { // "0" is the portable "owner gone" sentinel. std::fs::write(sessions.join(format!("{g}.alive")), "0").unwrap(); + // The lock file a crashed session leaves behind: present, held by no one. + std::fs::write(sessions.join(format!("{g}.lock")), "").unwrap(); std::fs::create_dir_all(dir.join("parts").join(g.to_string())).unwrap(); } std::fs::write(dir.join("gen"), crashed.to_string()).unwrap(); @@ -274,7 +283,20 @@ fn a_live_peers_session_is_not_mistaken_for_a_crash_whatever_pid_its_marker_name // peer: the parent's own, then init's. Neither may make the peer look dead, // nor make it look alive once it is gone. let marker = dir.join("sessions").join(format!("{generation}.alive")); - for foreign in [std::process::id().to_string(), "1".to_string()] { + // A reaped child's pid: gone from this namespace, as a peer's own pid is. + let mut reaped = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "no_such_test"]) + .stdout(Stdio::null()) + .spawn() + .unwrap(); + let _ = reaped.wait(); + let foreign_pids = [ + std::process::id().to_string(), + "1".to_string(), + "4294967290".to_string(), + reaped.id().to_string(), + ]; + for foreign in foreign_pids { std::fs::write(&marker, &foreign).unwrap(); run_workers(&dir, &log, 3); let _ = std::fs::remove_file(dir.join("GO")); @@ -289,3 +311,39 @@ fn a_live_peers_session_is_not_mistaken_for_a_crash_whatever_pid_its_marker_name run_workers(&dir, &log, 3); assert_eq!(delivered(&log), vec!["recovered-crash".to_string()]); } + +/// A launch that finds `.lock` momentarily held by someone else must not leave +/// an unheld lock file behind: every peer would read it as "that session died" and +/// recover (and delete) the live session. +#[test] +fn a_launch_that_loses_the_lock_race_is_not_recovered_as_a_crash() { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + std::fs::create_dir_all(dir.join("sessions")).unwrap(); + let log = tmp.0.join("delivered.log"); + + let mut holder = spawn("lockholder", &dir, &log); + let ready = dir.join("READY"); + let deadline = Instant::now() + Duration::from_secs(20); + while !ready.exists() { + assert!(Instant::now() < deadline, "holder never started"); + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(std::fs::read_to_string(&ready).unwrap(), "true"); + std::fs::remove_file(&ready).unwrap(); + + // Generation 1 is ours, but its lock file is held by the other process. + let session = bugsee_core::session::Session::begin(&dir).unwrap(); + assert_eq!(session.generation(), 1); + holder.kill().unwrap(); + let _ = holder.wait(); + + // Peers now look at a live session (this process) with no usable lock file. + run_workers(&dir, &log, 3); + assert!( + delivered(&log).is_empty(), + "live session reported as a crash" + ); + assert!(dir.join("sessions").join("1.alive").exists()); + drop(session); +} From 716c363a05c83c25f79cbb728bbe0a5c15256875 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:54:48 +0500 Subject: [PATCH 08/15] test: gate unix-only helpers so Windows clippy stays clean (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/tests/storage_failures.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index 175de28..bf04627 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -433,6 +433,7 @@ fn a_failed_put_retries_at_the_put_and_does_not_mint_a_duplicate_issue() { ); } +#[cfg(unix)] /// Lay down what a previous run left behind: a queued report nobody delivered and /// a session that died (marker left, owner gone). fn seed_leftovers(dir: &Path) { @@ -470,6 +471,7 @@ fn seed_leftovers(dir: &Path) { std::fs::write(dir.join("gen"), "1").unwrap(); } +#[cfg(unix)] fn wait_for_bundles(transport: &MockTransport, n: usize) -> usize { let start = std::time::Instant::now(); while transport.uploaded_bundles.lock().unwrap().len() < n From 0b78a1000d30068b7c195aebee31072937dbda93 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:05:35 +0500 Subject: [PATCH 09/15] fix: drain every trusted unwritable default tree, not just one; stale unlinkable claims proceed (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - recovery_dirs: all rejected existing candidates (incl. the private temp tree) are drained/recovered even when a later default wins; core takes extra_data_dirs - a stale claim that cannot be unlinked is Unavailable, not Taken - seed the in-memory blacklist from the on-disk one πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/claim.rs | 28 ++++++- crates/bugsee-core/src/queue.rs | 5 ++ crates/bugsee-core/src/runtime.rs | 78 +++++++++---------- crates/bugsee-core/tests/storage_failures.rs | 43 +++++++++++ crates/bugsee/src/api.rs | 5 +- crates/bugsee/src/options.rs | 80 ++++++++++++++++---- 7 files changed, 179 insertions(+), 62 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 4d06151..6b3cdbe 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -187,7 +187,7 @@ Global base scope behind `RwLock`; `with_scope` pushes a thread-local overlay. * |---|---| | Default candidate unusable, a later default works | falls through; one stderr line; `StorageStatus::skipped` lists why | | Requested directory unusable, or no default usable, or `Session::begin`/`PartStore::new` fails (also in a fork child) | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from an in-memory backlog capped at 16 (an entry counts until its delivery settles; overflow drops the newest with `DropReason::StorageUnavailable`); a `TooManySimilar` verdict is remembered in memory; a clean stop (guard drop) gives the backlog one last bounded (1.5 s) attempt, but a report still unsent when the process dies is lost; a fork child starts with an empty backlog (the parent's is the parent's to send) | -| Directory exists but cannot take NEW files (full, read-only remount) | memory mode for new reports, but the directory is still **drained and recovered from**: its queued bundles are delivered and its crashed sessions recovered (claims are best-effort there; a crash that cannot be re-queued on disk is delivered from memory). Bundles delivered but undeletable are remembered and not resent in this process; a later launch with a writable disk may resend them and the backend deduplicates. Only a missing or non-directory path is skipped | +| Directory exists but cannot take NEW files (full, read-only remount) | memory mode for new reports, but the directory is still **drained and recovered from**: its queued bundles are delivered and its crashed sessions recovered (claims are best-effort there; a crash that cannot be re-queued on disk is delivered from memory). Bundles delivered but undeletable are remembered and not resent in this process; a later launch with a writable disk may resend them and the backend deduplicates. Every such rejected candidate is drained, including a full default that lost to a later writable one and the private temp tree (which must be a real directory of ours); a stale claim that cannot be unlinked counts as "no claim possible", not "held elsewhere"; the on-disk `.blacklist` seeds the in-memory one. Only a missing or non-directory path is skipped | | Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and dropped; the rotation tick rewrites a missing `sessions/.alive` (a wiped-and-recreated tree would otherwise make the next launch's orphan sweep drop this session's crash); a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out; `StorageStatus::degraded` turns true and `crash_recovery_available()` false | | Disk full / permission denied at enqueue, snapshot, marker | same as above; the `reports/` snapshot dir is reclaimed by `SnapshotHandle::drop` | | Crash handler cannot write its marker | the raw `open`/`write` return values are ignored; the process still dies with the original signal (watchdog bounds the handler) | diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index 12ab003..341f1e0 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -86,7 +86,14 @@ fn claim_state(path: &Path) -> ClaimState { // Take over a claim whose owner is gone, then retry the create. // Losing the race to another taker just means `create_new` fails // again and we report the claim as held. - let _ = std::fs::remove_file(path); + match std::fs::remove_file(path) { + Ok(()) => {} + // Someone else took it over first: retry the create. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + // A read-only or full volume will not let us clear it, so no + // claim can ever be made here: not "held by someone else". + Err(_) => return ClaimState::Unavailable, + } } // Directory missing / unwritable: nothing can be claimed, and the // caller's own I/O will fail more usefully than we can here. @@ -150,6 +157,25 @@ mod tests { assert!(try_claim(&path).is_some()); } + #[cfg(unix)] + #[test] + fn a_stale_claim_that_cannot_be_removed_is_unavailable_not_taken() { + use std::os::unix::fs::PermissionsExt; + let dir = tmp(); + let path = dir.join("x.claim"); + std::fs::write(&path, "4294967290").unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o555)).unwrap(); + if std::fs::remove_file(dir.join("nonexistent-probe")).is_ok() + || std::fs::write(dir.join("probe"), b"x").is_ok() + { + let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)); + return; // permission bits not enforced (root) + } + // Proceed unclaimed instead of skipping the work as "held elsewhere". + assert!(matches!(try_claim_lenient(&path), Some(None))); + let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)); + } + #[test] fn a_freshly_created_unowned_claim_is_not_stolen() { let dir = tmp(); diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index ea6c033..2a1cbd2 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -186,6 +186,11 @@ fn blacklist_path(data_dir: &Path) -> PathBuf { queue_dir(data_dir).join(".blacklist") } +/// The locally blacklisted signatures in `data_dir` (empty if none or unreadable). +pub fn blacklisted_signatures(data_dir: &Path) -> BTreeSet { + load_blacklist(data_dir) +} + fn load_blacklist(data_dir: &Path) -> BTreeSet { std::fs::read_to_string(blacklist_path(data_dir)) .map(|s| { diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 57a9864..49fe3dc 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -179,6 +179,10 @@ pub struct RecorderConfig { /// crashed sessions and deliver its queue. For a host layer that moved its /// default location and must not strand reports left at the old one. pub legacy_data_dir: Option, + /// More existing directories to recover from and drain once, like + /// `legacy_data_dir`: trusted trees that could not take new files (a full or + /// read-only default) even though a later default was writable. + pub extra_data_dirs: Vec, } impl RecorderConfig { @@ -201,6 +205,7 @@ impl RecorderConfig { max_queued_entries: DEFAULT_MAX_QUEUED, memory_only: None, legacy_data_dir: None, + extra_data_dirs: Vec::new(), } } } @@ -414,7 +419,11 @@ impl Recorder { config.upload_backoff_base, config.sampler, true, - config.legacy_data_dir, + config + .legacy_data_dir + .into_iter() + .chain(config.extra_data_dirs) + .collect(), )?; Ok(Recorder { @@ -504,7 +513,7 @@ impl Recorder { self.backoff_base, sampler, false, - None, + Vec::new(), )?; // The parent's session is still the parent's: leave its marker alone. @@ -850,16 +859,16 @@ fn spawn_pipeline( backoff_base: Duration, sampler: Option>, fresh_launch: bool, - legacy_data_dir: Option, + legacy_data_dirs: Vec, ) -> std::io::Result { // Uploader thread: drains the durable queue with retry. let (upload_tx, upload_rx) = channel(); let uploader_shared = Arc::clone(shared); let uploader_data_dir = drain_dir.map(Path::to_path_buf); - let uploader_legacy = legacy_data_dir.clone(); + let uploader_legacy = legacy_data_dirs.clone(); // Set once the worker has finished recovering the legacy directory, so the // uploader does not retire it while recovery is still about to fill its queue. - let legacy_ready = Arc::new(AtomicBool::new(legacy_data_dir.is_none())); + let legacy_ready = Arc::new(AtomicBool::new(legacy_data_dirs.is_empty())); let uploader_ready = Arc::clone(&legacy_ready); let uploader = std::thread::Builder::new() .name("bugsee-uploader".into()) @@ -897,6 +906,14 @@ fn spawn_pipeline( if let (true, Some(dir)) = (fresh_launch, &worker_drain_dir) { queue::gc_orphans(dir); } + // With no writable disk the in-memory set is the only blacklist + // `process_report` consults, so seed it from what is on disk. + if let Some(dir) = &worker_drain_dir { + let known = queue::blacklisted_signatures(dir); + if !known.is_empty() { + worker_shared.mem_blacklist.lock().extend(known); + } + } // A fork child recovers too. A prefork master recovers only at ITS // launch, and its own generation is the lowest, so it can never // recover a worker that died later; the next worker started from @@ -907,7 +924,7 @@ fn spawn_pipeline( } // Best effort, once: sessions a previous default location left // behind. No generation of ours lives there, so none is skipped. - if let Some(dir) = &legacy_data_dir { + for dir in &legacy_data_dirs { queue::gc_orphans(dir); run_recovery(&worker_shared, dir, u64::MAX); } @@ -1422,39 +1439,27 @@ fn uploader_loop( rx: Receiver, shared: Arc, data_dir: Option, - legacy_data_dir: Option, + legacy_data_dirs: Vec, legacy_ready: Arc, backoff_base: Duration, fresh_launch: bool, ) { let poll = Duration::from_secs(5); - let mut legacy = legacy_data_dir; + let mut legacy = legacy_data_dirs; // A fresh launch retries any queued report immediately (force), ignoring // backoff scheduled by the prior session. A fork child skips this: the // parent's uploader is already working the shared queue. Guarded so a // panicking Transport impl can't kill the uploader. if fresh_launch { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_all( - &shared, - data_dir.as_deref(), - legacy.as_deref(), - backoff_base, - true, - ) + drain_all(&shared, data_dir.as_deref(), &legacy, backoff_base, true) })); } loop { match rx.recv_timeout(poll) { Ok(UploadMsg::Wake) => { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_all( - &shared, - data_dir.as_deref(), - legacy.as_deref(), - backoff_base, - false, - ) + drain_all(&shared, data_dir.as_deref(), &legacy, backoff_base, false) })); } Ok(UploadMsg::Drain { deadline, ack }) => { @@ -1462,7 +1467,7 @@ fn uploader_loop( let dirs: Vec<&Path> = data_dir .as_deref() .into_iter() - .chain(legacy.as_deref()) + .chain(legacy.iter().map(PathBuf::as_path)) .collect(); drain_until_empty(&shared, &dirs, backoff_base, deadline) })) @@ -1487,13 +1492,7 @@ fn uploader_loop( } Err(RecvTimeoutError::Timeout) => { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_all( - &shared, - data_dir.as_deref(), - legacy.as_deref(), - backoff_base, - false, - ) + drain_all(&shared, data_dir.as_deref(), &legacy, backoff_base, false) })); } Err(RecvTimeoutError::Disconnected) => break, @@ -1515,31 +1514,26 @@ fn pending_reports(shared: &Shared, dir: &Path) -> Vec { /// Stop polling the legacy directory once recovery has run (it runs once per /// launch, so nothing new can appear) and its queue is empty. -fn retire_legacy(shared: &Shared, legacy: &mut Option, ready: &AtomicBool) { +fn retire_legacy(shared: &Shared, legacy: &mut Vec, ready: &AtomicBool) { if !ready.load(Ordering::SeqCst) { return; } - if legacy - .as_deref() - .is_some_and(|dir| pending_reports(shared, dir).is_empty()) - { - *legacy = None; - } + legacy.retain(|dir| !pending_reports(shared, dir).is_empty()); } /// [`drain`] the data directory, then the legacy one a launch was asked to adopt. fn drain_all( shared: &Shared, data_dir: Option<&Path>, - legacy: Option<&Path>, + legacy: &[PathBuf], backoff_base: Duration, force: bool, ) -> bool { - let attempted = drain(shared, data_dir, backoff_base, force); - match legacy { - Some(dir) => drain_disk(shared, dir, backoff_base, force) | attempted, - None => attempted, + let mut attempted = drain(shared, data_dir, backoff_base, force); + for dir in legacy { + attempted |= drain_disk(shared, dir, backoff_base, force); } + attempted } /// Attempt delivery of every due queued report once. On the flush path `force` diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index bf04627..616fb67 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -656,3 +656,46 @@ fn a_directory_wiped_and_recreated_mid_run_gets_its_liveness_marker_back() { assert!(alive(), "a crash after the wipe must still be recoverable"); assert!(recorder.is_degraded(), "the lost window is a degradation"); } + +/// A later default can win while an earlier, full one still holds leftovers. +#[cfg(unix)] +#[test] +fn extra_directories_are_recovered_alongside_a_working_data_dir() { + let dir = TempDir::new(); + let data = dir.path.join("data"); + let full = dir.path.join("full-default"); + std::fs::create_dir_all(&full).unwrap(); + seed_leftovers(&full); + + let transport = Arc::new(MockTransport::default()); + let mut config = RecorderConfig::new(&data, "T"); + config.extra_data_dirs = vec![full.clone()]; + let recorder = Recorder::launch(config, transport.clone()).expect("launch"); + assert_eq!(recorder.storage_mode(), StorageMode::Disk); + assert_eq!(wait_for_bundles(&transport, 2), 2); + assert!(bugsee_core::queue::list_pending(&full).is_empty()); +} + +#[test] +fn signatures_blacklisted_on_disk_are_honoured_in_memory_mode() { + let dir = TempDir::new(); + let data = dir.path.join("data"); + std::fs::create_dir_all(data.join("queue")).unwrap(); + std::fs::write(data.join("queue").join(".blacklist"), "sig-known\n").unwrap(); + + let transport = Arc::new(MockTransport::default()); + let mut config = RecorderConfig::new(&data, "T"); + config.memory_only = Some("write probe failed".into()); + let recorder = Recorder::launch(config, transport.clone()).expect("launch"); + + let mut known = manual_upload_meta(); + known.signatures = vec!["sig-known".into()]; + recorder.report(known, None); + recorder.report(manual_upload_meta(), None); + assert!(recorder.flush(Duration::from_secs(10))); + assert_eq!( + transport.created_requests.lock().unwrap().len(), + 1, + "only the report without the blacklisted signature is sent" + ); +} diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 985b567..6c1bfa2 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -88,7 +88,7 @@ impl Bugsee { source, skipped, explicit_rejected, - recovery_dir, + recovery_dirs, legacy_dir, } = options.resolve_data_dir(); let memory_only = dir.is_none().then(|| { @@ -99,7 +99,7 @@ impl Bugsee { }); // In memory mode no new file is written there, but an existing directory // is still drained and recovered from; with none, an empty path. - let data_dir = dir.or(recovery_dir).unwrap_or_default(); + let data_dir = dir.unwrap_or_default(); if memory_only.is_none() && !skipped.is_empty() { static WARNED: AtomicBool = AtomicBool::new(false); bugsee_core::util::warn_once( @@ -134,6 +134,7 @@ impl Bugsee { let mut config = RecorderConfig::new(data_dir, options.app_token.clone()); config.memory_only = memory_only; config.legacy_data_dir = legacy_dir; + config.extra_data_dirs = recovery_dirs; config.caps = options.caps(); config.rotate_interval = options.rotate_interval; config.before_send = options.before_send; diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index a77619c..6688638 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -292,10 +292,11 @@ pub(crate) struct DataDirResolution { pub skipped: Vec, /// Why an explicit / environment directory was rejected (it is final). pub explicit_rejected: Option, - /// A rejected candidate that nevertheless EXISTS as a directory (full or - /// read-only). It may hold reports and crashes a previous run persisted, so - /// it is still drained and recovered from; no new files are written to it. - pub recovery_dir: Option, + /// Rejected candidates that nevertheless EXIST as directories we may trust + /// (full or read-only). They may hold reports and crashes a previous run + /// persisted, so each is still drained and recovered from, whether or not a + /// later candidate won; no new files are written to them. + pub recovery_dirs: Vec, /// The location earlier SDK versions used by default /// (`/bugsee/`), if it exists, is trustworthy and is not `dir`. /// Its leftovers are recovered once at launch. @@ -326,12 +327,12 @@ pub(crate) fn resolve_data_dir( source, skipped: Vec::new(), explicit_rejected: None, - recovery_dir: None, + recovery_dirs: Vec::new(), legacy_dir: None, }, Err(e) => DataDirResolution { explicit_rejected: Some(format!("{}: {e}", dir.display())), - recovery_dir: dir.is_dir().then_some(dir), + recovery_dirs: if dir.is_dir() { vec![dir] } else { Vec::new() }, dir: None, source: DataDirSource::None, skipped: Vec::new(), @@ -358,7 +359,7 @@ pub(crate) fn resolve_data_dir( let mut skipped = Vec::new(); let mut found = None; - let mut recovery_dir = None; + let mut recovery_dirs = Vec::new(); for (source, dir, private) in candidates { match prepare_dir(&dir, private) { Ok(()) => { @@ -367,10 +368,16 @@ pub(crate) fn resolve_data_dir( } Err(e) => { skipped.push(format!("{}: {e}", dir.display())); - // Only the user's own default location: the shared temp area is - // exactly where a planted directory would be refused. - if !private && recovery_dir.is_none() && dir.is_dir() { - recovery_dir = Some(dir); + // The shared temp area is where a planted directory would be + // refused, so there the tree must be a real directory of ours + // (and so must its parent) to be trusted for recovery. + let trusted = if private { + trusted_existing_dir(&dir) && dir.parent().is_some_and(trusted_existing_dir) + } else { + dir.is_dir() + }; + if trusted { + recovery_dirs.push(dir); } } } @@ -387,7 +394,7 @@ pub(crate) fn resolve_data_dir( source, skipped, explicit_rejected: None, - recovery_dir: None, + recovery_dirs, legacy_dir, }, None => DataDirResolution { @@ -395,7 +402,7 @@ pub(crate) fn resolve_data_dir( source: DataDirSource::None, skipped, explicit_rejected: None, - recovery_dir, + recovery_dirs, legacy_dir, }, } @@ -842,8 +849,8 @@ mod tests { let r = resolve_data_dir(Some(&ro), "tok", &env_of(&[]), &|| s.join("tmp")); assert!(r.dir.is_none() && r.explicit_rejected.is_some()); assert_eq!( - r.recovery_dir.as_deref(), - Some(ro.as_path()), + r.recovery_dirs, + vec![ro.clone()], "its queue and crashed sessions must still be drained" ); @@ -851,7 +858,7 @@ mod tests { let r = resolve_data_dir(Some(&blocked(&s, "f5")), "tok", &env_of(&[]), &|| { s.join("tmp") }); - assert!(r.recovery_dir.is_none()); + assert!(r.recovery_dirs.is_empty()); let _ = std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o755)); let _ = std::fs::remove_dir_all(&s); } @@ -871,6 +878,47 @@ mod tests { let _ = std::fs::remove_dir_all(&s); } + #[cfg(all(unix, not(target_vendor = "apple")))] + #[test] + fn a_full_default_that_loses_to_a_later_one_is_still_recovered_from() { + use std::os::unix::fs::PermissionsExt; + let s = scratch(); + let home = s.join("home"); + let default = home.join("bugsee").join("tok"); + std::fs::create_dir_all(&default).unwrap(); + std::fs::set_permissions(&default, std::fs::Permissions::from_mode(0o555)).unwrap(); + if std::fs::write(default.join("p"), b"x").is_ok() { + let _ = std::fs::remove_dir_all(&s); + return; // permission bits not enforced (root) + } + // The platform default is full/read-only; the temp fallback is writable. + let r = resolve_data_dir(None, "tok", &env_of(&home_env(&home)), &|| s.join("tmp")); + assert_eq!(r.source, DataDirSource::TempFallback); + assert_eq!(r.recovery_dirs, vec![default.clone()]); + let _ = std::fs::set_permissions(&default, std::fs::Permissions::from_mode(0o755)); + let _ = std::fs::remove_dir_all(&s); + } + + #[cfg(unix)] + #[test] + fn a_full_private_temp_tree_is_recovered_from_when_nothing_is_writable() { + use std::os::unix::fs::PermissionsExt; + let s = scratch(); + let tmp = s.join("tmp"); + let tree = tmp.join(private_temp_name()).join("tok"); + std::fs::create_dir_all(&tree).unwrap(); + std::fs::set_permissions(&tree, std::fs::Permissions::from_mode(0o500)).unwrap(); + if std::fs::write(tree.join("p"), b"x").is_ok() { + let _ = std::fs::remove_dir_all(&s); + return; // permission bits not enforced (root) + } + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); + assert!(r.dir.is_none()); + assert_eq!(r.recovery_dirs, vec![tree.clone()]); + let _ = std::fs::set_permissions(&tree, std::fs::Permissions::from_mode(0o755)); + let _ = std::fs::remove_dir_all(&s); + } + #[test] fn an_explicit_directory_never_asks_for_the_temp_location() { let s = scratch(); From ee3f8080c5a36501c498f27840d10a629409e954 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:20:37 +0500 Subject: [PATCH 10/15] fix(core): review round 3 on lock claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A leftover .lock that cannot be opened (0444, read-only volume) or whose lock call errors probes as Unknown, not Held, and the probe opens read-only, so it can no longer hide a crash forever. - Lock-less sessions (filesystem rejects locks) keep a .beat heartbeat file, touched every 2 s by the worker; a beat younger than 60 s means alive whatever pid the marker names, so a peer in another pid namespace is never mistaken for a crash. Without a beat the pid is probed as before. - claim_pending re-runs the liveness decision under the recovery claim, so a session that finished locking since the scan is not torn down. - Windows maps ERROR_INVALID_FUNCTION/NOT_SUPPORTED/CALL_NOT_IMPLEMENTED to Unsupported, so the documented degrade exists there; the dead-code allow is gone. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/claim.rs | 25 ++++-- crates/bugsee-core/src/recovery.rs | 140 ++++++++++++++++++++++------- crates/bugsee-core/src/runtime.rs | 26 +++++- crates/bugsee-core/src/session.rs | 20 +++++ 5 files changed, 170 insertions(+), 43 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 56449d1..8ad8f9c 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -207,7 +207,7 @@ Crash recovery needs the data directory to **outlive the process**: on a contain - **Costs of per-process locks, handled in `claim.rs`.** They do not exclude threads, so a registry of held paths does (keyed by pid and spin-locked with the holder's pid stamped in, so a `fork` child that inherits it mid-update breaks the lock instead of deadlocking). POSIX drops a process's locks on a file when *any* descriptor for it closes, so a lock file is never opened twice by one process: the registry is checked before each open, and a claim closes its descriptor before leaving the registry. - **Ordering at launch.** `Session::begin` creates the marker (`O_EXCL`), takes `.lock`, then writes the pid. Until the pid is written the marker is empty and fresh, which peers skip even if the lock reads free. A lock that cannot be held with kernel enforcement (contended past a short retry, or the filesystem rejects locks) is *removed*: an existing but unheld lock file reads as "that session died", so peers fall back to the marker's pid instead. On unix a probe asks `F_GETLK` and never takes the lock, so it cannot steal one from a launching peer. - **Unclean shutdown.** If the worker or uploader thread is detached at shutdown it may still be writing the session, so the session's lock is kept until the process exits (the kernel then drops it) rather than released on drop. -- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown so recovery uses the marker's pid. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. +- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown (as it does for a leftover lock file that cannot be opened, e.g. on a read-only volume, so a crash is never hidden by it). A session that could not take a lock keeps a `.beat` heartbeat file, touched every 2 s by the worker: a beat younger than 60 s (generous, since NFS caches attributes) means alive whatever pid the marker names, which is the only signal that works across pid namespaces without locks; with no beat the marker's pid is probed as before. The price is that a crash on such a filesystem is recovered only after its heartbeat has been quiet for a minute, so a faster restart defers it to a later launch. Under the recovery claim the liveness decision is re-run, so a session that finished launching since the scan is never torn down. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. - **Paths.** The in-process registry canonicalises the lock file's directory, so `/tmp` vs `/private/tmp` or relative vs absolute paths are one claim. A `fork` child clears the registry (`claim::reset_after_fork`, from the fork hook) before claiming anything. - **Stale scans.** After winning a session claim the marker is re-checked, since a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s: a peer's in-flight `enqueue` looks like an orphan, and reaping it lost the report. Lock files of finished items and dead sessions are swept when free. - **Compatibility.** The `.alive` marker is still a bare pid and the lock files are new names, so older builds sharing the directory keep working. They do not take part in the locking, so a mixed-version directory (a rolling upgrade) gives no exactly-once guarantee between old and new processes, and a new process treats an old build's session by its pid as before. Run one SDK version per directory for the guarantee. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index 39aea2f..5c2c730 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -116,7 +116,9 @@ pub(crate) enum Probe { } /// Is `path` currently claimed? A missing file is free. A file that cannot be -/// inspected is reported held: the conservative answer never disturbs a peer. +/// inspected is [`Probe::Unknown`], never assumed held: a crashed session's +/// leftover lock file must not hide the crash on a volume that has become +/// read-only. /// /// On unix this asks (`F_GETLK`) without taking the lock, so a probe can never /// steal the lock from a creator that has made the file but not yet locked it. @@ -128,20 +130,24 @@ pub(crate) fn probe(path: &Path) -> Probe { if !registry().insert(&key) { return Probe::Held; // held by a thread of this process, or being probed } - let result = match OpenOptions::new().read(true).write(true).open(path) { + // Read-only: a querying lock call needs no write access, and a leftover lock + // file on a read-only volume (or with 0444 mode) must still be probeable. + let result = match OpenOptions::new().read(true).open(path) { Ok(file) => { let r = match test_is_locked(&file) { Ok(Peek::Free) => Probe::Free, Ok(Peek::Held) => Probe::Held, - Ok(Peek::Unsupported) => Probe::Unknown, - Err(_) => Probe::Held, + // Cannot tell (no lock support, or the call failed): let the + // caller use another liveness signal rather than assume a live + // holder and hide a crash forever. + Ok(Peek::Unsupported) | Err(_) => Probe::Unknown, }; drop(file); // close before leaving the registry r } // Vanished since the existence check: nothing holds it. Err(_) if !path.exists() => Probe::Free, - Err(_) => Probe::Held, + Err(_) => Probe::Unknown, }; registry().remove(&key); result @@ -240,8 +246,6 @@ fn key_of(path: &Path) -> PathBuf { enum Lock { Got, Contended, - /// Only unix can tell "locks unsupported" apart from other errors. - #[cfg_attr(not(unix), allow(dead_code))] Unsupported, } @@ -446,6 +450,10 @@ mod os { const LOCKFILE_FAIL_IMMEDIATELY: u32 = 1; const LOCKFILE_EXCLUSIVE_LOCK: u32 = 2; const ERROR_LOCK_VIOLATION: i32 = 33; + // The filesystem has no byte-range locking (some network shares, FAT shims). + const ERROR_INVALID_FUNCTION: i32 = 1; + const ERROR_NOT_SUPPORTED: i32 = 50; + const ERROR_CALL_NOT_IMPLEMENTED: i32 = 120; #[repr(C)] struct Overlapped { @@ -493,6 +501,9 @@ mod os { let err = io::Error::last_os_error(); match err.raw_os_error() { Some(ERROR_LOCK_VIOLATION) => Ok(Lock::Contended), + Some(ERROR_INVALID_FUNCTION | ERROR_NOT_SUPPORTED | ERROR_CALL_NOT_IMPLEMENTED) => { + Ok(Lock::Unsupported) + } _ => Err(err), } } diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index ad23596..eeae9ea 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -80,30 +80,7 @@ pub fn find_pending(data_dir: &Path, current_generation: u64) -> Vec= current_generation { continue; } - // A session whose `.lock` exists is arbitrated by that OS lock: held - // means a live process (in any pid namespace), free means it died. The - // marker's pid is not consulted, since it is only meaningful inside the - // writer's own pid namespace and can be reused. - let lock = crate::session::Session::lock_path(data_dir, generation); - if lock.exists() { - match crate::claim::probe(&lock) { - crate::claim::Probe::Held => continue, - crate::claim::Probe::Free => { - // A launching session locks BEFORE it writes its pid, and a - // marker still empty and fresh may be one that is between - // creating the lock file and locking it. - if marker_is_empty(&marker) && marker_is_fresh(&marker) { - continue; - } - } - // The filesystem rejects locks: fall back to the pid. - crate::claim::Probe::Unknown => { - if !legacy_marker_is_dead(&marker, self_pid) { - continue; - } - } - } - } else if !legacy_marker_is_dead(&marker, self_pid) { + if !session_is_crashed(data_dir, generation, &marker, self_pid) { continue; } let parts_dir = data_dir.join("parts").join(generation.to_string()); @@ -125,10 +102,60 @@ fn exists_opt(p: PathBuf) -> Option { p.exists().then_some(p) } -/// Whether a marker with no lock file beside it (written by an older SDK build, -/// or by a session that could not lock) is a crashed session rather than a live -/// peer, judged from its recorded pid. Only meaningful within one pid namespace. +/// Whether the session `generation`, whose marker survives, has ended abnormally. +/// +/// A session whose `.lock` exists is arbitrated by that OS lock: held means a +/// live process (in any pid namespace), free means it died. The marker's pid is +/// not consulted, since it is only meaningful inside the writer's own pid +/// namespace and can be reused. Otherwise (no lock file, or the filesystem cannot +/// say) the marker is judged by [`legacy_marker_is_dead`]. +fn session_is_crashed(data_dir: &Path, generation: u64, marker: &Path, self_pid: u32) -> bool { + let lock = crate::session::Session::lock_path(data_dir, generation); + if lock.exists() { + match crate::claim::probe(&lock) { + crate::claim::Probe::Held => return false, + // A launching session locks BEFORE it writes its pid, and a marker + // still empty and fresh may be one that is between creating the lock + // file and locking it. + crate::claim::Probe::Free => { + return !(marker_is_empty(marker) && marker_is_fresh(marker)); + } + crate::claim::Probe::Unknown => {} + } + } + legacy_marker_is_dead(marker, self_pid) +} + +/// How recently a lock-less session's heartbeat file must have been touched for +/// the session to count as alive. Generous because network filesystems cache file +/// attributes for up to about a minute. +const HEARTBEAT_STALE: std::time::Duration = std::time::Duration::from_secs(60); + +/// The heartbeat file beside a marker. Only a session that could not take a lock +/// (the filesystem rejects them) keeps one: it is touched every couple of seconds. +pub(crate) fn heartbeat_path(marker: &Path) -> PathBuf { + marker.with_extension("beat") +} + +/// Whether a marker with no usable lock beside it (written by an older SDK build, +/// or by a session on a filesystem without locks) is a crashed session rather +/// than a live peer. +/// +/// A fresh heartbeat file means alive, whatever pid the marker names: that is the +/// only signal that works across pid namespaces when there are no locks. Without +/// one, the recorded pid is probed, which is only meaningful within one pid +/// namespace. A crash is therefore recovered only once the heartbeat has gone +/// quiet for [`HEARTBEAT_STALE`] (a restart faster than that defers recovery to a +/// later launch). fn legacy_marker_is_dead(marker: &Path, self_pid: u32) -> bool { + if std::fs::metadata(heartbeat_path(marker)) + .and_then(|m| m.modified()) + .ok() + .and_then(|t| t.elapsed().ok()) + .is_some_and(|age| age < HEARTBEAT_STALE) + { + return false; + } let owner = std::fs::read_to_string(marker).unwrap_or_default(); let owner = owner.trim(); if owner.is_empty() { @@ -325,6 +352,7 @@ fn read_minidump(pending: &PendingSession) -> Vec { pub fn discard(data_dir: &Path, pending: &PendingSession) { let sessions = data_dir.join("sessions"); let _ = std::fs::remove_file(sessions.join(format!("{}.alive", pending.generation))); + let _ = std::fs::remove_file(sessions.join(format!("{}.beat", pending.generation))); let _ = std::fs::remove_file(sessions.join(format!("{}.attempts", pending.generation))); let _ = std::fs::remove_dir_all(&pending.parts_dir); } @@ -373,11 +401,14 @@ pub fn claim_pending( // (its claim released, the marker gone). Recovering it now would deliver a // phantom second report for it. The marker is removed first by `discard`, so // its absence, re-checked while holding the claim, is conclusive. - data_dir - .join("sessions") - .join(format!("{}.alive", pending.generation)) - .exists() - .then_some(claim) + // + // It also re-evaluates liveness under the claim: between the scan and now a + // launching session may have finished taking its lock (the scan saw it free, + // then the pid appeared), and a live session must not be torn down. + let marker = crate::session::Session::marker_path(data_dir, pending.generation); + (marker.exists() + && session_is_crashed(data_dir, pending.generation, &marker, std::process::id())) + .then_some(claim) } /// Reap leftover on-disk state from generations that no longer have a liveness @@ -404,7 +435,10 @@ pub fn gc_orphans(data_dir: &Path, current_generation: u64) { if let Ok(read) = std::fs::read_dir(&sessions) { for entry in read.flatten() { let name = entry.file_name().to_string_lossy().into_owned(); - if let Some(gen_str) = name.strip_suffix(".attempts") { + if let Some(gen_str) = name + .strip_suffix(".attempts") + .or_else(|| name.strip_suffix(".beat")) + { if let Ok(generation) = gen_str.parse::() { if generation < current_generation && !alive(generation) { let _ = std::fs::remove_file(entry.path()); @@ -826,6 +860,46 @@ mod tests { assert_eq!(found, vec![1]); } + #[cfg(unix)] + #[test] + fn an_unwritable_leftover_lock_file_does_not_hide_a_crash() { + use std::os::unix::fs::PermissionsExt; + let dir = tmp(); + marker(&dir, 1, "4294967290"); + let lock = Session::lock_path(&dir, 1); + std::fs::write(&lock, "").unwrap(); + std::fs::set_permissions(&lock, std::fs::Permissions::from_mode(0o444)).unwrap(); + assert_eq!(gens(&dir), vec![1]); + } + + #[test] + fn a_fresh_heartbeat_keeps_a_lockless_session_alive_whatever_its_pid() { + // No lock file (the filesystem has none): the marker's pid, even our own + // (a peer in another pid namespace), must not condemn a beating session. + let dir = tmp(); + let m = marker(&dir, 1, &std::process::id().to_string()); + let beat = heartbeat_path(&m); + std::fs::write(&beat, "").unwrap(); + assert!(gens(&dir).is_empty(), "heartbeat is fresh: alive"); + let f = std::fs::OpenOptions::new().write(true).open(&beat).unwrap(); + f.set_modified(std::time::SystemTime::now() - HEARTBEAT_STALE * 2) + .unwrap(); + drop(f); + assert_eq!(gens(&dir), vec![1], "quiet heartbeat: crashed"); + } + + #[test] + fn claim_pending_gives_up_if_the_session_took_its_lock_since_the_scan() { + let dir = tmp(); + marker(&dir, 1, "4294967290"); + std::fs::write(Session::lock_path(&dir, 1), "").unwrap(); + let pending = find_pending(&dir, u64::MAX); + assert_eq!(pending.len(), 1); + // The launching session finishes locking after the scan. + let _live = try_claim(&Session::lock_path(&dir, 1)).unwrap(); + assert!(claim_pending(&dir, &pending[0]).is_none()); + } + #[test] fn old_scheme_claim_files_of_dead_sessions_are_swept() { let dir = tmp(); diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 3a41e66..c97288d 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1078,8 +1078,11 @@ fn worker_loop( // sampled (F17). Instead we recv with a timeout bounded by the next rotation // deadline and run the tick whenever that deadline passes, message or not. let mut next_rotate = Instant::now() + rotate_interval; + let mut next_heartbeat = Instant::now() + HEARTBEAT_EVERY; loop { - let timeout = next_rotate.saturating_duration_since(Instant::now()); + let timeout = next_rotate + .saturating_duration_since(Instant::now()) + .min(HEARTBEAT_EVERY); match rx.recv_timeout(timeout) { Ok(Msg::Capture(entry)) => { shared.queued.fetch_sub(1, Ordering::Relaxed); @@ -1159,14 +1162,33 @@ fn worker_loop( // elapsed β€” via a message OR a timeout β€” so steady traffic can't starve it. if Instant::now() >= next_rotate { rotate_tick(store.as_mut(), &shared, &mut sampler); + next_rotate = Instant::now() + rotate_interval; + } + // Independent of the rotation interval (which a host may set large). + if Instant::now() >= next_heartbeat { if let Some(marker) = &marker { ensure_marker(&shared, marker); + touch_heartbeat(marker); } - next_rotate = Instant::now() + rotate_interval; + next_heartbeat = Instant::now() + HEARTBEAT_EVERY; } } } +/// How often the worker checks the liveness marker and, for a session that could +/// not take a file lock, touches its heartbeat file. +const HEARTBEAT_EVERY: Duration = Duration::from_secs(2); + +/// Touch this session's heartbeat file, if it keeps one (see +/// `recovery::heartbeat_path`). Never creates it: a session with a kernel lock has +/// none, and one that has ended must not leave one behind. +fn touch_heartbeat(marker: &Path) { + let beat = recovery::heartbeat_path(marker); + if let Ok(file) = std::fs::OpenOptions::new().write(true).open(&beat) { + let _ = file.set_modified(std::time::SystemTime::now()); + } +} + /// Make sure this session's liveness marker still exists. If the data directory /// was removed and recreated under us (`create_dir_all` heals the tree but not /// the marker), the next launch would not see this session as live, and its diff --git a/crates/bugsee-core/src/session.rs b/crates/bugsee-core/src/session.rs index 6a8218e..b92515a 100644 --- a/crates/bugsee-core/src/session.rs +++ b/crates/bugsee-core/src/session.rs @@ -56,6 +56,16 @@ impl Session { // Lock BEFORE writing the pid: the marker is empty (and fresh, // so peers skip it) until then. let lock = Session::take_lock(data_dir, candidate); + if lock.is_none() { + // No kernel lock to tell peers we are alive: a heartbeat + // file, touched by the worker, is the substitute. + let _ = std::fs::write( + crate::recovery::heartbeat_path(&Session::marker_path( + data_dir, candidate, + )), + b"", + ); + } // The marker stays a bare pid, readable by older builds. let _ = write!(f, "{}", std::process::id()); break (candidate, lock); @@ -133,6 +143,10 @@ impl Session { pub fn end(&self) { let _ = std::fs::remove_file(Session::marker_path(&self.data_dir, self.generation)); let _ = std::fs::remove_file(Session::lock_path(&self.data_dir, self.generation)); + let _ = std::fs::remove_file(crate::recovery::heartbeat_path(&Session::marker_path( + &self.data_dir, + self.generation, + ))); } } @@ -217,6 +231,12 @@ mod tests { crate::claim::FORCE_UNSUPPORTED.with(|f| f.set(false)); assert!(session._lock.is_none()); assert!(!Session::lock_path(&dir, session.generation()).exists()); + // Instead it keeps a heartbeat file, removed on a clean end. + let beat = + crate::recovery::heartbeat_path(&Session::marker_path(&dir, session.generation())); + assert!(beat.exists()); + session.end(); + assert!(!beat.exists()); } #[test] From c4f81d6874cdba947cb6b0a1e7f29bb5dea628ee Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:33:18 +0500 Subject: [PATCH 11/15] fix(core): make the lock-less heartbeat robust to clock skew and slow launches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A heartbeat whose mtime is in the future (file server clock ahead) counts as fresh, like marker_is_fresh; only a demonstrably quiet beat condemns a session. The staleness window is 3 minutes, above NFS attribute-cache lifetimes. - Launch-time recovery (before the worker loop starts) now keeps the beat going. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/recovery.rs | 27 +++++++++++++++++++++------ crates/bugsee-core/src/runtime.rs | 21 ++++++++++++++++++--- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 8ad8f9c..2154dd9 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -207,7 +207,7 @@ Crash recovery needs the data directory to **outlive the process**: on a contain - **Costs of per-process locks, handled in `claim.rs`.** They do not exclude threads, so a registry of held paths does (keyed by pid and spin-locked with the holder's pid stamped in, so a `fork` child that inherits it mid-update breaks the lock instead of deadlocking). POSIX drops a process's locks on a file when *any* descriptor for it closes, so a lock file is never opened twice by one process: the registry is checked before each open, and a claim closes its descriptor before leaving the registry. - **Ordering at launch.** `Session::begin` creates the marker (`O_EXCL`), takes `.lock`, then writes the pid. Until the pid is written the marker is empty and fresh, which peers skip even if the lock reads free. A lock that cannot be held with kernel enforcement (contended past a short retry, or the filesystem rejects locks) is *removed*: an existing but unheld lock file reads as "that session died", so peers fall back to the marker's pid instead. On unix a probe asks `F_GETLK` and never takes the lock, so it cannot steal one from a launching peer. - **Unclean shutdown.** If the worker or uploader thread is detached at shutdown it may still be writing the session, so the session's lock is kept until the process exits (the kernel then drops it) rather than released on drop. -- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown (as it does for a leftover lock file that cannot be opened, e.g. on a read-only volume, so a crash is never hidden by it). A session that could not take a lock keeps a `.beat` heartbeat file, touched every 2 s by the worker: a beat younger than 60 s (generous, since NFS caches attributes) means alive whatever pid the marker names, which is the only signal that works across pid namespaces without locks; with no beat the marker's pid is probed as before. The price is that a crash on such a filesystem is recovered only after its heartbeat has been quiet for a minute, so a faster restart defers it to a later launch. Under the recovery claim the liveness decision is re-run, so a session that finished launching since the scan is never torn down. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. +- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown (as it does for a leftover lock file that cannot be opened, e.g. on a read-only volume, so a crash is never hidden by it). A session that could not take a lock keeps a `.beat` heartbeat file, touched every 2 s by the worker (and during launch-time recovery): a beat younger than 3 minutes, or dated in the future (generous: NFS caches attributes for up to a minute and its mtime is the server's clock) means alive whatever pid the marker names, which is the only signal that works across pid namespaces without locks; with no beat the marker's pid is probed as before. The price is that a crash on such a filesystem is recovered only after its heartbeat has been quiet for three minutes, so a faster restart defers it to a later launch. Under the recovery claim the liveness decision is re-run, so a session that finished launching since the scan is never torn down. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. - **Paths.** The in-process registry canonicalises the lock file's directory, so `/tmp` vs `/private/tmp` or relative vs absolute paths are one claim. A `fork` child clears the registry (`claim::reset_after_fork`, from the fork hook) before claiming anything. - **Stale scans.** After winning a session claim the marker is re-checked, since a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s: a peer's in-flight `enqueue` looks like an orphan, and reaping it lost the report. Lock files of finished items and dead sessions are swept when free. - **Compatibility.** The `.alive` marker is still a bare pid and the lock files are new names, so older builds sharing the directory keep working. They do not take part in the locking, so a mixed-version directory (a rolling upgrade) gives no exactly-once guarantee between old and new processes, and a new process treats an old build's session by its pid as before. Run one SDK version per directory for the guarantee. diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index eeae9ea..526717b 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -128,8 +128,9 @@ fn session_is_crashed(data_dir: &Path, generation: u64, marker: &Path, self_pid: /// How recently a lock-less session's heartbeat file must have been touched for /// the session to count as alive. Generous because network filesystems cache file -/// attributes for up to about a minute. -const HEARTBEAT_STALE: std::time::Duration = std::time::Duration::from_secs(60); +/// attributes (Linux NFS `acregmax` defaults to 60 s), so a live beat can look much +/// older than it is. +const HEARTBEAT_STALE: std::time::Duration = std::time::Duration::from_secs(180); /// The heartbeat file beside a marker. Only a session that could not take a lock /// (the filesystem rejects them) keeps one: it is touched every couple of seconds. @@ -145,14 +146,15 @@ pub(crate) fn heartbeat_path(marker: &Path) -> PathBuf { /// only signal that works across pid namespaces when there are no locks. Without /// one, the recorded pid is probed, which is only meaningful within one pid /// namespace. A crash is therefore recovered only once the heartbeat has gone -/// quiet for [`HEARTBEAT_STALE`] (a restart faster than that defers recovery to a +/// quiet for [`HEARTBEAT_STALE`] (three minutes; a restart faster than that defers recovery to a /// later launch). fn legacy_marker_is_dead(marker: &Path, self_pid: u32) -> bool { + // A beat whose age cannot be established (a modification time in the future: + // the file server's clock runs ahead of ours) counts as fresh, like + // `marker_is_fresh`: only a beat demonstrably quiet condemns a session. if std::fs::metadata(heartbeat_path(marker)) .and_then(|m| m.modified()) - .ok() - .and_then(|t| t.elapsed().ok()) - .is_some_and(|age| age < HEARTBEAT_STALE) + .is_ok_and(|t| t.elapsed().map(|age| age < HEARTBEAT_STALE).unwrap_or(true)) { return false; } @@ -888,6 +890,19 @@ mod tests { assert_eq!(gens(&dir), vec![1], "quiet heartbeat: crashed"); } + #[test] + fn a_future_dated_heartbeat_counts_as_alive() { + let dir = tmp(); + let m = marker(&dir, 1, "4294967290"); + let beat = heartbeat_path(&m); + std::fs::write(&beat, "").unwrap(); + let f = std::fs::OpenOptions::new().write(true).open(&beat).unwrap(); + f.set_modified(std::time::SystemTime::now() + std::time::Duration::from_secs(3600)) + .unwrap(); + drop(f); + assert!(gens(&dir).is_empty()); + } + #[test] fn claim_pending_gives_up_if_the_session_took_its_lock_since_the_scan() { let dir = tmp(); diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index c97288d..aef7dab 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -900,6 +900,9 @@ fn spawn_pipeline( let worker_data_dir = data_dir.map(Path::to_path_buf); let worker_drain_dir = drain_dir.map(Path::to_path_buf); let marker = data_dir.map(|d| Session::marker_path(d, generation)); + // Recovery can run long (a host `before_send`, many sessions); it keeps this + // session's heartbeat going meanwhile, as the worker loop is not yet running. + let beat_marker = marker.clone(); let worker_upload_tx = upload_tx.clone(); let worker = std::thread::Builder::new() @@ -930,13 +933,13 @@ fn spawn_pipeline( // it must, or that crash waits for the master to restart. Claims // keep siblings from recovering the same session twice. if let Some(dir) = &worker_drain_dir { - run_recovery(&worker_shared, dir, generation); + run_recovery(&worker_shared, dir, generation, beat_marker.as_deref()); } // Best effort, once: sessions a previous default location left // behind. No generation of ours lives there, so none is skipped. for dir in &legacy_data_dirs { queue::gc_orphans(dir); - run_recovery(&worker_shared, dir, u64::MAX); + run_recovery(&worker_shared, dir, u64::MAX, beat_marker.as_deref()); } })); legacy_ready.store(true, Ordering::SeqCst); @@ -965,8 +968,20 @@ fn spawn_pipeline( /// Recover any prior generation that ended abnormally, queueing it for delivery. /// The crashed session's state is discarded ONLY after the report is durably /// enqueued β€” a failure leaves it on disk for a later attempt (no crash loss). -fn run_recovery(shared: &Shared, data_dir: &Path, current_generation: u64) { +fn run_recovery( + shared: &Shared, + data_dir: &Path, + current_generation: u64, + heartbeat_of: Option<&Path>, +) { + let beat = || { + if let Some(marker) = heartbeat_of { + touch_heartbeat(marker); + } + }; + beat(); for pending in recovery::find_pending(data_dir, current_generation) { + beat(); // Several processes share this directory (a prefork master and its // workers, above all): exactly one recovers a given session. let Some(_claim) = recovery::claim_pending(data_dir, &pending) else { From 1d84feb8f754a2153964072a3b550e6510d66f4e Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:56:09 +0500 Subject: [PATCH 12/15] fix: never recover the same directory twice in one launch (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - legacy_is_adoptable compares against the chosen dir AND recovery_dirs by canonical path (Windows: temp fallback is the legacy location) - spawn path dedups legacy/extra dirs against the data dir and each other - seed the in-memory blacklist from legacy/extra dirs too - test: an existing temp tree is offered in exactly one role πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/runtime.rs | 34 ++++++++++++-- crates/bugsee/src/options.rs | 77 +++++++++++++++++++++++++++---- 2 files changed, 98 insertions(+), 13 deletions(-) diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 49fe3dc..15cdd3d 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -419,11 +419,14 @@ impl Recorder { config.upload_backoff_base, config.sampler, true, - config - .legacy_data_dir - .into_iter() - .chain(config.extra_data_dirs) - .collect(), + distinct_extra_dirs( + &data_dir, + config + .legacy_data_dir + .into_iter() + .chain(config.extra_data_dirs) + .collect(), + ), )?; Ok(Recorder { @@ -829,6 +832,23 @@ fn report_into( } } +/// `extra` without duplicates and without `data_dir` itself, compared by +/// canonical path. Recovering one directory twice in a launch would deliver its +/// sessions twice and burn their attempt cap. +fn distinct_extra_dirs(data_dir: &Path, extra: Vec) -> Vec { + let key = |p: &Path| p.canonicalize().unwrap_or_else(|_| p.to_path_buf()); + let mut seen = vec![key(data_dir)]; + let mut out = Vec::new(); + for dir in extra { + let k = key(&dir); + if !seen.contains(&k) { + seen.push(k); + out.push(dir); + } + } + out +} + /// Claim a session and open its part store, or fail without leaving a liveness /// marker behind (a marker with no store would be recovered as a phantom crash). fn open_disk(data_dir: &Path, caps: WindowCaps) -> std::io::Result<(Session, PartStore)> { @@ -925,6 +945,10 @@ fn spawn_pipeline( // Best effort, once: sessions a previous default location left // behind. No generation of ours lives there, so none is skipped. for dir in &legacy_data_dirs { + let known = queue::blacklisted_signatures(dir); + if !known.is_empty() { + worker_shared.mem_blacklist.lock().extend(known); + } queue::gc_orphans(dir); run_recovery(&worker_shared, dir, u64::MAX); } diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index 6688638..914f50d 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -386,8 +386,16 @@ pub(crate) fn resolve_data_dir( let legacy = temp.join("bugsee").join(&token); // Also with no usable directory: leftovers there are exactly what a launch // that cannot write anywhere new should still deliver. - let chosen = found.as_ref().map(|(_, dir)| dir.as_path()); - let legacy_dir = legacy_is_adoptable(&legacy, chosen).then_some(legacy); + // It must not be a directory we already use or recover from (on Windows the + // temp fallback IS the legacy location): a second recovery of the same + // session in one launch would deliver it twice and burn its attempt cap. + let others: Vec<&Path> = found + .as_ref() + .map(|(_, dir)| dir.as_path()) + .into_iter() + .chain(recovery_dirs.iter().map(PathBuf::as_path)) + .collect(); + let legacy_dir = legacy_is_adoptable(&legacy, &others).then_some(legacy); match found { Some((source, dir)) => DataDirResolution { dir: Some(dir), @@ -515,7 +523,7 @@ fn owned_by_us(_meta: &std::fs::Metadata) -> bool { /// would let someone else's tree, or our own live data directory, be processed /// (recovering our live session as a crash). Paths are compared canonicalised, /// not as strings, so another spelling of our own directory is rejected too. -fn legacy_is_adoptable(legacy: &Path, chosen: Option<&Path>) -> bool { +fn legacy_is_adoptable(legacy: &Path, others: &[&Path]) -> bool { let Some(parent) = legacy.parent() else { return false; }; @@ -525,11 +533,10 @@ fn legacy_is_adoptable(legacy: &Path, chosen: Option<&Path>) -> bool { let Ok(legacy) = legacy.canonicalize() else { return false; }; - match chosen { - // No live directory of ours to collide with. - None => true, - Some(chosen) => chosen.canonicalize().is_ok_and(|c| c != legacy), - } + // An unresolvable path cannot be shown distinct, so it counts as a collision. + others + .iter() + .all(|other| other.canonicalize().is_ok_and(|c| c != legacy)) } /// Prove a file can be created in `dir`. `create_new` with an unpredictable name @@ -919,6 +926,60 @@ mod tests { let _ = std::fs::remove_dir_all(&s); } + /// Platform-agnostic: wherever the temp fallback coincides with the legacy + /// location (Windows), an unwritable existing tree there must be offered for + /// recovery once, as a recovery dir or as the legacy dir, never both. + #[test] + fn an_unwritable_temp_tree_is_recovered_from_exactly_once() { + let s = scratch(); + let tmp = s.join("tmp"); + // Make BOTH the private-named tree and the legacy tree exist; on unix + // they differ, on Windows they are the same directory. + for tree in [ + tmp.join(private_temp_name()).join("tok"), + tmp.join("bugsee").join("tok"), + ] { + std::fs::create_dir_all(&tree).unwrap(); + } + // Block writes everywhere: no home, and the writability probe cannot + // create files because the trees are read-only (unix) -- elsewhere the + // dirs simply win, which is fine: the invariant below still holds. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for tree in [ + tmp.join(private_temp_name()).join("tok"), + tmp.join("bugsee").join("tok"), + ] { + std::fs::set_permissions(&tree, std::fs::Permissions::from_mode(0o500)).unwrap(); + } + } + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); + let mut all: Vec = r.recovery_dirs.clone(); + all.extend(r.legacy_dir.clone()); + all.extend(r.dir.clone()); + let canon: Vec = all.iter().filter_map(|p| p.canonicalize().ok()).collect(); + let mut unique = canon.clone(); + unique.sort(); + unique.dedup(); + assert_eq!( + canon.len(), + unique.len(), + "one directory must not be in two roles: {all:?}" + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for tree in [ + tmp.join(private_temp_name()).join("tok"), + tmp.join("bugsee").join("tok"), + ] { + let _ = std::fs::set_permissions(&tree, std::fs::Permissions::from_mode(0o755)); + } + } + let _ = std::fs::remove_dir_all(&s); + } + #[test] fn an_explicit_directory_never_asks_for_the_temp_location() { let s = scratch(); From 3b90a28b699597f8636cad576f14b86f3678a9bf Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:56:17 +0500 Subject: [PATCH 13/15] fix(core): a lock file that vanishes under a scan is not evidence of a crash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit probe returned Free both for an unheld lock and for a file that disappeared between a caller's exists() and the open. A session without kernel locks unlinks its .lock before writing its heartbeat, so a peer that saw the file could classify that live session as crashed. probe now reports Probe::Vanished for a missing file (no pre-check), and session_is_crashed treats it like Unknown and falls through to the heartbeat/pid decision. Test injects the vanish between the existence check and the probe's open. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/claim.rs | 31 +++++++++++++++++++++++------- crates/bugsee-core/src/recovery.rs | 25 +++++++++++++++++++++++- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 2154dd9..6dbacd7 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -207,7 +207,7 @@ Crash recovery needs the data directory to **outlive the process**: on a contain - **Costs of per-process locks, handled in `claim.rs`.** They do not exclude threads, so a registry of held paths does (keyed by pid and spin-locked with the holder's pid stamped in, so a `fork` child that inherits it mid-update breaks the lock instead of deadlocking). POSIX drops a process's locks on a file when *any* descriptor for it closes, so a lock file is never opened twice by one process: the registry is checked before each open, and a claim closes its descriptor before leaving the registry. - **Ordering at launch.** `Session::begin` creates the marker (`O_EXCL`), takes `.lock`, then writes the pid. Until the pid is written the marker is empty and fresh, which peers skip even if the lock reads free. A lock that cannot be held with kernel enforcement (contended past a short retry, or the filesystem rejects locks) is *removed*: an existing but unheld lock file reads as "that session died", so peers fall back to the marker's pid instead. On unix a probe asks `F_GETLK` and never takes the lock, so it cannot steal one from a launching peer. - **Unclean shutdown.** If the worker or uploader thread is detached at shutdown it may still be writing the session, so the session's lock is kept until the process exits (the kernel then drops it) rather than released on drop. -- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown (as it does for a leftover lock file that cannot be opened, e.g. on a read-only volume, so a crash is never hidden by it). A session that could not take a lock keeps a `.beat` heartbeat file, touched every 2 s by the worker (and during launch-time recovery): a beat younger than 3 minutes, or dated in the future (generous: NFS caches attributes for up to a minute and its mtime is the server's clock) means alive whatever pid the marker names, which is the only signal that works across pid namespaces without locks; with no beat the marker's pid is probed as before. The price is that a crash on such a filesystem is recovered only after its heartbeat has been quiet for three minutes, so a faster restart defers it to a later launch. Under the recovery claim the liveness decision is re-run, so a session that finished launching since the scan is never torn down. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. +- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown, and a lock file that vanishes under a peer's scan is likewise not taken as evidence of death (as a probe does for a leftover lock file that cannot be opened, e.g. on a read-only volume, so a crash is never hidden by it). A session that could not take a lock keeps a `.beat` heartbeat file, touched every 2 s by the worker (and during launch-time recovery): a beat younger than 3 minutes, or dated in the future (generous: NFS caches attributes for up to a minute and its mtime is the server's clock) means alive whatever pid the marker names, which is the only signal that works across pid namespaces without locks; with no beat the marker's pid is probed as before. The price is that a crash on such a filesystem is recovered only after its heartbeat has been quiet for three minutes, so a faster restart defers it to a later launch. Under the recovery claim the liveness decision is re-run, so a session that finished launching since the scan is never torn down. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. - **Paths.** The in-process registry canonicalises the lock file's directory, so `/tmp` vs `/private/tmp` or relative vs absolute paths are one claim. A `fork` child clears the registry (`claim::reset_after_fork`, from the fork hook) before claiming anything. - **Stale scans.** After winning a session claim the marker is re-checked, since a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s: a peer's in-flight `enqueue` looks like an orphan, and reaping it lost the report. Lock files of finished items and dead sessions are swept when free. - **Compatibility.** The `.alive` marker is still a bare pid and the lock files are new names, so older builds sharing the directory keep working. They do not take part in the locking, so a mixed-version directory (a rolling upgrade) gives no exactly-once guarantee between old and new processes, and a new process treats an old build's session by its pid as before. Run one SDK version per directory for the guarantee. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index 5c2c730..f3d0956 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -111,11 +111,15 @@ pub fn try_claim_lenient(path: &Path) -> Option> { pub(crate) enum Probe { Held, Free, + /// There is no lock file (any longer). Not the same as [`Probe::Free`]: a + /// caller that saw the file exist and finds it gone must not conclude that its + /// holder died, since a session without kernel locks unlinks its lock file. + Vanished, /// The filesystem cannot say (it rejects locks): use another liveness signal. Unknown, } -/// Is `path` currently claimed? A missing file is free. A file that cannot be +/// Is `path` currently claimed? A missing file is [`Probe::Vanished`]. A file that cannot be /// inspected is [`Probe::Unknown`], never assumed held: a crashed session's /// leftover lock file must not hide the crash on a volume that has become /// read-only. @@ -123,15 +127,18 @@ pub(crate) enum Probe { /// On unix this asks (`F_GETLK`) without taking the lock, so a probe can never /// steal the lock from a creator that has made the file but not yet locked it. pub(crate) fn probe(path: &Path) -> Probe { - if !path.exists() { - return Probe::Free; - } let key = key_of(path); if !registry().insert(&key) { return Probe::Held; // held by a thread of this process, or being probed } // Read-only: a querying lock call needs no write access, and a leftover lock // file on a read-only volume (or with 0444 mode) must still be probeable. + #[cfg(test)] + BEFORE_PROBE_OPEN.with(|h| { + if let Some(hook) = h.borrow().as_ref() { + hook(); + } + }); let result = match OpenOptions::new().read(true).open(path) { Ok(file) => { let r = match test_is_locked(&file) { @@ -145,8 +152,7 @@ pub(crate) fn probe(path: &Path) -> Probe { drop(file); // close before leaving the registry r } - // Vanished since the existence check: nothing holds it. - Err(_) if !path.exists() => Probe::Free, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Probe::Vanished, Err(_) => Probe::Unknown, }; registry().remove(&key); @@ -263,6 +269,17 @@ thread_local! { pub(crate) static FORCE_UNSUPPORTED: std::cell::Cell = const { std::cell::Cell::new(false) }; } +#[cfg(test)] +type Hook = Box; + +#[cfg(test)] +thread_local! { + /// Test hook: runs inside `probe` just before it opens the file, to simulate + /// the lock file vanishing between a caller's existence check and the probe. + pub(crate) static BEFORE_PROBE_OPEN: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + fn test_try_lock(file: &File) -> std::io::Result { #[cfg(test)] if FORCE_UNSUPPORTED.with(|f| f.get()) { @@ -569,7 +586,7 @@ mod tests { fn a_missing_lock_file_probes_free_and_is_not_created() { let dir = tmp(); let path = dir.join("nope.lock"); - assert!(matches!(probe(&path), Probe::Free)); + assert!(matches!(probe(&path), Probe::Vanished)); assert!(!path.exists()); } diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 526717b..fe50447 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -120,7 +120,10 @@ fn session_is_crashed(data_dir: &Path, generation: u64, marker: &Path, self_pid: crate::claim::Probe::Free => { return !(marker_is_empty(marker) && marker_is_fresh(marker)); } - crate::claim::Probe::Unknown => {} + // No lock file after all (it vanished since `exists`, e.g. a session + // without kernel locks removing it), or the filesystem cannot say: + // that is not evidence of death. Use the heartbeat / pid. + crate::claim::Probe::Vanished | crate::claim::Probe::Unknown => {} } } legacy_marker_is_dead(marker, self_pid) @@ -890,6 +893,26 @@ mod tests { assert_eq!(gens(&dir), vec![1], "quiet heartbeat: crashed"); } + #[test] + fn a_lock_file_that_vanishes_under_the_scan_is_not_evidence_of_a_crash() { + // A lock-less session unlinks its `.lock` before it writes its + // heartbeat; a peer that saw the file must fall back to the beat. + let dir = tmp(); + let m = marker(&dir, 1, "4294967290"); + let lock = Session::lock_path(&dir, 1); + std::fs::write(&lock, "").unwrap(); + std::fs::write(heartbeat_path(&m), "").unwrap(); + let lock2 = lock.clone(); + crate::claim::BEFORE_PROBE_OPEN.with(|h| { + *h.borrow_mut() = Some(Box::new(move || { + let _ = std::fs::remove_file(&lock2); + })); + }); + let found = gens(&dir); + crate::claim::BEFORE_PROBE_OPEN.with(|h| *h.borrow_mut() = None); + assert!(found.is_empty(), "fresh heartbeat: the session is alive"); + } + #[test] fn a_future_dated_heartbeat_counts_as_alive() { let dir = tmp(); From fbd6644169a9a087e46d0a4325a21344f62eebf3 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 17:44:30 +0500 Subject: [PATCH 14/15] fix: remember every undeletable retired bundle, not just delivered ones (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One retire_queued helper for all arms (delivered, TooManySimilar, duplicate, permanent, retries exhausted); recovered crashes consult the in-memory blacklist. Test: terminal verdicts on a read-only queue are sent once. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/runtime.rs | 30 ++++--- crates/bugsee-core/tests/storage_failures.rs | 94 ++++++++++++++++++++ 3 files changed, 114 insertions(+), 12 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 6b3cdbe..4d05c12 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -187,7 +187,7 @@ Global base scope behind `RwLock`; `with_scope` pushes a thread-local overlay. * |---|---| | Default candidate unusable, a later default works | falls through; one stderr line; `StorageStatus::skipped` lists why | | Requested directory unusable, or no default usable, or `Session::begin`/`PartStore::new` fails (also in a fork child) | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from an in-memory backlog capped at 16 (an entry counts until its delivery settles; overflow drops the newest with `DropReason::StorageUnavailable`); a `TooManySimilar` verdict is remembered in memory; a clean stop (guard drop) gives the backlog one last bounded (1.5 s) attempt, but a report still unsent when the process dies is lost; a fork child starts with an empty backlog (the parent's is the parent's to send) | -| Directory exists but cannot take NEW files (full, read-only remount) | memory mode for new reports, but the directory is still **drained and recovered from**: its queued bundles are delivered and its crashed sessions recovered (claims are best-effort there; a crash that cannot be re-queued on disk is delivered from memory). Bundles delivered but undeletable are remembered and not resent in this process; a later launch with a writable disk may resend them and the backend deduplicates. Every such rejected candidate is drained, including a full default that lost to a later writable one and the private temp tree (which must be a real directory of ours); a stale claim that cannot be unlinked counts as "no claim possible", not "held elsewhere"; the on-disk `.blacklist` seeds the in-memory one. Only a missing or non-directory path is skipped | +| Directory exists but cannot take NEW files (full, read-only remount) | memory mode for new reports, but the directory is still **drained and recovered from**: its queued bundles are delivered and its crashed sessions recovered (claims are best-effort there; a crash that cannot be re-queued on disk is delivered from memory). Bundles retired by ANY verdict (delivered, `TooManySimilar`, duplicate, permanent, retries exhausted) but undeletable are remembered and not resent in this process; a recovered crash that is dropped by a `TooManySimilar` verdict on such a tree keeps its `.alive` marker, so the next launch rebuilds it once more; a later launch with a writable disk may resend them and the backend deduplicates. Every such rejected candidate is drained, including a full default that lost to a later writable one and the private temp tree (which must be a real directory of ours); a stale claim that cannot be unlinked counts as "no claim possible", not "held elsewhere"; the on-disk `.blacklist` seeds the in-memory one. Only a missing or non-directory path is skipped | | Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and dropped; the rotation tick rewrites a missing `sessions/.alive` (a wiped-and-recreated tree would otherwise make the next launch's orphan sweep drop this session's crash); a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out; `StorageStatus::degraded` turns true and `crash_recovery_available()` false | | Disk full / permission denied at enqueue, snapshot, marker | same as above; the `reports/` snapshot dir is reclaimed by `SnapshotHandle::drop` | | Crash handler cannot write its marker | the raw `open`/`write` return values are ignored; the process still dies with the original signal (watchdog bounds the handler) | diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 15cdd3d..45bffa0 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1017,7 +1017,10 @@ fn recover_one(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSess Some(hook) => hook(&mut built.meta), None => true, }; - if !keep || queue::any_blacklisted(data_dir, &built.meta.signatures) { + if !keep + || mem_blacklisted(shared, &built.meta.signatures) + || queue::any_blacklisted(data_dir, &built.meta.signatures) + { recovery::discard(data_dir, pending); return; } @@ -1525,6 +1528,17 @@ fn uploader_loop( } } +/// Retire a queued report whose work is over (delivered, or abandoned for any +/// reason). If the directory will not let us delete it (read-only or full), it is +/// remembered so no later drain selects it again: otherwise a terminal verdict on +/// such a directory would be re-sent every 20 ms of a flush and every poll. +fn retire_queued(shared: &Shared, report: &queue::QueuedReport) { + queue::remove(report); + if report.bundle.exists() { + shared.delivered_stuck.lock().insert(report.bundle.clone()); + } +} + /// Queued reports in `dir` still awaiting delivery: everything on disk except /// bundles already delivered whose deletion failed. fn pending_reports(shared: &Shared, dir: &Path) -> Vec { @@ -1646,13 +1660,7 @@ fn deliver_queued( cached_endpoint.as_deref(), &mut endpoint_out, ) { - Ok(()) => { - queue::remove(report); - if report.bundle.exists() { - // Delivered, but the directory will not let us delete it. - shared.delivered_stuck.lock().insert(report.bundle.clone()); - } - } + Ok(()) => retire_queued(shared, report), // A transient failure OR a still-expired session are both retryable. Err(TransportError::Transient(_)) | Err(TransportError::SessionExpired) => { // Persist the presigned endpoint if the failure left one to resume @@ -1664,7 +1672,7 @@ fn deliver_queued( if count_failure { let n = retry + 1; if n >= UPLOAD_RETRY_CAP { - queue::remove(report); + retire_queued(shared, report); notify_dropped(shared, DropReason::RetriesExhausted); } else { queue::set_meta(report, n, now + backoff_delay(n, backoff_base)); @@ -1683,12 +1691,12 @@ fn deliver_queued( }; queue::blacklist_add(data_dir, &sigs); shared.mem_blacklist.lock().extend(sigs); - queue::remove(report); + retire_queued(shared, report); notify_dropped(shared, DropReason::TooManySimilar); } // Duplicate / permanent β€” abandon the report. Err(e) => { - queue::remove(report); + retire_queued(shared, report); notify_dropped(shared, drop_reason_for(e)); } } diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index 616fb67..58a81da 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -699,3 +699,97 @@ fn signatures_blacklisted_on_disk_are_honoured_in_memory_mode() { "only the report without the blacklisted signature is sent" ); } + +/// A terminal verdict on a queued report in a directory we cannot delete from +/// must not be re-sent on every drain (a flush retries every 20 ms, the uploader +/// polls every 5 s). +#[cfg(unix)] +#[test] +fn terminal_verdicts_on_an_undeletable_queue_are_not_resent() { + use bugsee_core::model::report::TimeWindow; + use bugsee_core::reporting::assemble_docs; + use bugsee_core::TransportError; + use std::os::unix::fs::PermissionsExt; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[derive(Clone, Copy)] + enum Verdict { + TooManySimilar, + Duplicate, + Permanent, + } + struct Terminal(Verdict, Arc); + impl bugsee_core::Transport for Terminal { + fn register_session(&self, _: &str, _: &[u8]) -> Result { + Ok("token".into()) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + self.1.fetch_add(1, Ordering::SeqCst); + Err(match self.0 { + Verdict::TooManySimilar => TransportError::TooManySimilar { + signatures: vec!["sig".into()], + }, + Verdict::Duplicate => TransportError::DuplicateDropped, + Verdict::Permanent => TransportError::Permanent("rejected".into()), + }) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), TransportError> { + Ok(()) + } + } + + for verdict in [ + Verdict::TooManySimilar, + Verdict::Duplicate, + Verdict::Permanent, + ] { + let dir = TempDir::new(); + let data = dir.path.join("data"); + let env = bugsee_core::model::environment::Environment::detect_with( + "t", + &Default::default(), + &Default::default(), + ); + let queued = assemble_docs( + Vec::new(), + TimeWindow { start: 0, end: 1 }, + &manual_upload_meta(), + &env, + None, + &[], + "T", + 1, + ) + .unwrap(); + bugsee_core::queue::enqueue(&data, &queued).unwrap(); + for d in [data.join("queue"), data.clone()] { + std::fs::set_permissions(d, std::fs::Permissions::from_mode(0o555)).unwrap(); + } + if std::fs::write(data.join("probe"), b"x").is_ok() { + eprintln!("skipped: permission bits are not enforced for this user"); + return; + } + + let issues = Arc::new(AtomicUsize::new(0)); + let mut config = RecorderConfig::new(&data, "T"); + config.memory_only = Some("write probe failed".into()); + let recorder = + Recorder::launch(config, Arc::new(Terminal(verdict, issues.clone()))).expect("launch"); + // A flush spins its drain loop until the deadline, the case that stormed. + let _ = recorder.flush(Duration::from_millis(600)); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!( + issues.load(Ordering::SeqCst), + 1, + "the verdict is final; the undeletable bundle must not be re-sent" + ); + drop(recorder); + restore_permissions(&data); + restore_permissions(&data.join("queue")); + } +} From 1ecfca866ecccdf02dbe5a7f20edd65ca6dfcc2c Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 19:17:23 +0500 Subject: [PATCH 15/15] fix: de-duplicate the unix libc dependency after merging main (#27's optional form vs the unconditional one) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee/Cargo.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/bugsee/Cargo.toml b/crates/bugsee/Cargo.toml index a7fc135..068b3f2 100644 --- a/crates/bugsee/Cargo.toml +++ b/crates/bugsee/Cargo.toml @@ -17,6 +17,11 @@ backtrace = "0.3" ureq = { version = "2", optional = true, features = ["json"] } sysinfo = { version = "0.33", optional = true, default-features = false, features = ["system", "disk"] } +# Unconditional on unix: the data-directory ownership checks (`options.rs`) call +# `geteuid`, and the `telemetry` feature's local-UTC-offset / hardware-model probes +# (`hostinfo.rs`) call libc too. It is already in the tree via `bugsee-core` / +# `bugsee-native`, so this adds nothing; keeping it non-optional is what lets +# `--no-default-features` build. [target.'cfg(unix)'.dependencies] libc = "0.2" @@ -32,7 +37,7 @@ panic = ["dep:bugsee-panic"] # Native fatal-crash handler (POSIX signal / Windows SEH) + next-launch recovery. native = ["dep:bugsee-native"] # System/process telemetry sampling (traces.system) via sysinfo. -telemetry = ["dep:sysinfo", "dep:libc"] +telemetry = ["dep:sysinfo"] # The crash matrix needs the REAL main thread: an uncaught panic must unwind out # of `main`, and the panic observer treats any other thread as "a thread died", @@ -43,10 +48,5 @@ name = "crash_e2e_matrix" harness = false required-features = ["native"] -# Local UTC offset and the hardware model are libc calls on unix; `libc` is -# already in the tree via `bugsee-core` / `bugsee-native`, so this adds nothing. -[target.'cfg(unix)'.dependencies] -libc = { version = "0.2", optional = true } - [target.'cfg(unix)'.dev-dependencies] libc = "0.2"