From 08a31bd75b40a82e36616dd2c38031013d2d90f4 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 14:10:18 +0500 Subject: [PATCH 01/52] feat: degrade to in-memory reporting on read-only/ephemeral file systems (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve the data directory as explicit option, BUGSEE_DATA_DIR, platform default, then temp dir, taking the first that is actually writable. If none is, or the disk fails mid-run (full, read-only, removed), launch still succeeds and handled errors / manual reports are assembled without capture context and delivered from a bounded in-memory backlog. Native crash handler and panic snapshots are skipped in memory mode. Adds Bugsee::storage_status() so a host can check crash recovery is available and whether the dir may be ephemeral. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 12 + crates/bugsee-core/src/reporting.rs | 28 +- crates/bugsee-core/src/runtime.rs | 410 +++++++++++++++---- crates/bugsee-core/src/util.rs | 11 + crates/bugsee-core/tests/storage_failures.rs | 239 +++++++++++ crates/bugsee/src/api.rs | 81 +++- crates/bugsee/src/lib.rs | 4 +- crates/bugsee/src/options.rs | 278 ++++++++++++- crates/bugsee/tests/storage_fallback.rs | 107 +++++ 9 files changed, 1082 insertions(+), 88 deletions(-) create mode 100644 crates/bugsee-core/tests/storage_failures.rs create mode 100644 crates/bugsee/tests/storage_fallback.rs diff --git a/DESIGN.md b/DESIGN.md index badb64a..3029573 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -181,6 +181,18 @@ Global base scope behind `RwLock`; `with_scope` pushes a thread-local overlay. * **Durability envelope (be honest):** buffered append survives **process death** (panic, `abort`, signal, `kill -9`) because completed `write()`s live in the kernel page cache independent of the crashing process β€” this is what the `recovery_e2e` / `kill -9` tests exercise. It does **not** guarantee survival of **power loss / kernel panic**, since nothing is `fsync`'d; a hardening tier (periodic `fsync`, or the originally-envisioned `mmap(MAP_SHARED)` + `msync`) is **future work**, not current behavior. +**Data directory resolution and failure modes (read-only / ephemeral hosts).** The facade picks the first **writable** candidate (create + write-probe, not mere existence), skipping and recording any that fail: (1) `LaunchOptions::data_dir`; (2) `BUGSEE_DATA_DIR`; (3) platform default `bugsee/` under `$XDG_DATA_HOME`/`~/.local/share` (Linux), `~/Library/Application Support` (macOS), `%LOCALAPPDATA%` (Windows), skipped if the base is unset (container with no home); (4) `/bugsee/`. Launch never fails for storage reasons. Outcomes: + +| Failure | Outcome | +|---|---| +| Preferred candidate unusable, a later one works | falls through; one stderr line; `StorageStatus::skipped` lists why | +| No candidate usable, or `Session::begin`/`PartStore::new` fails | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from a bounded in-memory backlog (16; overflow drops the newest with `DropReason::StorageUnavailable`); an unsent report dies with the process | +| Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and silently dropped; a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out | +| Disk full / permission denied at enqueue, snapshot, marker | same as above; the `reports/` snapshot dir is reclaimed by `SnapshotHandle::drop` | +| Crash handler cannot write its marker | the raw `open`/`write` return values are ignored; the process still dies with the original signal (watchdog bounds the handler) | + +Crash recovery needs the data directory to **outlive the process**: on a container, mount a volume and point `data_dir`/`BUGSEE_DATA_DIR` at it. The temp fallback is commonly wiped on restart. `Bugsee::storage_status()` reports `mode`, `data_dir`, `source`, `skipped`, `crash_recovery_available()` and `may_be_ephemeral()` (the SDK cannot tell a volume from a scratch disk; the temp fallback is the only case it flags). Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised; failures are injected by blocking path components and removing the directory mid-run. + **Generations:** each launch bumps `gen`; next-launch recovery reads the **previous** gen's parts before advancing. **Snapshot (no seal, no copy):** pin `[start,end]` (`end`=snapshot instant, `start`=`endβˆ’window`) + **hard-link current parts including the active one** into the report dir (`PartStore::snapshot_into`, after a `flush`). At export, read linked parts and include only entries with `start ≀ timestamp ≀ end` (monotonic order β†’ early-stop at the tail; the self-terminating `RecordIter` cleanly ignores any torn trailing write on the active part). Bounds become `manifest.time.start/end`. Cost: 2 timestamps + O(#parts) `link` syscalls. Fallback to copy on FAT/exFAT/cross-volume (capability-probed once). Reflink (APFS `clonefile` / Linux `FICLONE`) a noted future tier. diff --git a/crates/bugsee-core/src/reporting.rs b/crates/bugsee-core/src/reporting.rs index 1fdcb4c..3116578 100644 --- a/crates/bugsee-core/src/reporting.rs +++ b/crates/bugsee-core/src/reporting.rs @@ -14,7 +14,7 @@ use std::path::Path; use serde_json::{Map, Value}; use crate::bundle::{bundle_filename, write_bundle, BundleEntry}; -use crate::capture::export_report; +use crate::capture::{export_report, ChannelDocument}; use crate::model::enums::{IssueType, Severity}; use crate::model::environment::Environment; use crate::model::report::{FileDescriptor, IssueRequest, Manifest, Source, TimeWindow}; @@ -99,7 +99,33 @@ pub fn assemble_with_extras( created_on_ms: i64, ) -> std::io::Result { let docs = export_report(report_dir, window.start, window.end)?; + assemble_docs( + docs, + window, + meta, + env, + crash_json, + extra_files, + app_token, + created_on_ms, + ) +} +/// Like [`assemble_with_extras`] but from already-exported capture `docs` +/// instead of a snapshot directory. With an empty `docs` it builds a bundle that +/// needs no file system at all: the in-memory delivery path used when the data +/// directory is unusable (read-only, full, gone). +#[allow(clippy::too_many_arguments)] +pub fn assemble_docs( + docs: Vec, + window: TimeWindow, + meta: &ReportMeta, + env: &Environment, + crash_json: Option>, + extra_files: &[ExtraFile], + app_token: &str, + created_on_ms: i64, +) -> std::io::Result { let mut files: Vec = Vec::new(); let mut entries: Vec = Vec::new(); diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index fcb2ecf..faf62cc 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -61,6 +61,30 @@ const DEFAULT_MAX_QUEUED: usize = 8192; /// NON-crash reports are dropped-newest; crash reports are never dropped. const DEFAULT_MAX_REPORT_QUEUED: usize = 256; +/// Where a recorder keeps its state. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StorageMode { + /// The data directory is usable: capture parts, the liveness marker and the + /// retry queue are on disk, so a crash can be recovered on the next launch. + Disk, + /// No usable data directory (read-only, missing, full, ...). Handled errors + /// and manual reports are still delivered, from memory, but there is no + /// capture window and nothing survives the process: fatal crashes cannot be + /// recovered and an unsent report is lost on exit. + Memory, +} + +/// Most reports held in memory awaiting delivery. Reports are rare and a bundle +/// is small, so this only bounds a pathological storm while offline. +const MAX_MEMORY_REPORTS: usize = 16; + +/// A report waiting in memory for delivery, because it could not be queued on disk. +struct MemReport { + report: AssembledReport, + retry: u32, + next_attempt_ms: i64, +} + /// A callback run on every report before delivery. Mutate the metadata in place; /// return `false` to drop the report entirely. pub type BeforeSend = Box bool + Send + Sync>; @@ -83,6 +107,10 @@ pub enum DropReason { TooManySimilar, /// Delivery kept failing transiently until the durable retry cap. RetriesExhausted, + /// The report could not be stored on disk and the in-memory backlog was + /// full, so it was dropped rather than queued. Only seen with an unusable + /// data directory. + StorageUnavailable, } /// Called when a report is abandoned. See [`DropReason`]. @@ -136,6 +164,9 @@ pub struct RecorderConfig { pub sample_rate: f64, /// Max in-flight capture entries before new ones are dropped (back-pressure). pub max_queued_entries: usize, + /// Skip the disk entirely and run in [`StorageMode::Memory`]. For a host + /// layer that already established no data directory is usable. + pub memory_only: bool, } impl RecorderConfig { @@ -156,6 +187,7 @@ impl RecorderConfig { before_breadcrumb: None, sample_rate: 1.0, max_queued_entries: DEFAULT_MAX_QUEUED, + memory_only: false, } } } @@ -199,6 +231,8 @@ struct Shared { max_report_queued: usize, /// Whether telemetry sampling is paused (mirrors the facade's pause state). paused: AtomicBool, + /// Reports that could not be queued on disk, delivered from memory. + mem_queue: ForkLock>, } /// The recorder's own locks join the `fork()` lock protocol, so a forked child @@ -208,8 +242,10 @@ impl ForkLockable for Shared { fn fork_lock(&self) { self.scope.fork_lock(); self.session.fork_lock(); + self.mem_queue.fork_lock(); } fn fork_unlock(&self) { + self.mem_queue.fork_unlock(); self.session.fork_unlock(); self.scope.fork_unlock(); } @@ -253,7 +289,10 @@ pub struct Recorder { shared: Arc, caps: WindowCaps, data_dir: PathBuf, - session: Session, + /// `None` in memory mode: there is no liveness marker to own. + session: Option, + storage: StorageMode, + storage_reason: Option, rotate_interval: Duration, backoff_base: Duration, worker: Option>, @@ -294,17 +333,40 @@ impl Recorder { report_queued: AtomicUsize::new(0), max_report_queued: DEFAULT_MAX_REPORT_QUEUED, paused: AtomicBool::new(false), + mem_queue: ForkLock::new(Vec::new()), }); crate::fork::install_fork_hook(); crate::fork::register_weak(&shared); - let session = Session::begin(&config.data_dir)?; let data_dir = config.data_dir.clone(); + // An unusable data directory degrades to memory mode instead of failing + // the launch: the host must not lose error reporting (or its startup) + // because a volume is read-only or full. + let (disk, storage_reason) = if config.memory_only { + (None, Some("no usable data directory".to_string())) + } else { + match open_disk(&data_dir, config.caps) { + Ok(disk) => (Some(disk), None), + Err(e) => (None, Some(format!("{}: {e}", data_dir.display()))), + } + }; + if let Some(reason) = &storage_reason { + crate::util::warn_once(&format!( + "data directory unusable ({reason}); running in memory only: handled errors \ + and manual reports are still sent, but crashes cannot be recovered and \ + nothing survives this process" + )); + } + let (session, store) = match disk { + Some((session, store)) => (Some(session), Some(store)), + None => (None, None), + }; let pipeline = spawn_pipeline( &shared, - &data_dir, - session.generation(), + store.is_some().then_some(data_dir.as_path()), + session.as_ref().map_or(0, Session::generation), + store, config.caps, config.rotate_interval, config.upload_backoff_base, @@ -318,6 +380,12 @@ impl Recorder { shared, caps: config.caps, data_dir, + storage: if session.is_some() { + StorageMode::Disk + } else { + StorageMode::Memory + }, + storage_reason, session, rotate_interval: config.rotate_interval, backoff_base: config.upload_backoff_base, @@ -359,11 +427,19 @@ impl Recorder { self.shared.queued.store(0, Ordering::SeqCst); self.shared.report_queued.store(0, Ordering::SeqCst); - let session = Session::begin(&self.data_dir)?; + let disk = match self.storage { + StorageMode::Disk => Some(open_disk(&self.data_dir, self.caps)?), + StorageMode::Memory => None, + }; + let (session, store) = match disk { + Some((session, store)) => (Some(session), Some(store)), + None => (None, None), + }; let pipeline = spawn_pipeline( &self.shared, - &self.data_dir, - session.generation(), + store.is_some().then_some(self.data_dir.as_path()), + session.as_ref().map_or(0, Session::generation), + store, self.caps, self.rotate_interval, self.backoff_base, @@ -462,6 +538,18 @@ impl Recorder { &self.data_dir } + /// Where this recorder keeps its state. In [`StorageMode::Memory`] crash + /// capture is pointless (nothing could be recovered), so the host layer + /// should not install the native handler. + pub fn storage_mode(&self) -> StorageMode { + self.storage + } + + /// Why the recorder fell back to [`StorageMode::Memory`], if it did. + pub fn storage_reason(&self) -> Option<&str> { + self.storage_reason.as_deref() + } + /// Path where the native crash handler should write its crash-info marker. pub fn crash_info_path(&self) -> PathBuf { self.gen_dir().join(crate::recovery::CRASH_INFO_NAME) @@ -473,9 +561,12 @@ impl Recorder { } fn gen_dir(&self) -> PathBuf { - self.data_dir - .join("parts") - .join(self.session.generation().to_string()) + self.data_dir.join("parts").join( + self.session + .as_ref() + .map_or(0, Session::generation) + .to_string(), + ) } /// Configured window caps. @@ -525,7 +616,9 @@ impl Drop for Recorder { // unwinding, and its snapshot is deleted by `report_caught`, so this // cannot turn a handled panic into a phantom crash. if worker_acked && uploader_acked && !std::thread::panicking() { - self.session.end(); + if let Some(session) = &self.session { + session.end(); + } } } } @@ -660,9 +753,19 @@ fn report_into( } } -/// Recover any prior generation that ended abnormally, queueing it for delivery. -/// The crashed session's state is discarded ONLY after the report is durably -/// enqueued β€” a failure leaves it on disk for a later attempt (no crash loss). +/// Claim a session and open its part store, or fail without leaving a liveness +/// marker behind (a marker with no store would be recovered as a phantom crash). +fn open_disk(data_dir: &Path, caps: WindowCaps) -> std::io::Result<(Session, PartStore)> { + let session = Session::begin(data_dir)?; + match PartStore::new(data_dir, session.generation(), caps) { + Ok(store) => Ok((session, store)), + Err(e) => { + session.end(); + Err(e) + } + } +} + /// Start the uploader and the capture worker for `generation`. /// /// `fresh_launch` is true for a process's first pipeline and false for one @@ -671,8 +774,9 @@ fn report_into( #[allow(clippy::too_many_arguments)] fn spawn_pipeline( shared: &Arc, - data_dir: &Path, + data_dir: Option<&Path>, generation: u64, + store: Option, caps: WindowCaps, rotate_interval: Duration, backoff_base: Duration, @@ -682,7 +786,7 @@ fn spawn_pipeline( // Uploader thread: drains the durable queue with retry. let (upload_tx, upload_rx) = channel(); let uploader_shared = Arc::clone(shared); - let uploader_data_dir = data_dir.to_path_buf(); + let uploader_data_dir = data_dir.map(Path::to_path_buf); let uploader = std::thread::Builder::new() .name("bugsee-uploader".into()) .spawn(move || { @@ -697,9 +801,8 @@ fn spawn_pipeline( // Capture worker: owns the part store; enqueues reports. let (tx, rx) = channel(); - let store = PartStore::new(data_dir, generation, caps)?; let worker_shared = Arc::clone(shared); - let worker_data_dir = data_dir.to_path_buf(); + let worker_data_dir = data_dir.map(Path::to_path_buf); let worker_upload_tx = upload_tx.clone(); let worker = std::thread::Builder::new() @@ -713,15 +816,17 @@ fn spawn_pipeline( // first (runs before this session enqueues anything). Only a fresh // launch: a fork child's parent may be mid-enqueue right now, and // its in-flight sidecars would look orphaned. - if fresh_launch { - queue::gc_orphans(&worker_data_dir); + if let (true, Some(dir)) = (fresh_launch, &worker_data_dir) { + queue::gc_orphans(dir); } // A fork child recovers too. A prefork master recovers only at ITS // launch, and its own generation is the lowest, so it can never // recover a worker that died later; the next worker started from // it must, or that crash waits for the master to restart. Claims // keep siblings from recovering the same session twice. - run_recovery(&worker_shared, &worker_data_dir, generation); + if let Some(dir) = &worker_data_dir { + run_recovery(&worker_shared, dir, generation); + } })); let _ = worker_upload_tx.send(UploadMsg::Wake); worker_loop( @@ -744,6 +849,9 @@ fn spawn_pipeline( }) } +/// Recover any prior generation that ended abnormally, queueing it for delivery. +/// The crashed session's state is discarded ONLY after the report is durably +/// enqueued β€” a failure leaves it on disk for a later attempt (no crash loss). fn run_recovery(shared: &Shared, data_dir: &Path, current_generation: u64) { for pending in recovery::find_pending(data_dir, current_generation) { // Several processes share this directory (a prefork master and its @@ -827,10 +935,10 @@ fn merge_scope(shared: &Shared, meta: &mut ReportMeta) { #[allow(clippy::too_many_arguments)] fn worker_loop( rx: Receiver, - mut store: PartStore, + mut store: Option, shared: Arc, rotate_interval: Duration, - data_dir: PathBuf, + data_dir: Option, caps: WindowCaps, mut sampler: Option>, upload_tx: Sender, @@ -850,6 +958,8 @@ fn worker_loop( shared.queued.fetch_sub(1, Ordering::Relaxed); // A panic while serializing/appending must not kill the worker. let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + // Memory mode has no capture window: the entry is dropped. + let Some(store) = store.as_mut() else { return }; if let Ok(value) = entry.to_json() { if let Ok(bytes) = serde_json::to_vec(&value) { let _ = store.append(entry.wire_type(), entry.timestamp(), &bytes); @@ -865,7 +975,13 @@ fn worker_loop( shared.report_queued.fetch_sub(1, Ordering::Relaxed); let ok = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { process_report( - &mut store, &shared, &data_dir, caps, meta, window_end, crash_json, + store.as_mut(), + &shared, + data_dir.as_deref(), + caps, + meta, + window_end, + crash_json, ) .is_ok() })) @@ -876,7 +992,7 @@ fn worker_loop( } Ok(Msg::Snapshot { window_end, ack }) => { let handle = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - take_snapshot(&mut store, &data_dir, caps, window_end) + take_snapshot(store.as_mut()?, data_dir.as_deref()?, caps, window_end) })) .ok() .flatten(); @@ -884,7 +1000,7 @@ fn worker_loop( } Ok(Msg::UploadSnapshot { handle, meta }) => { let ok = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - deliver_snapshot(&shared, &data_dir, handle, meta) + deliver_snapshot(&shared, data_dir.as_deref(), handle, meta) })) .unwrap_or(false); if ok { @@ -892,11 +1008,15 @@ fn worker_loop( } } Ok(Msg::Flush(ack)) => { - let _ = store.flush(); + if let Some(store) = store.as_mut() { + let _ = store.flush(); + } let _ = ack.send(()); } Ok(Msg::Stop(ack)) => { - let _ = store.flush(); + if let Some(store) = store.as_mut() { + let _ = store.flush(); + } let _ = ack.send(()); break; } @@ -906,7 +1026,7 @@ fn worker_loop( // Run the rotation/eviction/telemetry tick whenever the interval has // elapsed β€” via a message OR a timeout β€” so steady traffic can't starve it. if Instant::now() >= next_rotate { - rotate_tick(&mut store, &shared, &mut sampler); + rotate_tick(store.as_mut(), &shared, &mut sampler); next_rotate = Instant::now() + rotate_interval; } } @@ -916,7 +1036,7 @@ fn worker_loop( /// window. Panic-contained so a host `TelemetrySampler` panic can't kill the /// worker. fn rotate_tick( - store: &mut PartStore, + mut store: Option<&mut PartStore>, shared: &Shared, sampler: &mut Option>, ) { @@ -934,13 +1054,15 @@ fn rotate_tick( value, custom: Default::default(), }; - if let Ok(bytes) = serde_json::to_vec(&entry) { + if let (Some(store), Ok(bytes)) = (store.as_mut(), serde_json::to_vec(&entry)) { let _ = store.append("traces.system", ts, &bytes); } } } } - let _ = store.rotate(); + if let Some(store) = store.as_mut() { + let _ = store.rotate(); + } })); } @@ -971,7 +1093,7 @@ fn take_snapshot( /// Assemble + enqueue a report from a pre-taken snapshot (deferred upload path). fn deliver_snapshot( shared: &Shared, - data_dir: &Path, + data_dir: Option<&Path>, handle: SnapshotHandle, mut meta: ReportMeta, ) -> bool { @@ -982,7 +1104,7 @@ fn deliver_snapshot( } } // The snapshot dir is reclaimed by `handle`'s Drop on return (any path). - reporting::assemble( + let assembled = reporting::assemble( &handle.dir, handle.window, &meta, @@ -991,14 +1113,73 @@ fn deliver_snapshot( &shared.app_token, handle.window.end, ) - .and_then(|assembled| queue::enqueue(data_dir, &assembled)) - .is_ok() + // The snapshot became unreadable (disk gone): still send what we know. + .or_else(|_| assemble_in_memory(shared, &meta, handle.window, None)); + match assembled { + Ok(assembled) => persist_or_hold(shared, data_dir, assembled), + Err(_) => false, + } +} + +/// Queue `assembled` on disk, or hold it in memory if the disk refuses it +/// (read-only, full, directory removed) or there is no disk. Always `true`: the +/// report is held either way. +fn persist_or_hold(shared: &Shared, data_dir: Option<&Path>, assembled: AssembledReport) -> bool { + if let Some(dir) = data_dir { + if queue::enqueue(dir, &assembled).is_ok() { + return true; + } + // A failed enqueue can leave a partial sidecar set; the next launch's + // orphan sweep reaps it. Do not retry the disk for this report. + } + hold_in_memory(shared, assembled); + true +} + +/// Assemble a bundle with no capture context and no file system access. +fn assemble_in_memory( + shared: &Shared, + meta: &ReportMeta, + window: TimeWindow, + crash_json: Option>, +) -> std::io::Result { + reporting::assemble_docs( + Vec::new(), + window, + meta, + &shared.env, + crash_json, + &[], + &shared.app_token, + window.end, + ) +} + +/// Park a report in the bounded in-memory queue for the uploader. Over the cap +/// the NEWEST report is dropped (and the host told), keeping the backlog bounded. +fn hold_in_memory(shared: &Shared, report: AssembledReport) { + let full = { + let mut queue = shared.mem_queue.lock(); + if queue.len() >= MAX_MEMORY_REPORTS { + true + } else { + queue.push(MemReport { + report, + retry: 0, + next_attempt_ms: 0, + }); + false + } + }; + if full { + notify_dropped(shared, DropReason::StorageUnavailable); + } } fn process_report( - store: &mut PartStore, + store: Option<&mut PartStore>, shared: &Shared, - data_dir: &Path, + data_dir: Option<&Path>, caps: WindowCaps, mut meta: ReportMeta, window_end: i64, @@ -1022,49 +1203,80 @@ fn process_report( } // Suppress reports the server blacklisted (12004 TooManySimilar). - if queue::any_blacklisted(data_dir, &meta.signatures) { - return Ok(()); + if let Some(dir) = data_dir { + if queue::any_blacklisted(dir, &meta.signatures) { + return Ok(()); + } } let window_start = window_end.saturating_sub(caps.max_window_ms); - let report_id = crate::util::random_hex(8); - let report_dir = data_dir.join("reports").join(&report_id); - std::fs::create_dir_all(&report_dir)?; - - // Own the report dir with a `SnapshotHandle` so EVERY exit path reclaims its - // hard links via Drop. The earlier manual `remove_dir_all` ran only after a - // successful assemble, so a `snapshot_into`/`assemble` failure (most plausibly - // disk-full) permanently orphaned a `reports/` dir full of hard-linked - // parts, and `reports/` was never GC'd (F7). let window = TimeWindow { start: window_start, end: window_end, }; + + // With a usable disk, snapshot + assemble from it. Any failure along the way + // (most plausibly disk-full or a vanished directory) falls back to a bundle + // without capture context, so the report still goes out. + let from_disk = match (store, data_dir) { + (Some(store), Some(data_dir)) => { + assemble_from_disk(store, shared, data_dir, &meta, window, crash_json.clone()) + } + _ => None, + }; + let assembled = match from_disk { + Some(assembled) => assembled, + None => assemble_in_memory(shared, &meta, window, crash_json)?, + }; + + // Persist to the durable queue; the uploader delivers it. Failing that, hold + // it in memory. + persist_or_hold(shared, data_dir, assembled); + Ok(()) +} + +/// Snapshot the window into a fresh report dir and assemble it, or `None` if the +/// disk refuses any step. +fn assemble_from_disk( + store: &mut PartStore, + shared: &Shared, + data_dir: &Path, + meta: &ReportMeta, + window: TimeWindow, + crash_json: Option>, +) -> Option { + let report_id = crate::util::random_hex(8); + let report_dir = data_dir.join("reports").join(&report_id); + // Own the report dir with a `SnapshotHandle` BEFORE creating it so EVERY exit + // path reclaims its hard links via Drop. The earlier manual `remove_dir_all` + // ran only after a successful assemble, so a `snapshot_into`/`assemble` + // failure (most plausibly disk-full) permanently orphaned a `reports/` dir + // full of hard-linked parts, and `reports/` was never GC'd (F7). let handle = SnapshotHandle { dir: report_dir, window, }; + std::fs::create_dir_all(&handle.dir).ok()?; - store.snapshot_into(&handle.dir, window_start, window_end)?; - let assembled = reporting::assemble( + store + .snapshot_into(&handle.dir, window.start, window.end) + .ok()?; + reporting::assemble( &handle.dir, window, - &meta, + meta, &shared.env, crash_json, &shared.app_token, - window_end, - )?; - - // Persist to the durable queue; the uploader delivers it. `handle`'s Drop - // reclaims the snapshot dir on return (success or the `?` paths above). - queue::enqueue(data_dir, &assembled) + window.end, + ) + .ok() } fn uploader_loop( rx: Receiver, shared: Arc, - data_dir: PathBuf, + data_dir: Option, backoff_base: Duration, fresh_launch: bool, ) { @@ -1075,19 +1287,19 @@ fn uploader_loop( // panicking Transport impl can't kill the uploader. if fresh_launch { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain(&shared, &data_dir, backoff_base, true) + drain(&shared, data_dir.as_deref(), backoff_base, true) })); } loop { match rx.recv_timeout(poll) { Ok(UploadMsg::Wake) => { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain(&shared, &data_dir, backoff_base, false) + drain(&shared, data_dir.as_deref(), backoff_base, false) })); } Ok(UploadMsg::Drain { deadline, ack }) => { let empty = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_until_empty(&shared, &data_dir, backoff_base, deadline) + drain_until_empty(&shared, data_dir.as_deref(), backoff_base, deadline) })) .unwrap_or(false); let _ = ack.send(empty); @@ -1098,7 +1310,7 @@ fn uploader_loop( } Err(RecvTimeoutError::Timeout) => { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain(&shared, &data_dir, backoff_base, false) + drain(&shared, data_dir.as_deref(), backoff_base, false) })); } Err(RecvTimeoutError::Disconnected) => break, @@ -1109,9 +1321,12 @@ fn uploader_loop( /// Attempt delivery of every due queued report once. On the flush path `force` /// is set so backing-off reports are retried immediately. Returns whether any /// report was attempted (used to pace `drain_until_empty`). -fn drain(shared: &Shared, data_dir: &Path, backoff_base: Duration, force: bool) -> bool { +fn drain(shared: &Shared, data_dir: Option<&Path>, backoff_base: Duration, force: bool) -> bool { let now = epoch_ms(); - let mut attempted = false; + let mut attempted = drain_memory(shared, backoff_base, force, true); + let Some(data_dir) = data_dir else { + return attempted; + }; for report in queue::list_pending(data_dir) { // Another process sharing the queue may be delivering this one. let Some(_claim) = queue::try_claim(&report) else { @@ -1242,6 +1457,61 @@ fn notify_dropped(shared: &Shared, reason: DropReason) { } } +/// Attempt delivery of the in-memory backlog. Mirrors [`deliver_queued`]: a +/// transient failure keeps the report (counting toward the retry cap only on the +/// backoff-paced path), anything terminal drops it and tells the host. The queue +/// is taken out for the duration so the lock is never held across the network. +fn drain_memory(shared: &Shared, backoff_base: Duration, force: bool, count_failure: bool) -> bool { + let pending = std::mem::take(&mut *shared.mem_queue.lock()); + if pending.is_empty() { + return false; + } + let now = epoch_ms(); + let mut attempted = false; + let mut keep = Vec::new(); + for mut item in pending { + if !force && now < item.next_attempt_ms { + keep.push(item); + continue; + } + attempted = true; + let mut endpoint_out = None; + match transport::deliver( + &*shared.transport, + &shared.app_token, + &shared.environment_json, + &shared.session, + &item.report, + None, + &mut endpoint_out, + ) { + Ok(()) => {} + Err(TransportError::Transient(_)) | Err(TransportError::SessionExpired) => { + if count_failure { + item.retry += 1; + if item.retry >= UPLOAD_RETRY_CAP { + notify_dropped(shared, DropReason::RetriesExhausted); + continue; + } + item.next_attempt_ms = now + backoff_delay(item.retry, backoff_base); + } + keep.push(item); + } + Err(TransportError::TooManySimilar { .. }) => { + notify_dropped(shared, DropReason::TooManySimilar); + } + Err(e) => notify_dropped(shared, drop_reason_for(e)), + } + } + if !keep.is_empty() { + // Reports held while we were delivering stay behind the retained ones. + let mut queue = shared.mem_queue.lock(); + keep.append(&mut queue); + *queue = keep; + } + attempted +} + /// Exponential backoff: `base * 2^(retry-1)`, capped at 300 s. fn backoff_delay(retry: u32, base: Duration) -> i64 { let shift = retry.saturating_sub(1).min(20); @@ -1260,14 +1530,15 @@ fn backoff_delay(retry: u32, base: Duration) -> i64 { /// whether the queue ended up empty. fn drain_until_empty( shared: &Shared, - data_dir: &Path, + data_dir: Option<&Path>, backoff_base: Duration, deadline: Duration, ) -> bool { let start = Instant::now(); loop { let now = epoch_ms(); - for report in queue::list_pending(data_dir) { + drain_memory(shared, backoff_base, true, false); + for report in data_dir.map(queue::list_pending).unwrap_or_default() { let Some(_claim) = queue::try_claim(&report) else { continue; // a peer is delivering it; the loop below waits it out }; @@ -1276,7 +1547,7 @@ fn drain_until_empty( }; deliver_queued( shared, - data_dir, + data_dir.unwrap_or(Path::new("")), backoff_base, &report, zip, @@ -1289,7 +1560,8 @@ fn drain_until_empty( break; } } - if queue::list_pending(data_dir).is_empty() { + let disk_empty = data_dir.is_none_or(|d| queue::list_pending(d).is_empty()); + if disk_empty && shared.mem_queue.lock().is_empty() { return true; } if start.elapsed() >= deadline { diff --git a/crates/bugsee-core/src/util.rs b/crates/bugsee-core/src/util.rs index 4a00dae..ed27ce0 100644 --- a/crates/bugsee-core/src/util.rs +++ b/crates/bugsee-core/src/util.rs @@ -10,6 +10,17 @@ use std::time::{SystemTime, UNIX_EPOCH}; +/// Print `message` to stderr the first time it is called in this process and do +/// nothing after. For conditions the SDK must announce (it otherwise does no +/// logging) but that would be noise if repeated, such as a degraded data dir. +pub fn warn_once(message: &str) { + use std::sync::atomic::{AtomicBool, Ordering}; + static WARNED: AtomicBool = AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + eprintln!("bugsee: {message}"); + } +} + /// Current wall-clock time in milliseconds since the Unix epoch. pub fn epoch_ms() -> i64 { match SystemTime::now().duration_since(UNIX_EPOCH) { diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs new file mode 100644 index 0000000..94f6b1b --- /dev/null +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -0,0 +1,239 @@ +// +// storage_failures.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! An unusable data directory must degrade, never fail the host. +//! +//! Each test drives a real `Recorder` through one failure mode and asserts the +//! documented outcome: no panic, no hang, and the report still reaches the +//! transport (from memory where the disk refuses it). +//! +//! Failures are injected by making a path component unusable (a regular file +//! where a directory is needed, a read-only directory, a directory removed +//! mid-run). A true ENOSPC needs a size-capped filesystem and root to mount one; +//! writing under a path blocked by a regular file fails the same way at the +//! same call sites (`create_dir_all` / `write` returning an `io::Error`). + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use bugsee_core::model::entry::{CaptureEntry, EventEntry}; +use bugsee_core::reporting::manual_upload_meta; +use bugsee_core::runtime::StorageMode; +use bugsee_core::util::epoch_ms; +use bugsee_core::{MockTransport, Recorder, RecorderConfig}; +use serde_json::Map; + +struct TempDir { + path: PathBuf, +} +impl TempDir { + fn new() -> Self { + let mut path = std::env::temp_dir(); + path.push(format!( + "bugsee-storage-{}", + bugsee_core::util::random_hex(8) + )); + std::fs::create_dir_all(&path).unwrap(); + TempDir { path } + } +} +impl Drop for TempDir { + fn drop(&mut self) { + restore_permissions(&self.path); + let _ = std::fs::remove_dir_all(&self.path); + } +} + +#[cfg(unix)] +fn restore_permissions(path: &Path) { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)); +} +#[cfg(not(unix))] +fn restore_permissions(_path: &Path) {} + +fn event(name: &str) -> CaptureEntry { + CaptureEntry::UserEvent(EventEntry { + timestamp: epoch_ms(), + name: Some(name.into()), + params: Map::new(), + custom: Map::new(), + }) +} + +/// Report once and wait for delivery; returns how many bundles arrived. +fn report_and_flush(recorder: &Recorder, transport: &MockTransport) -> usize { + recorder.capture(event("before")); + recorder.report(manual_upload_meta(), None); + assert!( + recorder.flush(Duration::from_secs(10)), + "flush must complete, not hang" + ); + let n = transport.uploaded_bundles.lock().unwrap().len(); + n +} + +/// A path that can never be created: a child of a regular file. +fn blocked_path(dir: &Path) -> PathBuf { + let file = dir.join("not-a-dir"); + std::fs::write(&file, b"x").unwrap(); + file.join("data") +} + +#[test] +fn missing_uncreatable_directory_runs_in_memory_and_still_reports() { + let dir = TempDir::new(); + let transport = Arc::new(MockTransport::default()); + + let recorder = Recorder::launch( + RecorderConfig::new(blocked_path(&dir.path), "T"), + transport.clone(), + ) + .expect("an unusable directory must not fail launch"); + + assert_eq!(recorder.storage_mode(), StorageMode::Memory); + assert!(recorder.storage_reason().is_some()); + assert_eq!(report_and_flush(&recorder, &transport), 1); +} + +#[test] +fn memory_only_config_skips_the_disk_entirely() { + let dir = TempDir::new(); + let transport = Arc::new(MockTransport::default()); + let data_dir = dir.path.join("must-stay-absent"); + let mut config = RecorderConfig::new(&data_dir, "T"); + config.memory_only = true; + + let recorder = Recorder::launch(config, transport.clone()).expect("launch"); + + assert_eq!(recorder.storage_mode(), StorageMode::Memory); + assert_eq!(report_and_flush(&recorder, &transport), 1); + drop(recorder); + assert!(!data_dir.exists(), "memory mode must not touch the disk"); +} + +#[cfg(unix)] +#[test] +fn read_only_directory_runs_in_memory_and_still_reports() { + use std::os::unix::fs::PermissionsExt; + let dir = TempDir::new(); + let ro = dir.path.join("ro"); + std::fs::create_dir(&ro).unwrap(); + std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o555)).unwrap(); + // root ignores permission bits, which would make this test vacuous. + if std::fs::write(ro.join("probe"), b"x").is_ok() { + eprintln!("skipped: permission bits are not enforced for this user"); + return; + } + let transport = Arc::new(MockTransport::default()); + + let recorder = Recorder::launch(RecorderConfig::new(&ro, "T"), transport.clone()) + .expect("a read-only directory must not fail launch"); + + assert_eq!(recorder.storage_mode(), StorageMode::Memory); + assert_eq!(report_and_flush(&recorder, &transport), 1); +} + +#[test] +fn directory_removed_mid_run_still_reports() { + let dir = TempDir::new(); + let data_dir = dir.path.join("data"); + let transport = Arc::new(MockTransport::default()); + let recorder = + Recorder::launch(RecorderConfig::new(&data_dir, "T"), transport.clone()).expect("launch"); + assert_eq!(recorder.storage_mode(), StorageMode::Disk); + recorder.capture(event("early")); + assert!(recorder.flush(Duration::from_secs(10))); + + // The volume vanishes under a running process. + std::fs::remove_dir_all(&data_dir).unwrap(); + + assert_eq!(report_and_flush(&recorder, &transport), 1); + // Capture and shutdown after the loss must not panic either. + recorder.capture(event("late")); + drop(recorder); +} + +#[test] +fn directory_replaced_by_a_file_mid_run_falls_back_to_memory_delivery() { + let dir = TempDir::new(); + let data_dir = dir.path.join("data"); + let transport = Arc::new(MockTransport::default()); + let recorder = + Recorder::launch(RecorderConfig::new(&data_dir, "T"), transport.clone()).expect("launch"); + + // Nothing under the data dir can be created any more: every disk write site + // (parts, report snapshot, queue) fails. + std::fs::remove_dir_all(&data_dir).unwrap(); + std::fs::write(&data_dir, b"a file now").unwrap(); + + assert_eq!(report_and_flush(&recorder, &transport), 1); + drop(recorder); +} + +#[test] +fn queue_write_failure_holds_the_report_in_memory() { + let dir = TempDir::new(); + let data_dir = dir.path.join("data"); + std::fs::create_dir_all(&data_dir).unwrap(); + // The queue directory cannot be created: the enqueue step fails while the + // capture parts and the liveness marker work. This is what a full disk or a + // quota looks like at the queue. + std::fs::write(data_dir.join("queue"), b"blocker").unwrap(); + let transport = Arc::new(MockTransport::default()); + + let recorder = + Recorder::launch(RecorderConfig::new(&data_dir, "T"), transport.clone()).expect("launch"); + assert_eq!(recorder.storage_mode(), StorageMode::Disk); + + assert_eq!(report_and_flush(&recorder, &transport), 1); +} + +#[test] +fn memory_backlog_is_bounded_and_overflow_is_reported() { + use bugsee_core::runtime::DropReason; + use bugsee_core::transport::{Transport, TransportError}; + use std::sync::Mutex; + + // Never delivers, so everything stays in the memory backlog. + struct Offline; + impl Transport for Offline { + fn register_session(&self, _: &str, _: &[u8]) -> Result { + Err(TransportError::Transient("offline".into())) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + Err(TransportError::Transient("offline".into())) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), TransportError> { + Err(TransportError::Transient("offline".into())) + } + } + + let dir = TempDir::new(); + let dropped = Arc::new(Mutex::new(Vec::new())); + let sink = dropped.clone(); + let mut config = RecorderConfig::new(blocked_path(&dir.path), "T"); + config.on_report_dropped = Some(Box::new(move |r| sink.lock().unwrap().push(r))); + let recorder = Recorder::launch(config, Arc::new(Offline)).expect("launch"); + + for _ in 0..40 { + recorder.report(manual_upload_meta(), None); + } + // Flush cannot succeed (offline) but must return promptly, not hang. + assert!(!recorder.flush(Duration::from_millis(500))); + + let seen = dropped.lock().unwrap().clone(); + assert!( + !seen.is_empty() && seen.iter().all(|r| *r == DropReason::StorageUnavailable), + "overflow must be surfaced, got {seen:?}" + ); +} diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 138dace..02b552d 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -23,8 +23,9 @@ use bugsee_core::util::epoch_ms; use bugsee_core::{Recorder, RecorderConfig}; use serde_json::{Map, Value}; -use crate::options::LaunchOptions; +use crate::options::{DataDirResolution, DataDirSource, LaunchOptions, StorageStatus}; use crate::report::Report; +use bugsee_core::runtime::StorageMode; // A `Mutex` (not `RwLock`) because `mpsc::Sender` inside `Recorder` is `Send` // but not `Sync`; the lock is held only long enough to enqueue. @@ -44,6 +45,14 @@ struct ReviveInfo { } static REVIVE_INFO: ForkMutex> = ForkMutex::new(|| None); +/// How the current launch resolved its data directory, for `storage_status`. +struct Resolved { + dir: Option, + source: DataDirSource, + skipped: Vec, +} +static RESOLUTION: ForkMutex> = ForkMutex::new(|| None); + /// The fork epoch this process last reconciled with. static SEEN_FORK_EPOCH: AtomicU64 = AtomicU64::new(0); @@ -71,8 +80,22 @@ impl Bugsee { pub fn launch_with(options: LaunchOptions) -> io::Result { bugsee_core::fork::install_fork_hook(); check_fork(); - let data_dir = options.resolved_data_dir(); - std::fs::create_dir_all(&data_dir)?; + // No writable directory is not an error: the recorder runs in memory so + // handled errors still go out (see `DataDirSource`). + let DataDirResolution { + dir, + source, + skipped, + } = options.resolve_data_dir(); + let memory_only = dir.is_none(); + let data_dir = dir.unwrap_or_else(|| std::env::temp_dir().join("bugsee")); + if !memory_only && !skipped.is_empty() { + bugsee_core::util::warn_once(&format!( + "data directory unusable ({}); using {}", + skipped.join("; "), + data_dir.display() + )); + } let transport = match options.transport.clone() { Some(t) => t, @@ -94,6 +117,7 @@ impl Bugsee { }; let mut config = RecorderConfig::new(data_dir, options.app_token.clone()); + config.memory_only = memory_only; config.caps = options.caps(); config.rotate_interval = options.rotate_interval; config.before_send = options.before_send; @@ -121,6 +145,18 @@ impl Bugsee { } let recorder = Recorder::launch(config, transport)?; + // Nothing can be recovered without a disk, so crash markers and panic + // snapshots have nowhere to go. + let persistent = recorder.storage_mode() == StorageMode::Disk; + *RESOLUTION.lock() = Some(Resolved { + dir: persistent.then(|| recorder.data_dir().to_path_buf()), + source: if persistent { + source + } else { + DataDirSource::None + }, + skipped, + }); #[cfg(feature = "panic")] let panic_info_path = recorder.panic_info_path(); @@ -128,7 +164,7 @@ impl Bugsee { // Install the native crash handler pointing at this generation's marker. // Replace under a brief lock, then drop the OLD handler outside it. #[cfg(feature = "native")] - if options.native_crash_capture { + if options.native_crash_capture && persistent { if let Ok(handler) = bugsee_native::install(recorder.crash_info_path()) { let old = NATIVE.lock().replace(handler); drop(old); @@ -161,7 +197,11 @@ impl Bugsee { // SIGABRT on the next launch β€” and so an uncaught UNWINDING panic is // recovered as a fatal panic (the marker is kept because // `Recorder::drop` sees `thread::panicking()`). - bugsee_panic::set_snapshot_path(panic_info_path); + if persistent { + bugsee_panic::set_snapshot_path(panic_info_path); + } else { + bugsee_panic::clear_snapshot_path(); + } // Opt-in: report immediately from the hook instead of via recovery. bugsee_panic::set_report_from_hook(options.report_panics_from_hook); } @@ -169,6 +209,33 @@ impl Bugsee { Ok(LaunchGuard { _private: () }) } + /// Where the SDK is keeping its state, or `None` before `launch`. + /// + /// Use it to confirm a deployment is set up for crash recovery: it needs + /// [`StorageStatus::crash_recovery_available`], and on a container a data + /// directory on a volume rather than ephemeral storage + /// ([`StorageStatus::may_be_ephemeral`]). + pub fn storage_status() -> Option { + check_fork(); + let (mode, reason) = { + let guard = RECORDER.lock(); + let recorder = guard.as_ref()?; + ( + recorder.storage_mode(), + recorder.storage_reason().map(String::from), + ) + }; + let resolved = RESOLUTION.lock(); + let resolved = resolved.as_ref()?; + Some(StorageStatus { + mode, + data_dir: resolved.dir.clone(), + source: resolved.source, + skipped: resolved.skipped.clone(), + reason, + }) + } + /// Stop the SDK and flush pending work (dropping the recorder joins the worker). pub fn stop() { check_fork(); @@ -496,7 +563,9 @@ fn revive_after_fork(epoch: u64) { } } #[cfg(feature = "panic")] - bugsee_panic::set_snapshot_path(recorder.panic_info_path()); + if recorder.storage_mode() == StorageMode::Disk { + bugsee_panic::set_snapshot_path(recorder.panic_info_path()); + } *RECORDER.lock() = Some(recorder); *failures = 0; diff --git a/crates/bugsee/src/lib.rs b/crates/bugsee/src/lib.rs index 867dde3..ce610ef 100644 --- a/crates/bugsee/src/lib.rs +++ b/crates/bugsee/src/lib.rs @@ -67,7 +67,7 @@ macro_rules! app_package_id { } pub use api::{Bugsee, LaunchGuard, ResultExt}; -pub use options::LaunchOptions; +pub use options::{DataDirSource, LaunchOptions, StorageStatus, DATA_DIR_ENV}; pub use perf::{Span, Transaction}; pub use report::Report; @@ -84,5 +84,5 @@ pub use bugsee_core::model::enums::{ }; pub use bugsee_core::model::perf::Status; pub use bugsee_core::reporting::{Attachment, ReportMeta}; -pub use bugsee_core::runtime::DropReason; +pub use bugsee_core::runtime::{DropReason, StorageMode}; pub use bugsee_core::transport::{Transport, TransportError}; diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index 9d64222..acd48e3 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -8,7 +8,8 @@ //! Typed launch configuration. Mirrors the mobile SDK option surface, adapted //! to a builder rather than a string-keyed registry. -use std::path::PathBuf; +use std::ffi::OsString; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -121,7 +122,12 @@ impl LaunchOptions { self } - /// Override the on-disk data directory (defaults to a per-app temp dir). + /// Set the on-disk data directory, taking precedence over everything else. + /// + /// The directory holds the capture window, the liveness marker and the retry + /// queue, and crash recovery works only if it OUTLIVES the process: on a + /// container, point it at a volume. See [`DataDirSource`] for the full + /// resolution order and what happens when it is not writable. pub fn data_dir(mut self, dir: impl Into) -> Self { self.data_dir = Some(dir.into()); self @@ -225,14 +231,162 @@ impl LaunchOptions { } } - pub(crate) fn resolved_data_dir(&self) -> PathBuf { - self.data_dir.clone().unwrap_or_else(|| { - let mut dir = std::env::temp_dir(); - dir.push("bugsee"); - // Isolate by app token so multiple apps don't collide. - dir.push(short_token(&self.app_token)); - dir - }) + /// Pick the data directory: the first candidate in [`DataDirSource`] order + /// that can actually be created and written to. + pub(crate) fn resolve_data_dir(&self) -> DataDirResolution { + resolve_data_dir( + self.data_dir.as_deref(), + &self.app_token, + &|name| std::env::var_os(name), + &std::env::temp_dir(), + ) + } +} + +/// Environment variable naming the data directory when +/// [`LaunchOptions::data_dir`] is not set. +pub const DATA_DIR_ENV: &str = "BUGSEE_DATA_DIR"; + +/// Which step of the resolution order produced the data directory. +/// +/// Candidates are tried in this order and the first one that is writable wins; +/// an unusable one is skipped (and recorded in +/// [`StorageStatus::skipped`](crate::StorageStatus::skipped)), never fatal: +/// +/// 1. [`Explicit`](DataDirSource::Explicit): [`LaunchOptions::data_dir`]. +/// 2. [`Environment`](DataDirSource::Environment): `BUGSEE_DATA_DIR`. +/// 3. [`PlatformDefault`](DataDirSource::PlatformDefault): `bugsee/` under +/// `$XDG_DATA_HOME` (else `~/.local/share`) on Linux, `~/Library/Application +/// Support` on macOS, `%LOCALAPPDATA%` on Windows. Skipped when the base is +/// unset, as in a container with no home directory. +/// 4. [`TempFallback`](DataDirSource::TempFallback): `bugsee/` under the +/// system temp directory. Often wiped on restart, so crash recovery may not +/// survive it; [`StorageStatus::may_be_ephemeral`](crate::StorageStatus::may_be_ephemeral) +/// reports it. +/// +/// If none is writable the SDK runs +/// [in memory](bugsee_core::runtime::StorageMode::Memory): handled errors and +/// manual reports are still sent, but there is no capture window and no crash +/// recovery. It logs one line to stderr when it degrades. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DataDirSource { + Explicit, + Environment, + PlatformDefault, + TempFallback, + /// No directory is in use (memory mode). + None, +} + +/// The outcome of [`resolve_data_dir`]. +pub(crate) struct DataDirResolution { + /// The chosen directory, or `None` if no candidate was writable. + pub dir: Option, + pub source: DataDirSource, + /// Candidates tried and rejected before the winner, with the reason. + pub skipped: Vec, +} + +/// Pure resolution core: `env` and `temp` are injected so tests need not touch +/// the process environment. +pub(crate) fn resolve_data_dir( + explicit: Option<&Path>, + app_token: &str, + env: &dyn Fn(&str) -> Option, + temp: &Path, +) -> DataDirResolution { + let mut candidates: Vec<(DataDirSource, PathBuf)> = Vec::new(); + if let Some(dir) = explicit { + candidates.push((DataDirSource::Explicit, dir.to_path_buf())); + } + if let Some(dir) = env(DATA_DIR_ENV).filter(|v| !v.is_empty()) { + candidates.push((DataDirSource::Environment, PathBuf::from(dir))); + } + // Isolate by app token so multiple apps don't collide. + let app = |base: PathBuf| base.join("bugsee").join(short_token(app_token)); + if let Some(base) = platform_data_base(env) { + candidates.push((DataDirSource::PlatformDefault, app(base))); + } + candidates.push((DataDirSource::TempFallback, app(temp.to_path_buf()))); + + let mut skipped = Vec::new(); + for (source, dir) in candidates { + match probe_writable(&dir) { + Ok(()) => { + return DataDirResolution { + dir: Some(dir), + source, + skipped, + } + } + Err(e) => skipped.push(format!("{}: {e}", dir.display())), + } + } + DataDirResolution { + dir: None, + source: DataDirSource::None, + skipped, + } +} + +/// The per-user data base directory for this platform, if its inputs are set. +fn platform_data_base(env: &dyn Fn(&str) -> Option) -> Option { + let set = |name: &str| env(name).filter(|v| !v.is_empty()).map(PathBuf::from); + if cfg!(windows) { + set("LOCALAPPDATA") + } else if cfg!(target_vendor = "apple") { + set("HOME").map(|h| h.join("Library").join("Application Support")) + } else { + set("XDG_DATA_HOME").or_else(|| set("HOME").map(|h| h.join(".local").join("share"))) + } +} + +/// Create `dir` and prove a file can be written in it. Checking writability (not +/// just existence) is what catches a read-only root file system or a volume +/// mounted read-only. +fn probe_writable(dir: &Path) -> std::io::Result<()> { + use std::io::Write; + std::fs::create_dir_all(dir)?; + let probe = dir.join(format!(".bugsee-probe-{}", std::process::id())); + let result = std::fs::File::create(&probe).and_then(|mut f| f.write_all(b"1")); + let _ = std::fs::remove_file(&probe); + result +} + +/// Where the SDK keeps its state right now; see +/// [`Bugsee::storage_status`](crate::Bugsee::storage_status). +/// +/// On ephemeral storage, crash recovery only works if the directory outlives +/// the process (a volume mount). The SDK cannot tell a volume from a scratch +/// disk, so check [`may_be_ephemeral`](StorageStatus::may_be_ephemeral) and +/// your own deployment. +#[derive(Debug, Clone)] +pub struct StorageStatus { + /// `Disk` when the data directory is usable; `Memory` when it is not. + pub mode: bugsee_core::runtime::StorageMode, + /// The directory in use; `None` in memory mode. + pub data_dir: Option, + /// Which resolution step produced `data_dir`. + pub source: DataDirSource, + /// Candidates rejected before the one in use, with the error for each. + pub skipped: Vec, + /// Why the SDK is in memory mode, if it is. + pub reason: Option, +} + +impl StorageStatus { + /// Whether a fatal crash can be recovered on the next launch: true only with + /// a usable data directory. (It also needs that directory to survive the + /// process; see [`may_be_ephemeral`](Self::may_be_ephemeral).) + pub fn crash_recovery_available(&self) -> bool { + self.mode == bugsee_core::runtime::StorageMode::Disk + } + + /// Whether the directory is the temp-dir fallback, which is commonly wiped + /// on restart and so may not preserve a crash for the next launch. Set + /// [`LaunchOptions::data_dir`] (or `BUGSEE_DATA_DIR`) to a volume to fix it. + pub fn may_be_ephemeral(&self) -> bool { + self.source == DataDirSource::TempFallback } } @@ -243,3 +397,107 @@ fn short_token(token: &str) -> String { .take(16) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch() -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "bugsee-resolve-{}", + bugsee_core::util::random_hex(8) + )); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + /// A path no directory can be created at: a child of a regular file. + fn blocked(scratch: &Path, name: &str) -> PathBuf { + let file = scratch.join(name); + std::fs::write(&file, b"x").unwrap(); + file.join("sub") + } + + fn env_of<'a>(vars: &'a [(&'a str, OsString)]) -> impl Fn(&str) -> Option + 'a { + move |name| { + vars.iter() + .find(|(k, _)| *k == name) + .map(|(_, v)| v.clone()) + } + } + + #[test] + fn explicit_directory_wins_when_writable() { + let s = scratch(); + let explicit = s.join("explicit"); + let env = [(DATA_DIR_ENV, s.join("from-env").into_os_string())]; + let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&env), &s.join("tmp")); + assert_eq!(r.source, DataDirSource::Explicit); + assert_eq!(r.dir.as_deref(), Some(explicit.as_path())); + assert!(r.skipped.is_empty()); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn unwritable_explicit_falls_through_to_the_environment_variable() { + let s = scratch(); + let bad = blocked(&s, "f1"); + let good = s.join("from-env"); + let env = [(DATA_DIR_ENV, good.clone().into_os_string())]; + let r = resolve_data_dir(Some(&bad), "tok", &env_of(&env), &s.join("tmp")); + assert_eq!(r.source, DataDirSource::Environment); + assert_eq!(r.dir, Some(good)); + assert_eq!(r.skipped.len(), 1, "the rejected candidate is recorded"); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn platform_default_is_used_when_home_is_set() { + let s = scratch(); + let home = s.join("home"); + let env = [ + ("HOME", home.clone().into_os_string()), + ("XDG_DATA_HOME", home.clone().into_os_string()), + ("LOCALAPPDATA", home.clone().into_os_string()), + ]; + let r = resolve_data_dir(None, "tok-1", &env_of(&env), &s.join("tmp")); + assert_eq!(r.source, DataDirSource::PlatformDefault); + assert!(r.dir.unwrap().starts_with(&home)); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn no_home_falls_back_to_the_temp_directory() { + let s = scratch(); + let tmp = s.join("tmp"); + let r = resolve_data_dir(None, "tok-1", &env_of(&[]), &tmp); + assert_eq!(r.source, DataDirSource::TempFallback); + assert_eq!(r.dir, Some(tmp.join("bugsee").join("tok1"))); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn nothing_writable_resolves_to_none_and_lists_every_attempt() { + let s = scratch(); + let env = [(DATA_DIR_ENV, blocked(&s, "f2").into_os_string())]; + let r = resolve_data_dir( + Some(&blocked(&s, "f3")), + "tok", + &env_of(&env), + &blocked(&s, "f4"), + ); + assert_eq!(r.source, DataDirSource::None); + assert!(r.dir.is_none()); + assert_eq!(r.skipped.len(), 3); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn an_empty_environment_variable_is_ignored() { + let s = scratch(); + let env = [(DATA_DIR_ENV, OsString::new())]; + let r = resolve_data_dir(None, "tok", &env_of(&env), &s.join("tmp")); + assert_eq!(r.source, DataDirSource::TempFallback); + let _ = std::fs::remove_dir_all(&s); + } +} diff --git a/crates/bugsee/tests/storage_fallback.rs b/crates/bugsee/tests/storage_fallback.rs new file mode 100644 index 0000000..ba82295 --- /dev/null +++ b/crates/bugsee/tests/storage_fallback.rs @@ -0,0 +1,107 @@ +// +// storage_fallback.rs +// bugsee +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! The facade on read-only / ephemeral hosts: `launch` must succeed and handled +//! errors must still go out when no data directory is usable. +//! +//! One test function on purpose: the SDK is a process-wide singleton and the +//! scenarios steer it through process environment variables. + +#![cfg(unix)] + +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use bugsee::core::MockTransport; +use bugsee::{Bugsee, DataDirSource, LaunchOptions, LogLevel, StorageMode, DATA_DIR_ENV}; + +fn scratch() -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "bugsee-fallback-{}", + bugsee::core::util::random_hex(8) + )); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +#[test] +fn launch_degrades_through_the_resolution_order() { + let s = scratch(); + // A path nothing can be created at. + let file = s.join("blocker"); + std::fs::write(&file, b"x").unwrap(); + let blocked = file.join("data"); + + // Keep every default away from the real home directory and temp dir. + std::env::remove_var("HOME"); + std::env::remove_var("XDG_DATA_HOME"); + std::env::set_var("TMPDIR", &file); + std::env::remove_var(DATA_DIR_ENV); + + assert!(Bugsee::storage_status().is_none(), "nothing before launch"); + + // 1. A writable explicit directory is used as given. + let good = s.join("explicit"); + let transport = Arc::new(MockTransport::default()); + let guard = Bugsee::launch_with( + LaunchOptions::new("TOKEN_A") + .data_dir(&good) + .with_transport(transport.clone()), + ) + .expect("launch"); + let status = Bugsee::storage_status().expect("status"); + assert_eq!(status.mode, StorageMode::Disk); + assert_eq!(status.source, DataDirSource::Explicit); + assert_eq!(status.data_dir.as_deref(), Some(good.as_path())); + assert!(status.crash_recovery_available()); + assert!(!status.may_be_ephemeral()); + drop(guard); + + // 2. An unusable explicit directory falls through to the environment variable. + let from_env = s.join("from-env"); + std::env::set_var(DATA_DIR_ENV, &from_env); + let guard = Bugsee::launch_with( + LaunchOptions::new("TOKEN_A") + .data_dir(&blocked) + .with_transport(transport.clone()), + ) + .expect("launch"); + let status = Bugsee::storage_status().expect("status"); + assert_eq!(status.source, DataDirSource::Environment); + assert_eq!(status.data_dir.as_deref(), Some(from_env.as_path())); + assert_eq!(status.skipped.len(), 1); + drop(guard); + std::env::remove_var(DATA_DIR_ENV); + + // 3. Nothing usable anywhere: memory mode, and reports still go out. + let transport = Arc::new(MockTransport::default()); + let guard = Bugsee::launch_with( + LaunchOptions::new("TOKEN_A") + .data_dir(&blocked) + .with_transport(transport.clone()), + ) + .expect("an unwritable environment must not fail launch"); + let status = Bugsee::storage_status().expect("status"); + assert_eq!(status.mode, StorageMode::Memory); + assert_eq!(status.source, DataDirSource::None); + assert!(status.data_dir.is_none()); + assert!(!status.crash_recovery_available()); + assert!(!status.skipped.is_empty()); + + Bugsee::event("ignored_without_a_window"); + Bugsee::capture_message(LogLevel::Error, "handled error with no disk"); + Bugsee::create_report().upload(); + assert!(Bugsee::flush(Duration::from_secs(10))); + assert_eq!( + transport.uploaded_bundles.lock().unwrap().len(), + 2, + "both reports must be delivered from memory" + ); + drop(guard); + + let _ = std::fs::remove_dir_all(&s); +} From 184d102976d76c186b1b636d1359ca3d31589bfe Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 14:25:12 +0500 Subject: [PATCH 02/52] fix: address review of data-directory fallback (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - memory backlog: entries stay queued while in flight so the cap is real; cache the presigned endpoint (no duplicate issue on a retried PUT); honour the flush deadline; clear the backlog in fork children - a requested data_dir / BUGSEE_DATA_DIR is final: unusable means memory mode (explicit_rejected), never another disk location - temp fallback is bugsee-/, 0700, symlink/owner checked; probe uses create_new with a random name - recover and deliver leftovers from the previous default location once - mid-run degradation is visible (StorageStatus::degraded); fork child falls back to memory instead of failing; warn_once per message, never panics - #[non_exhaustive] on the new public types; ffi test no longer writes to the real data directory πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 18 +- crates/bugsee-core/src/reporting.rs | 1 + crates/bugsee-core/src/runtime.rs | 294 +++++++++++---- crates/bugsee-core/src/util.rs | 17 +- crates/bugsee-core/tests/storage_failures.rs | 224 ++++++++++-- crates/bugsee-ffi/tests/lifecycle.rs | 7 +- crates/bugsee/Cargo.toml | 3 + crates/bugsee/src/api.rs | 65 ++-- crates/bugsee/src/options.rs | 365 ++++++++++++++++--- crates/bugsee/tests/fork_safety.rs | 78 ++++ crates/bugsee/tests/storage_fallback.rs | 27 +- 11 files changed, 894 insertions(+), 205 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 3029573..f2f03f6 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -181,17 +181,25 @@ Global base scope behind `RwLock`; `with_scope` pushes a thread-local overlay. * **Durability envelope (be honest):** buffered append survives **process death** (panic, `abort`, signal, `kill -9`) because completed `write()`s live in the kernel page cache independent of the crashing process β€” this is what the `recovery_e2e` / `kill -9` tests exercise. It does **not** guarantee survival of **power loss / kernel panic**, since nothing is `fsync`'d; a hardening tier (periodic `fsync`, or the originally-envisioned `mmap(MAP_SHARED)` + `msync`) is **future work**, not current behavior. -**Data directory resolution and failure modes (read-only / ephemeral hosts).** The facade picks the first **writable** candidate (create + write-probe, not mere existence), skipping and recording any that fail: (1) `LaunchOptions::data_dir`; (2) `BUGSEE_DATA_DIR`; (3) platform default `bugsee/` under `$XDG_DATA_HOME`/`~/.local/share` (Linux), `~/Library/Application Support` (macOS), `%LOCALAPPDATA%` (Windows), skipped if the base is unset (container with no home); (4) `/bugsee/`. Launch never fails for storage reasons. Outcomes: +**Data directory resolution and failure modes (read-only / ephemeral hosts).** The facade picks the data directory in this order: (1) `LaunchOptions::data_dir`; (2) `BUGSEE_DATA_DIR`; (3) platform default `bugsee/` under `$XDG_DATA_HOME`/`~/.local/share` (Linux), `~/Library/Application Support` (macOS), `%LOCALAPPDATA%` (Windows), skipped if the base is unset (container with no home); (4) `/bugsee-/` (`bugsee/` on Windows). A candidate must pass a create + `create_new` write-probe (not mere existence). **A directory the host asked for (1 or 2) is final**: if unusable the SDK does not use another disk location, it runs in memory and reports why in `StorageStatus::explicit_rejected`; only the defaults (3, 4) fall through to each other. The temp fallback is created `0700` per component and refused if a component is a symlink or owned by another user (the unix temp dir is world-writable). Launch never fails for storage reasons. Outcomes: | Failure | Outcome | |---|---| -| Preferred candidate unusable, a later one works | falls through; one stderr line; `StorageStatus::skipped` lists why | -| No candidate usable, or `Session::begin`/`PartStore::new` fails | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from a bounded in-memory backlog (16; overflow drops the newest with `DropReason::StorageUnavailable`); an unsent report dies with the process | -| Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and silently dropped; a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out | +| Default candidate unusable, a later default works | falls through; one stderr line; `StorageStatus::skipped` lists why | +| Requested directory unusable, or no default usable, or `Session::begin`/`PartStore::new` fails (also in a fork child) | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from an in-memory backlog capped at 16 (an entry counts until its delivery settles; overflow drops the newest with `DropReason::StorageUnavailable`); an unsent report dies with the process; a fork child starts with an empty backlog (the parent's is the parent's to send) | +| Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and dropped; a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out; `StorageStatus::degraded` turns true and `crash_recovery_available()` false | | Disk full / permission denied at enqueue, snapshot, marker | same as above; the `reports/` snapshot dir is reclaimed by `SnapshotHandle::drop` | | Crash handler cannot write its marker | the raw `open`/`write` return values are ignored; the process still dies with the original signal (watchdog bounds the handler) | -Crash recovery needs the data directory to **outlive the process**: on a container, mount a volume and point `data_dir`/`BUGSEE_DATA_DIR` at it. The temp fallback is commonly wiped on restart. `Bugsee::storage_status()` reports `mode`, `data_dir`, `source`, `skipped`, `crash_recovery_available()` and `may_be_ephemeral()` (the SDK cannot tell a volume from a scratch disk; the temp fallback is the only case it flags). Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised; failures are injected by blocking path components and removing the directory mid-run. +The in-memory backlog retries like the disk queue: it caches the presigned endpoint (a failed PUT resumes at the PUT, no duplicate issue), backs off on the scheduled path, and a flush honours its deadline. + +Crash recovery needs the data directory to **outlive the process**: on a container, mount a volume and point `data_dir`/`BUGSEE_DATA_DIR` at it. The temp fallback is commonly wiped on restart. `Bugsee::storage_status()` reports `mode`, `data_dir`, `source`, `skipped`, `explicit_rejected`, `degraded`, `crash_recovery_available()` and `may_be_ephemeral()` (the SDK cannot tell a volume from a scratch disk; the temp fallback is the only case it flags). No FFI accessor yet (follow-up). + +**Release note, default location moved.** Earlier builds defaulted to `/bugsee/`; the default is now the platform location (3) or the private temp fallback (4). When no directory was requested and the old location exists, is a real directory and is owned by the current user, the launch recovers its crashed sessions and delivers its queue once, best effort (delivery is asynchronous and `flush` does not wait for it). It is not claimed to be lossless: a legacy session whose process is still alive is skipped, and a legacy directory that fails the ownership check is left alone. Hosts that set `data_dir` were never using it. + +**Unsafe.** The facade makes two `geteuid()` calls (unix) to name and vet the private temp directory, alongside core's single `kill(pid, 0)`. + +Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, a memory-mode case in `bugsee/tests/fork_safety.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised, since mounting one needs root; failures are injected by blocking path components and removing the directory mid-run. **Generations:** each launch bumps `gen`; next-launch recovery reads the **previous** gen's parts before advancing. diff --git a/crates/bugsee-core/src/reporting.rs b/crates/bugsee-core/src/reporting.rs index 3116578..a91844c 100644 --- a/crates/bugsee-core/src/reporting.rs +++ b/crates/bugsee-core/src/reporting.rs @@ -54,6 +54,7 @@ pub struct ExtraFile { } /// A finished, ready-to-deliver report. +#[derive(Clone)] pub struct AssembledReport { /// `.bundle.zip` file name. pub bundle_name: String, diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index faf62cc..21b16a5 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -26,7 +26,7 @@ //! limit. use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender}; use std::sync::Arc; use std::thread::JoinHandle; @@ -62,6 +62,7 @@ const DEFAULT_MAX_QUEUED: usize = 8192; const DEFAULT_MAX_REPORT_QUEUED: usize = 256; /// Where a recorder keeps its state. +#[non_exhaustive] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum StorageMode { /// The data directory is usable: capture parts, the liveness marker and the @@ -80,9 +81,14 @@ const MAX_MEMORY_REPORTS: usize = 16; /// A report waiting in memory for delivery, because it could not be queued on disk. struct MemReport { + /// Identifies the entry across the unlocked delivery attempt. + id: u64, report: AssembledReport, retry: u32, next_attempt_ms: i64, + /// Presigned upload endpoint cached after a failed PUT, so a retry resumes at + /// the PUT instead of minting a duplicate issue (mirrors the disk queue). + endpoint: Option, } /// A callback run on every report before delivery. Mutate the metadata in place; @@ -164,9 +170,14 @@ pub struct RecorderConfig { pub sample_rate: f64, /// Max in-flight capture entries before new ones are dropped (back-pressure). pub max_queued_entries: usize, - /// Skip the disk entirely and run in [`StorageMode::Memory`]. For a host - /// layer that already established no data directory is usable. - pub memory_only: bool, + /// `Some(reason)` skips the disk entirely and runs in [`StorageMode::Memory`]. + /// For a host layer that already established no data directory is usable; + /// the reason is logged and reported by [`Recorder::storage_reason`]. + pub memory_only: Option, + /// A previous default data directory to drain once at launch: recover its + /// crashed sessions and deliver its queue. For a host layer that moved its + /// default location and must not strand reports left at the old one. + pub legacy_data_dir: Option, } impl RecorderConfig { @@ -187,7 +198,8 @@ impl RecorderConfig { before_breadcrumb: None, sample_rate: 1.0, max_queued_entries: DEFAULT_MAX_QUEUED, - memory_only: false, + memory_only: None, + legacy_data_dir: None, } } } @@ -231,8 +243,13 @@ struct Shared { max_report_queued: usize, /// Whether telemetry sampling is paused (mirrors the facade's pause state). paused: AtomicBool, - /// Reports that could not be queued on disk, delivered from memory. + /// Reports that could not be queued on disk, delivered from memory. An entry + /// stays here until its delivery is settled, so the length is the real backlog. mem_queue: ForkLock>, + next_mem_id: AtomicU64, + /// Set the first time a disk write fails while running in disk mode: the + /// recorder is no longer fully durable (see [`Recorder::is_degraded`]). + degraded: AtomicBool, } /// The recorder's own locks join the `fork()` lock protocol, so a forked child @@ -334,6 +351,8 @@ impl Recorder { max_report_queued: DEFAULT_MAX_REPORT_QUEUED, paused: AtomicBool::new(false), mem_queue: ForkLock::new(Vec::new()), + next_mem_id: AtomicU64::new(0), + degraded: AtomicBool::new(false), }); crate::fork::install_fork_hook(); @@ -343,8 +362,8 @@ impl Recorder { // An unusable data directory degrades to memory mode instead of failing // the launch: the host must not lose error reporting (or its startup) // because a volume is read-only or full. - let (disk, storage_reason) = if config.memory_only { - (None, Some("no usable data directory".to_string())) + let (disk, storage_reason) = if let Some(reason) = config.memory_only.clone() { + (None, Some(reason)) } else { match open_disk(&data_dir, config.caps) { Ok(disk) => (Some(disk), None), @@ -352,11 +371,15 @@ impl Recorder { } }; if let Some(reason) = &storage_reason { - crate::util::warn_once(&format!( - "data directory unusable ({reason}); running in memory only: handled errors \ - and manual reports are still sent, but crashes cannot be recovered and \ - nothing survives this process" - )); + static WARNED: AtomicBool = AtomicBool::new(false); + crate::util::warn_once( + &WARNED, + &format!( + "data directory unusable ({reason}); running in memory only: handled \ + errors and manual reports are still sent, but crashes cannot be \ + recovered and nothing survives this process" + ), + ); } let (session, store) = match disk { Some((session, store)) => (Some(session), Some(store)), @@ -372,6 +395,7 @@ impl Recorder { config.upload_backoff_base, config.sampler, true, + config.legacy_data_dir, )?; Ok(Recorder { @@ -427,8 +451,21 @@ impl Recorder { self.shared.queued.store(0, Ordering::SeqCst); self.shared.report_queued.store(0, Ordering::SeqCst); + // The parent's unsent in-memory reports are the PARENT's to send; the child + // inheriting them would deliver every one a second time. + self.shared.mem_queue.lock().clear(); + + // A child that cannot open its own session degrades to memory rather than + // failing: the parent's disk may be gone or read-only by now. let disk = match self.storage { - StorageMode::Disk => Some(open_disk(&self.data_dir, self.caps)?), + StorageMode::Disk => match open_disk(&self.data_dir, self.caps) { + Ok(disk) => Some(disk), + Err(e) => { + self.storage = StorageMode::Memory; + self.storage_reason = Some(format!("{}: {e}", self.data_dir.display())); + None + } + }, StorageMode::Memory => None, }; let (session, store) = match disk { @@ -445,6 +482,7 @@ impl Recorder { self.backoff_base, sampler, false, + None, )?; // The parent's session is still the parent's: leave its marker alone. @@ -545,6 +583,13 @@ impl Recorder { self.storage } + /// Whether a disk write has failed since launch (disk full, permission + /// denied, directory removed). Reports still go out, rebuilt without capture + /// context or held in memory, but crash recovery can no longer be relied on. + pub fn is_degraded(&self) -> bool { + self.shared.degraded.load(Ordering::Relaxed) + } + /// Why the recorder fell back to [`StorageMode::Memory`], if it did. pub fn storage_reason(&self) -> Option<&str> { self.storage_reason.as_deref() @@ -782,11 +827,13 @@ fn spawn_pipeline( backoff_base: Duration, sampler: Option>, fresh_launch: bool, + legacy_data_dir: Option, ) -> std::io::Result { // Uploader thread: drains the durable queue with retry. let (upload_tx, upload_rx) = channel(); let uploader_shared = Arc::clone(shared); let uploader_data_dir = data_dir.map(Path::to_path_buf); + let uploader_legacy = legacy_data_dir.clone(); let uploader = std::thread::Builder::new() .name("bugsee-uploader".into()) .spawn(move || { @@ -794,6 +841,7 @@ fn spawn_pipeline( upload_rx, uploader_shared, uploader_data_dir, + uploader_legacy, backoff_base, fresh_launch, ) @@ -827,6 +875,11 @@ fn spawn_pipeline( if let Some(dir) = &worker_data_dir { run_recovery(&worker_shared, dir, generation); } + // Best effort, once: sessions a previous default location left + // behind. No generation of ours lives there, so none is skipped. + if let Some(dir) = &legacy_data_dir { + run_recovery(&worker_shared, dir, u64::MAX); + } })); let _ = worker_upload_tx.send(UploadMsg::Wake); worker_loop( @@ -962,7 +1015,12 @@ fn worker_loop( let Some(store) = store.as_mut() else { return }; if let Ok(value) = entry.to_json() { if let Ok(bytes) = serde_json::to_vec(&value) { - let _ = store.append(entry.wire_type(), entry.timestamp(), &bytes); + if store + .append(entry.wire_type(), entry.timestamp(), &bytes) + .is_err() + { + shared.degraded.store(true, Ordering::Relaxed); + } } } })); @@ -1114,7 +1172,10 @@ fn deliver_snapshot( handle.window.end, ) // The snapshot became unreadable (disk gone): still send what we know. - .or_else(|_| assemble_in_memory(shared, &meta, handle.window, None)); + .or_else(|_| { + shared.degraded.store(true, Ordering::Relaxed); + assemble_in_memory(shared, &meta, handle.window, None) + }); match assembled { Ok(assembled) => persist_or_hold(shared, data_dir, assembled), Err(_) => false, @@ -1129,6 +1190,7 @@ fn persist_or_hold(shared: &Shared, data_dir: Option<&Path>, assembled: Assemble if queue::enqueue(dir, &assembled).is_ok() { return true; } + shared.degraded.store(true, Ordering::Relaxed); // A failed enqueue can leave a partial sidecar set; the next launch's // orphan sweep reaps it. Do not retry the disk for this report. } @@ -1164,9 +1226,11 @@ fn hold_in_memory(shared: &Shared, report: AssembledReport) { true } else { queue.push(MemReport { + id: shared.next_mem_id.fetch_add(1, Ordering::Relaxed), report, retry: 0, next_attempt_ms: 0, + endpoint: None, }); false } @@ -1218,6 +1282,7 @@ fn process_report( // With a usable disk, snapshot + assemble from it. Any failure along the way // (most plausibly disk-full or a vanished directory) falls back to a bundle // without capture context, so the report still goes out. + let has_disk = store.is_some() && data_dir.is_some(); let from_disk = match (store, data_dir) { (Some(store), Some(data_dir)) => { assemble_from_disk(store, shared, data_dir, &meta, window, crash_json.clone()) @@ -1226,7 +1291,12 @@ fn process_report( }; let assembled = match from_disk { Some(assembled) => assembled, - None => assemble_in_memory(shared, &meta, window, crash_json)?, + None => { + if has_disk { + shared.degraded.store(true, Ordering::Relaxed); + } + assemble_in_memory(shared, &meta, window, crash_json)? + } }; // Persist to the durable queue; the uploader delivers it. Failing that, hold @@ -1277,24 +1347,38 @@ fn uploader_loop( rx: Receiver, shared: Arc, data_dir: Option, + legacy_data_dir: Option, backoff_base: Duration, fresh_launch: bool, ) { let poll = Duration::from_secs(5); + let legacy = legacy_data_dir; // A fresh launch retries any queued report immediately (force), ignoring // backoff scheduled by the prior session. A fork child skips this: the // parent's uploader is already working the shared queue. Guarded so a // panicking Transport impl can't kill the uploader. if fresh_launch { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain(&shared, data_dir.as_deref(), backoff_base, true) + drain_all( + &shared, + data_dir.as_deref(), + legacy.as_deref(), + backoff_base, + true, + ) })); } loop { match rx.recv_timeout(poll) { Ok(UploadMsg::Wake) => { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain(&shared, data_dir.as_deref(), backoff_base, false) + drain_all( + &shared, + data_dir.as_deref(), + legacy.as_deref(), + backoff_base, + false, + ) })); } Ok(UploadMsg::Drain { deadline, ack }) => { @@ -1310,7 +1394,13 @@ fn uploader_loop( } Err(RecvTimeoutError::Timeout) => { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain(&shared, data_dir.as_deref(), backoff_base, false) + drain_all( + &shared, + data_dir.as_deref(), + legacy.as_deref(), + backoff_base, + false, + ) })); } Err(RecvTimeoutError::Disconnected) => break, @@ -1318,15 +1408,36 @@ fn uploader_loop( } } +/// [`drain`] the data directory, then the legacy one a launch was asked to adopt. +fn drain_all( + shared: &Shared, + data_dir: Option<&Path>, + legacy: Option<&Path>, + backoff_base: Duration, + force: bool, +) -> bool { + let attempted = drain(shared, data_dir, backoff_base, force); + match legacy { + Some(dir) => drain_disk(shared, dir, backoff_base, force) | attempted, + None => attempted, + } +} + /// Attempt delivery of every due queued report once. On the flush path `force` /// is set so backing-off reports are retried immediately. Returns whether any /// report was attempted (used to pace `drain_until_empty`). fn drain(shared: &Shared, data_dir: Option<&Path>, backoff_base: Duration, force: bool) -> bool { + let attempted = drain_memory(shared, backoff_base, force, true, None); + match data_dir { + Some(dir) => drain_disk(shared, dir, backoff_base, force) | attempted, + None => attempted, + } +} + +/// The on-disk half of [`drain`]. +fn drain_disk(shared: &Shared, data_dir: &Path, backoff_base: Duration, force: bool) -> bool { let now = epoch_ms(); - let mut attempted = drain_memory(shared, backoff_base, force, true); - let Some(data_dir) = data_dir else { - return attempted; - }; + let mut attempted = false; for report in queue::list_pending(data_dir) { // Another process sharing the queue may be delivering this one. let Some(_claim) = queue::try_claim(&report) else { @@ -1459,55 +1570,86 @@ fn notify_dropped(shared: &Shared, reason: DropReason) { /// Attempt delivery of the in-memory backlog. Mirrors [`deliver_queued`]: a /// transient failure keeps the report (counting toward the retry cap only on the -/// backoff-paced path), anything terminal drops it and tells the host. The queue -/// is taken out for the duration so the lock is never held across the network. -fn drain_memory(shared: &Shared, backoff_base: Duration, force: bool, count_failure: bool) -> bool { - let pending = std::mem::take(&mut *shared.mem_queue.lock()); - if pending.is_empty() { - return false; - } - let now = epoch_ms(); +/// backoff-paced path), anything terminal drops it and tells the host. +/// +/// An entry STAYS in the queue while it is being delivered (the lock is held only +/// to pick it and to settle it, never across the network), so the queue length is +/// the true backlog and [`MAX_MEMORY_REPORTS`] holds. Stops at `deadline`, like +/// the disk loop in [`drain_until_empty`]. +fn drain_memory( + shared: &Shared, + backoff_base: Duration, + force: bool, + count_failure: bool, + deadline: Option, +) -> bool { let mut attempted = false; - let mut keep = Vec::new(); - for mut item in pending { - if !force && now < item.next_attempt_ms { - keep.push(item); - continue; + // Each entry is tried once per call, whatever its outcome. + let mut tried: Vec = Vec::new(); + loop { + if deadline.is_some_and(|d| Instant::now() >= d) { + break; } + let now = epoch_ms(); + let next = shared + .mem_queue + .lock() + .iter() + .find(|m| !tried.contains(&m.id) && (force || now >= m.next_attempt_ms)) + .map(|m| (m.id, m.report.clone(), m.endpoint.clone())); + let Some((id, report, cached_endpoint)) = next else { + break; + }; + tried.push(id); attempted = true; + let mut endpoint_out = None; - match transport::deliver( + let result = transport::deliver( &*shared.transport, &shared.app_token, &shared.environment_json, &shared.session, - &item.report, - None, + &report, + cached_endpoint.as_deref(), &mut endpoint_out, - ) { - Ok(()) => {} + ); + + // Settle: `None` keeps the (updated) entry, `Some(reason)` ends it. + let mut dropped = None; + let mut queue = shared.mem_queue.lock(); + let Some(pos) = queue.iter().position(|m| m.id == id) else { + continue; // cleared under us (fork child); nothing to settle + }; + match result { + Ok(()) => { + queue.remove(pos); + } Err(TransportError::Transient(_)) | Err(TransportError::SessionExpired) => { + let item = &mut queue[pos]; + item.endpoint = endpoint_out; if count_failure { item.retry += 1; if item.retry >= UPLOAD_RETRY_CAP { - notify_dropped(shared, DropReason::RetriesExhausted); - continue; + queue.remove(pos); + dropped = Some(DropReason::RetriesExhausted); + } else { + item.next_attempt_ms = now + backoff_delay(item.retry, backoff_base); } - item.next_attempt_ms = now + backoff_delay(item.retry, backoff_base); } - keep.push(item); } Err(TransportError::TooManySimilar { .. }) => { - notify_dropped(shared, DropReason::TooManySimilar); + queue.remove(pos); + dropped = Some(DropReason::TooManySimilar); + } + Err(e) => { + queue.remove(pos); + dropped = Some(drop_reason_for(e)); } - Err(e) => notify_dropped(shared, drop_reason_for(e)), } - } - if !keep.is_empty() { - // Reports held while we were delivering stay behind the retained ones. - let mut queue = shared.mem_queue.lock(); - keep.append(&mut queue); - *queue = keep; + drop(queue); + if let Some(reason) = dropped { + notify_dropped(shared, reason); + } } attempted } @@ -1537,27 +1679,29 @@ fn drain_until_empty( let start = Instant::now(); loop { let now = epoch_ms(); - drain_memory(shared, backoff_base, true, false); - for report in data_dir.map(queue::list_pending).unwrap_or_default() { - let Some(_claim) = queue::try_claim(&report) else { - continue; // a peer is delivering it; the loop below waits it out - }; - let Ok((zip, request_json, retry, _next)) = queue::load(&report) else { - continue; - }; - deliver_queued( - shared, - data_dir.unwrap_or(Path::new("")), - backoff_base, - &report, - zip, - request_json, - retry, - now, - false, - ); - if start.elapsed() >= deadline { - break; + drain_memory(shared, backoff_base, true, false, Some(start + deadline)); + if let Some(data_dir) = data_dir { + for report in queue::list_pending(data_dir) { + let Some(_claim) = queue::try_claim(&report) else { + continue; // a peer is delivering it; the loop below waits it out + }; + let Ok((zip, request_json, retry, _next)) = queue::load(&report) else { + continue; + }; + deliver_queued( + shared, + data_dir, + backoff_base, + &report, + zip, + request_json, + retry, + now, + false, + ); + if start.elapsed() >= deadline { + break; + } } } let disk_empty = data_dir.is_none_or(|d| queue::list_pending(d).is_empty()); diff --git a/crates/bugsee-core/src/util.rs b/crates/bugsee-core/src/util.rs index ed27ce0..0c0f3c8 100644 --- a/crates/bugsee-core/src/util.rs +++ b/crates/bugsee-core/src/util.rs @@ -10,14 +10,17 @@ use std::time::{SystemTime, UNIX_EPOCH}; -/// Print `message` to stderr the first time it is called in this process and do +/// Print `message` to stderr the first time it is called with `flag` and do /// nothing after. For conditions the SDK must announce (it otherwise does no -/// logging) but that would be noise if repeated, such as a degraded data dir. -pub fn warn_once(message: &str) { - use std::sync::atomic::{AtomicBool, Ordering}; - static WARNED: AtomicBool = AtomicBool::new(false); - if !WARNED.swap(true, Ordering::Relaxed) { - eprintln!("bugsee: {message}"); +/// logging) but that would be noise if repeated, such as a degraded data dir; +/// each kind of message owns a `static` flag so one does not mute another. +/// +/// Never panics: a closed or broken stderr is ignored (`eprintln!` would panic). +pub fn warn_once(flag: &std::sync::atomic::AtomicBool, message: &str) { + use std::io::Write; + use std::sync::atomic::Ordering; + if !flag.swap(true, Ordering::Relaxed) { + let _ = writeln!(std::io::stderr(), "bugsee: {message}"); } } diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index 94f6b1b..eb021de 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -106,7 +106,7 @@ fn memory_only_config_skips_the_disk_entirely() { let transport = Arc::new(MockTransport::default()); let data_dir = dir.path.join("must-stay-absent"); let mut config = RecorderConfig::new(&data_dir, "T"); - config.memory_only = true; + config.memory_only = Some("test".to_string()); let recorder = Recorder::launch(config, transport.clone()).expect("launch"); @@ -140,6 +140,9 @@ fn read_only_directory_runs_in_memory_and_still_reports() { #[test] fn directory_removed_mid_run_still_reports() { + // The SDK recreates a vanished directory (`create_dir_all`) and open part + // files keep accepting writes to their unlinked inodes, so this report takes + // the disk queue again (the earlier capture window is gone with the files). let dir = TempDir::new(); let data_dir = dir.path.join("data"); let transport = Arc::new(MockTransport::default()); @@ -171,7 +174,12 @@ fn directory_replaced_by_a_file_mid_run_falls_back_to_memory_delivery() { std::fs::remove_dir_all(&data_dir).unwrap(); std::fs::write(&data_dir, b"a file now").unwrap(); + assert!(!recorder.is_degraded()); assert_eq!(report_and_flush(&recorder, &transport), 1); + assert!( + recorder.is_degraded(), + "mid-run degradation must be visible" + ); drop(recorder); } @@ -191,49 +199,215 @@ fn queue_write_failure_holds_the_report_in_memory() { assert_eq!(recorder.storage_mode(), StorageMode::Disk); assert_eq!(report_and_flush(&recorder, &transport), 1); + assert!(recorder.is_degraded()); + drop(recorder); + // The failed enqueue must not leak its hard-linked snapshot. + let leftovers = std::fs::read_dir(data_dir.join("reports")) + .map(|rd| rd.flatten().count()) + .unwrap_or(0); + assert_eq!(leftovers, 0, "reports/ snapshot dirs must be reclaimed"); } #[test] -fn memory_backlog_is_bounded_and_overflow_is_reported() { - use bugsee_core::runtime::DropReason; - use bugsee_core::transport::{Transport, TransportError}; - use std::sync::Mutex; +fn legacy_directory_leftovers_are_delivered_once() { + use bugsee_core::model::report::TimeWindow; + use bugsee_core::reporting::assemble_docs; - // Never delivers, so everything stays in the memory backlog. - struct Offline; - impl Transport for Offline { - fn register_session(&self, _: &str, _: &[u8]) -> Result { - Err(TransportError::Transient("offline".into())) - } - fn create_issue( - &self, - _: &str, - _: Option<&str>, - _: &[u8], - ) -> Result { - Err(TransportError::Transient("offline".into())) - } - fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), TransportError> { - Err(TransportError::Transient("offline".into())) + let dir = TempDir::new(); + let legacy = dir.path.join("legacy"); + let data_dir = dir.path.join("data"); + let meta = manual_upload_meta(); + let env = bugsee_core::model::environment::Environment::detect_with( + "t", + &Default::default(), + &Default::default(), + ); + let window = TimeWindow { start: 0, end: 1 }; + let stranded = assemble_docs(Vec::new(), window, &meta, &env, None, &[], "T", 1).unwrap(); + bugsee_core::queue::enqueue(&legacy, &stranded).unwrap(); + + let transport = Arc::new(MockTransport::default()); + let mut config = RecorderConfig::new(&data_dir, "T"); + config.legacy_data_dir = Some(legacy.clone()); + let recorder = Recorder::launch(config, transport.clone()).expect("launch"); + assert!(recorder.flush(Duration::from_secs(10))); + // The startup drain is asynchronous; flush covers only the data dir queue. + let start = std::time::Instant::now(); + while transport.uploaded_bundles.lock().unwrap().is_empty() + && start.elapsed() < Duration::from_secs(10) + { + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 1); + assert!( + bugsee_core::queue::list_pending(&legacy).is_empty(), + "a delivered legacy report must be removed" + ); +} + +/// Blocks every delivery until released, so the backlog cannot drain. +struct Gate { + open: std::sync::Mutex, + cv: std::sync::Condvar, +} +impl Gate { + fn release(&self) { + *self.open.lock().unwrap() = true; + self.cv.notify_all(); + } +} +struct Gated(Arc); +impl bugsee_core::Transport for Gated { + fn register_session(&self, _: &str, _: &[u8]) -> Result { + let mut open = self.0.open.lock().unwrap(); + while !*open { + open = self.0.cv.wait(open).unwrap(); } + Err(bugsee_core::TransportError::Transient("offline".into())) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + Err(bugsee_core::TransportError::Transient("offline".into())) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), bugsee_core::TransportError> { + Err(bugsee_core::TransportError::Transient("offline".into())) } +} + +#[test] +fn memory_backlog_is_bounded_even_while_a_delivery_is_in_flight() { + use bugsee_core::runtime::DropReason; + use std::sync::Mutex; let dir = TempDir::new(); + let gate = Arc::new(Gate { + open: Mutex::new(false), + cv: std::sync::Condvar::new(), + }); let dropped = Arc::new(Mutex::new(Vec::new())); let sink = dropped.clone(); let mut config = RecorderConfig::new(blocked_path(&dir.path), "T"); config.on_report_dropped = Some(Box::new(move |r| sink.lock().unwrap().push(r))); - let recorder = Recorder::launch(config, Arc::new(Offline)).expect("launch"); + let recorder = Recorder::launch(config, Arc::new(Gated(gate.clone()))).expect("launch"); for _ in 0..40 { recorder.report(manual_upload_meta(), None); } - // Flush cannot succeed (offline) but must return promptly, not hang. + // The worker has handled every report once a Flush message round-trips + // (the flush itself cannot finish: the uploader is held in the gate). assert!(!recorder.flush(Duration::from_millis(500))); let seen = dropped.lock().unwrap().clone(); + gate.release(); + assert_eq!( + seen.len(), + 40 - 16, + "exactly the reports over the cap are dropped, got {seen:?}" + ); + assert!(seen.iter().all(|r| *r == DropReason::StorageUnavailable)); +} + +#[test] +fn memory_flush_respects_its_deadline_with_a_full_backlog() { + use std::time::Instant; + + // Every attempt costs 100 ms and fails transiently. + struct Slow; + impl bugsee_core::Transport for Slow { + fn register_session( + &self, + _: &str, + _: &[u8], + ) -> Result { + std::thread::sleep(Duration::from_millis(100)); + Err(bugsee_core::TransportError::Transient("slow".into())) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + Err(bugsee_core::TransportError::Transient("slow".into())) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), bugsee_core::TransportError> { + Err(bugsee_core::TransportError::Transient("slow".into())) + } + } + + let dir = TempDir::new(); + let recorder = Recorder::launch( + RecorderConfig::new(blocked_path(&dir.path), "T"), + Arc::new(Slow), + ) + .expect("launch"); + for _ in 0..16 { + recorder.report(manual_upload_meta(), None); + } + let start = Instant::now(); + assert!(!recorder.flush(Duration::from_millis(400))); + // 16 attempts at 100 ms would be 1.6 s per pass if the deadline were ignored. assert!( - !seen.is_empty() && seen.iter().all(|r| *r == DropReason::StorageUnavailable), - "overflow must be surfaced, got {seen:?}" + start.elapsed() < Duration::from_millis(1200), + "flush overran its deadline: {:?}", + start.elapsed() + ); +} + +#[test] +fn a_failed_put_retries_at_the_put_and_does_not_mint_a_duplicate_issue() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + // The issue is created, then the first PUT fails transiently. + #[derive(Default)] + struct FlakyPut { + issues: AtomicUsize, + puts: AtomicUsize, + } + struct Handle(Arc); + impl bugsee_core::Transport for Handle { + fn register_session( + &self, + _: &str, + _: &[u8], + ) -> Result { + Ok("token".into()) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + self.0.issues.fetch_add(1, Ordering::SeqCst); + Ok("https://uploads.example/x".into()) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), bugsee_core::TransportError> { + if self.0.puts.fetch_add(1, Ordering::SeqCst) == 0 { + Err(bugsee_core::TransportError::Transient("reset".into())) + } else { + Ok(()) + } + } + } + + let dir = TempDir::new(); + let flaky = Arc::new(FlakyPut::default()); + let recorder = Recorder::launch( + RecorderConfig::new(blocked_path(&dir.path), "T"), + Arc::new(Handle(flaky.clone())), + ) + .expect("launch"); + recorder.report(manual_upload_meta(), None); + assert!(recorder.flush(Duration::from_secs(10))); + assert_eq!(flaky.puts.load(Ordering::SeqCst), 2); + assert_eq!( + flaky.issues.load(Ordering::SeqCst), + 1, + "the retry must resume at the PUT, not create the issue again" ); } diff --git a/crates/bugsee-ffi/tests/lifecycle.rs b/crates/bugsee-ffi/tests/lifecycle.rs index 702199f..1eaf113 100644 --- a/crates/bugsee-ffi/tests/lifecycle.rs +++ b/crates/bugsee-ffi/tests/lifecycle.rs @@ -21,6 +21,10 @@ fn full_c_abi_lifecycle() { let token = c(&format!("ffiLifecycle{}", std::process::id())); let endpoint = c("http://127.0.0.1:1"); // connection refused; offline-safe + // Keep the SDK's state out of the developer's real data directory. + let data = std::env::temp_dir().join(format!("bugsee-ffi-{}", std::process::id())); + std::env::set_var(bugsee::DATA_DIR_ENV, &data); + assert_eq!( unsafe { bugsee_launch_with_endpoint(token.as_ptr(), endpoint.as_ptr()) }, BugseeStatus::Ok @@ -85,8 +89,5 @@ fn full_c_abi_lifecycle() { assert_eq!(bugsee_stop(), BugseeStatus::Ok); assert_eq!(bugsee_is_active(), 0, "stopped is not active"); - // Cleanup the default per-token data dir. - let mut data = std::env::temp_dir(); - data.push("bugsee"); let _ = std::fs::remove_dir_all(data); } diff --git a/crates/bugsee/Cargo.toml b/crates/bugsee/Cargo.toml index bc971a1..3051090 100644 --- a/crates/bugsee/Cargo.toml +++ b/crates/bugsee/Cargo.toml @@ -17,6 +17,9 @@ backtrace = "0.3" ureq = { version = "2", optional = true, features = ["json"] } sysinfo = { version = "0.33", optional = true, default-features = false, features = ["system", "disk"] } +[target.'cfg(unix)'.dependencies] +libc = "0.2" + [dev-dependencies] zip = { version = "2", default-features = false, features = ["zstd"] } diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 02b552d..54caa8e 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -50,6 +50,7 @@ struct Resolved { dir: Option, source: DataDirSource, skipped: Vec, + explicit_rejected: Option, } static RESOLUTION: ForkMutex> = ForkMutex::new(|| None); @@ -86,15 +87,26 @@ impl Bugsee { dir, source, skipped, + explicit_rejected, + legacy_dir, } = options.resolve_data_dir(); - let memory_only = dir.is_none(); + let memory_only = dir.is_none().then(|| { + explicit_rejected + .clone() + .or_else(|| (!skipped.is_empty()).then(|| skipped.join("; "))) + .unwrap_or_else(|| "no usable data directory".to_string()) + }); let data_dir = dir.unwrap_or_else(|| std::env::temp_dir().join("bugsee")); - if !memory_only && !skipped.is_empty() { - bugsee_core::util::warn_once(&format!( - "data directory unusable ({}); using {}", - skipped.join("; "), - data_dir.display() - )); + if memory_only.is_none() && !skipped.is_empty() { + static WARNED: AtomicBool = AtomicBool::new(false); + bugsee_core::util::warn_once( + &WARNED, + &format!( + "default data directory unusable ({}); using {}", + skipped.join("; "), + data_dir.display() + ), + ); } let transport = match options.transport.clone() { @@ -118,6 +130,7 @@ impl Bugsee { let mut config = RecorderConfig::new(data_dir, options.app_token.clone()); config.memory_only = memory_only; + config.legacy_data_dir = legacy_dir; config.caps = options.caps(); config.rotate_interval = options.rotate_interval; config.before_send = options.before_send; @@ -148,7 +161,7 @@ impl Bugsee { // Nothing can be recovered without a disk, so crash markers and panic // snapshots have nowhere to go. let persistent = recorder.storage_mode() == StorageMode::Disk; - *RESOLUTION.lock() = Some(Resolved { + let resolved = Resolved { dir: persistent.then(|| recorder.data_dir().to_path_buf()), source: if persistent { source @@ -156,7 +169,8 @@ impl Bugsee { DataDirSource::None }, skipped, - }); + explicit_rejected, + }; #[cfg(feature = "panic")] let panic_info_path = recorder.panic_info_path(); @@ -183,7 +197,13 @@ impl Bugsee { // `Recorder::drop` joins the worker/uploader threads (blocking up to a few // seconds), and doing that under the global lock would stall every // concurrent capture call β€” the same hazard `stop()` guards against. - let old = RECORDER.lock().replace(recorder); + // The resolution is published under the same lock as the recorder, so + // `storage_status` never pairs one launch's recorder with another's. + let old = { + let mut slot = RECORDER.lock(); + *RESOLUTION.lock() = Some(resolved); + slot.replace(recorder) + }; drop(old); PAUSED.store(false, Ordering::SeqCst); *REVIVE_INFO.lock() = Some(ReviveInfo { @@ -217,22 +237,20 @@ impl Bugsee { /// ([`StorageStatus::may_be_ephemeral`]). pub fn storage_status() -> Option { check_fork(); - let (mode, reason) = { - let guard = RECORDER.lock(); - let recorder = guard.as_ref()?; - ( - recorder.storage_mode(), - recorder.storage_reason().map(String::from), - ) - }; + // Both locks, always in this order (as in `launch_with`), so the pair is + // one launch's. + let recorder = RECORDER.lock(); + let recorder = recorder.as_ref()?; let resolved = RESOLUTION.lock(); let resolved = resolved.as_ref()?; Some(StorageStatus { - mode, + mode: recorder.storage_mode(), data_dir: resolved.dir.clone(), source: resolved.source, skipped: resolved.skipped.clone(), - reason, + explicit_rejected: resolved.explicit_rejected.clone(), + reason: recorder.storage_reason().map(String::from), + degraded: recorder.is_degraded(), }) } @@ -558,8 +576,11 @@ fn revive_after_fork(epoch: u64) { #[cfg(feature = "native")] if old_native.is_some() { drop(old_native); - if let Ok(handler) = bugsee_native::install(recorder.crash_info_path()) { - *NATIVE.lock() = Some(handler); + // A child that fell back to memory has nowhere to keep a crash marker. + if recorder.storage_mode() == StorageMode::Disk { + if let Ok(handler) = bugsee_native::install(recorder.crash_info_path()) { + *NATIVE.lock() = Some(handler); + } } } #[cfg(feature = "panic")] diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index acd48e3..757d084 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -231,8 +231,8 @@ impl LaunchOptions { } } - /// Pick the data directory: the first candidate in [`DataDirSource`] order - /// that can actually be created and written to. + /// Pick the data directory; see [`DataDirSource`] for the order and what an + /// unusable choice means. pub(crate) fn resolve_data_dir(&self) -> DataDirResolution { resolve_data_dir( self.data_dir.as_deref(), @@ -249,25 +249,30 @@ pub const DATA_DIR_ENV: &str = "BUGSEE_DATA_DIR"; /// Which step of the resolution order produced the data directory. /// -/// Candidates are tried in this order and the first one that is writable wins; -/// an unusable one is skipped (and recorded in -/// [`StorageStatus::skipped`](crate::StorageStatus::skipped)), never fatal: -/// /// 1. [`Explicit`](DataDirSource::Explicit): [`LaunchOptions::data_dir`]. /// 2. [`Environment`](DataDirSource::Environment): `BUGSEE_DATA_DIR`. /// 3. [`PlatformDefault`](DataDirSource::PlatformDefault): `bugsee/` under /// `$XDG_DATA_HOME` (else `~/.local/share`) on Linux, `~/Library/Application /// Support` on macOS, `%LOCALAPPDATA%` on Windows. Skipped when the base is /// unset, as in a container with no home directory. -/// 4. [`TempFallback`](DataDirSource::TempFallback): `bugsee/` under the -/// system temp directory. Often wiped on restart, so crash recovery may not -/// survive it; [`StorageStatus::may_be_ephemeral`](crate::StorageStatus::may_be_ephemeral) +/// 4. [`TempFallback`](DataDirSource::TempFallback): `bugsee-/` under +/// the system temp directory (`bugsee/` on Windows), created private +/// (`0700`) and refused if it is a symlink or owned by another user. Often +/// wiped on restart, so crash recovery may not survive it; +/// [`StorageStatus::may_be_ephemeral`](crate::StorageStatus::may_be_ephemeral) /// reports it. /// -/// If none is writable the SDK runs +/// **A directory you asked for is final.** If 1 or 2 is set and not writable the +/// SDK does NOT quietly use another location on disk (that would scatter +/// reports somewhere the operator never chose); it runs in memory and reports +/// why in [`StorageStatus::explicit_rejected`](crate::StorageStatus::explicit_rejected). +/// Only the defaults (3, then 4) fall through to each other. +/// +/// With no usable directory the SDK runs /// [in memory](bugsee_core::runtime::StorageMode::Memory): handled errors and /// manual reports are still sent, but there is no capture window and no crash /// recovery. It logs one line to stderr when it degrades. +#[non_exhaustive] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DataDirSource { Explicit, @@ -283,8 +288,14 @@ pub(crate) struct DataDirResolution { /// The chosen directory, or `None` if no candidate was writable. pub dir: Option, pub source: DataDirSource, - /// Candidates tried and rejected before the winner, with the reason. + /// Default candidates tried and rejected before the winner, with the reason. pub skipped: Vec, + /// Why an explicit / environment directory was rejected (it is final). + pub explicit_rejected: Option, + /// The location earlier SDK versions used by default + /// (`/bugsee/`), if it exists, is trustworthy and is not `dir`. + /// Its leftovers are recovered once at launch. + pub legacy_dir: Option, } /// Pure resolution core: `env` and `temp` are injected so tests need not touch @@ -295,37 +306,80 @@ pub(crate) fn resolve_data_dir( env: &dyn Fn(&str) -> Option, temp: &Path, ) -> DataDirResolution { - let mut candidates: Vec<(DataDirSource, PathBuf)> = Vec::new(); - if let Some(dir) = explicit { - candidates.push((DataDirSource::Explicit, dir.to_path_buf())); - } - if let Some(dir) = env(DATA_DIR_ENV).filter(|v| !v.is_empty()) { - candidates.push((DataDirSource::Environment, PathBuf::from(dir))); - } - // Isolate by app token so multiple apps don't collide. - let app = |base: PathBuf| base.join("bugsee").join(short_token(app_token)); + let token = short_token(app_token); + + let chosen = explicit + .map(|d| (DataDirSource::Explicit, d.to_path_buf())) + .or_else(|| { + env(DATA_DIR_ENV) + .filter(|v| !v.is_empty()) + .map(|v| (DataDirSource::Environment, PathBuf::from(v))) + }); + if let Some((source, dir)) = chosen { + return match prepare_dir(&dir, false) { + Ok(()) => DataDirResolution { + dir: Some(dir), + source, + skipped: Vec::new(), + explicit_rejected: None, + legacy_dir: None, + }, + Err(e) => DataDirResolution { + dir: None, + source: DataDirSource::None, + skipped: Vec::new(), + explicit_rejected: Some(format!("{}: {e}", dir.display())), + legacy_dir: None, + }, + }; + } + + let mut candidates: Vec<(DataDirSource, PathBuf, bool)> = Vec::new(); if let Some(base) = platform_data_base(env) { - candidates.push((DataDirSource::PlatformDefault, app(base))); + candidates.push(( + DataDirSource::PlatformDefault, + base.join("bugsee").join(&token), + false, + )); } - candidates.push((DataDirSource::TempFallback, app(temp.to_path_buf()))); + candidates.push(( + DataDirSource::TempFallback, + temp.join(private_temp_name()).join(&token), + true, + )); let mut skipped = Vec::new(); - for (source, dir) in candidates { - match probe_writable(&dir) { + let mut found = None; + for (source, dir, private) in candidates { + match prepare_dir(&dir, private) { Ok(()) => { - return DataDirResolution { - dir: Some(dir), - source, - skipped, - } + found = Some((source, dir)); + break; } Err(e) => skipped.push(format!("{}: {e}", dir.display())), } } - DataDirResolution { - dir: None, - source: DataDirSource::None, - skipped, + + let legacy = temp.join("bugsee").join(&token); + let legacy_dir = found + .as_ref() + .filter(|(_, dir)| *dir != legacy) + .and_then(|_| trusted_existing_dir(&legacy).then_some(legacy)); + match found { + Some((source, dir)) => DataDirResolution { + dir: Some(dir), + source, + skipped, + explicit_rejected: None, + legacy_dir, + }, + None => DataDirResolution { + dir: None, + source: DataDirSource::None, + skipped, + explicit_rejected: None, + legacy_dir: None, + }, } } @@ -341,14 +395,113 @@ fn platform_data_base(env: &dyn Fn(&str) -> Option) -> Option } } +/// The per-user directory name under the shared temp dir. The system temp +/// directory is world-writable on unix, so a fixed name would let another user +/// pre-create (or symlink) it and read or plant our reports. +#[cfg(unix)] +fn private_temp_name() -> String { + // SAFETY: `geteuid` has no preconditions and cannot fail. + format!("bugsee-{}", unsafe { libc::geteuid() }) +} +#[cfg(not(unix))] +fn private_temp_name() -> String { + "bugsee".to_string() +} + /// Create `dir` and prove a file can be written in it. Checking writability (not /// just existence) is what catches a read-only root file system or a volume -/// mounted read-only. +/// mounted read-only. With `private` (the shared temp area) each created +/// component is `0700` and an existing one is refused unless it is a real +/// directory owned by us. +fn prepare_dir(dir: &Path, private: bool) -> std::io::Result<()> { + if private { + make_private(dir)?; + } else { + std::fs::create_dir_all(dir)?; + } + probe_writable(dir) +} + +#[cfg(unix)] +fn make_private(dir: &Path) -> std::io::Result<()> { + use std::os::unix::fs::{DirBuilderExt, MetadataExt, PermissionsExt}; + // The shared temp dir itself is the system's; only the two components below + // it are ours to vet. + let leaf = dir; + let top = leaf.parent().unwrap_or(leaf); + if let Some(shared) = top.parent() { + std::fs::create_dir_all(shared)?; + } + // SAFETY: `geteuid` has no preconditions and cannot fail. + let uid = unsafe { libc::geteuid() }; + for component in [top, leaf] { + match std::fs::DirBuilder::new().mode(0o700).create(component) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(e), + } + // `symlink_metadata`: a symlink planted at our name must not be followed. + let meta = std::fs::symlink_metadata(component)?; + if !meta.file_type().is_dir() { + return Err(std::io::Error::other( + "not a real directory (symlink or file at our private name)", + )); + } + if meta.uid() != uid { + return Err(std::io::Error::other("owned by another user")); + } + if meta.mode() & 0o077 != 0 { + std::fs::set_permissions(component, std::fs::Permissions::from_mode(0o700))?; + } + } + Ok(()) +} +#[cfg(not(unix))] +fn make_private(dir: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(dir) +} + +/// Whether `dir` exists and may be trusted to hold OUR leftovers: a real +/// directory (not a symlink) owned by the current user. The legacy location is +/// in the shared temp area, where anyone could have planted files. +fn trusted_existing_dir(dir: &Path) -> bool { + let Ok(meta) = std::fs::symlink_metadata(dir) else { + return false; + }; + if !meta.file_type().is_dir() { + return false; + } + owned_by_us(&meta) +} + +#[cfg(unix)] +fn owned_by_us(meta: &std::fs::Metadata) -> bool { + use std::os::unix::fs::MetadataExt; + // SAFETY: `geteuid` has no preconditions and cannot fail. + meta.uid() == unsafe { libc::geteuid() } +} +#[cfg(not(unix))] +fn owned_by_us(_meta: &std::fs::Metadata) -> bool { + true +} + +/// Prove a file can be created in `dir`. `create_new` with an unpredictable name +/// refuses to follow or reuse anything pre-planted at that path. fn probe_writable(dir: &Path) -> std::io::Result<()> { use std::io::Write; - std::fs::create_dir_all(dir)?; - let probe = dir.join(format!(".bugsee-probe-{}", std::process::id())); - let result = std::fs::File::create(&probe).and_then(|mut f| f.write_all(b"1")); + let probe = dir.join(format!( + ".bugsee-probe-{}-{}", + std::process::id(), + bugsee_core::util::random_hex(8) + )); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let result = options.open(&probe).and_then(|mut f| f.write_all(b"1")); let _ = std::fs::remove_file(&probe); result } @@ -360,6 +513,7 @@ fn probe_writable(dir: &Path) -> std::io::Result<()> { /// the process (a volume mount). The SDK cannot tell a volume from a scratch /// disk, so check [`may_be_ephemeral`](StorageStatus::may_be_ephemeral) and /// your own deployment. +#[non_exhaustive] #[derive(Debug, Clone)] pub struct StorageStatus { /// `Disk` when the data directory is usable; `Memory` when it is not. @@ -368,18 +522,26 @@ pub struct StorageStatus { pub data_dir: Option, /// Which resolution step produced `data_dir`. pub source: DataDirSource, - /// Candidates rejected before the one in use, with the error for each. + /// Default candidates rejected before the one in use, with the error for each. pub skipped: Vec, + /// Set when the directory the host asked for (`data_dir` or + /// `BUGSEE_DATA_DIR`) was unusable. It is final: the SDK is in memory mode, + /// not on some other disk location. + pub explicit_rejected: Option, /// Why the SDK is in memory mode, if it is. pub reason: Option, + /// A disk write has failed since launch (full, read-only, removed). Reports + /// still go out, but crash recovery can no longer be relied on. + pub degraded: bool, } impl StorageStatus { /// Whether a fatal crash can be recovered on the next launch: true only with - /// a usable data directory. (It also needs that directory to survive the - /// process; see [`may_be_ephemeral`](Self::may_be_ephemeral).) + /// a usable data directory that has not failed since launch. (It also needs + /// that directory to survive the process; see + /// [`may_be_ephemeral`](Self::may_be_ephemeral).) pub fn crash_recovery_available(&self) -> bool { - self.mode == bugsee_core::runtime::StorageMode::Disk + self.mode == bugsee_core::runtime::StorageMode::Disk && !self.degraded } /// Whether the directory is the temp-dir fallback, which is commonly wiped @@ -426,6 +588,14 @@ mod tests { } } + fn home_env(home: &Path) -> Vec<(&'static str, OsString)> { + vec![ + ("HOME", home.as_os_str().to_owned()), + ("XDG_DATA_HOME", home.as_os_str().to_owned()), + ("LOCALAPPDATA", home.as_os_str().to_owned()), + ] + } + #[test] fn explicit_directory_wins_when_writable() { let s = scratch(); @@ -434,20 +604,53 @@ mod tests { let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&env), &s.join("tmp")); assert_eq!(r.source, DataDirSource::Explicit); assert_eq!(r.dir.as_deref(), Some(explicit.as_path())); - assert!(r.skipped.is_empty()); + assert!(r.skipped.is_empty() && r.explicit_rejected.is_none()); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn unwritable_explicit_is_final_and_never_falls_to_another_location() { + let s = scratch(); + let good_env = s.join("from-env"); + let env = [(DATA_DIR_ENV, good_env.clone().into_os_string())]; + let mut full_env = env.to_vec(); + full_env.extend(home_env(&s.join("home"))); + let r = resolve_data_dir( + Some(&blocked(&s, "f1")), + "tok", + &env_of(&full_env), + &s.join("tmp"), + ); + assert!(r.dir.is_none()); + assert_eq!(r.source, DataDirSource::None); + assert!(r.explicit_rejected.is_some()); + assert!( + !good_env.exists(), + "the environment variable must not be tried" + ); + assert!(!s.join("home").exists() && !s.join("tmp").exists()); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn unwritable_environment_directory_is_final_too() { + let s = scratch(); + let mut env = vec![(DATA_DIR_ENV, blocked(&s, "f2").into_os_string())]; + env.extend(home_env(&s.join("home"))); + let r = resolve_data_dir(None, "tok", &env_of(&env), &s.join("tmp")); + assert!(r.dir.is_none() && r.explicit_rejected.is_some()); + assert!(!s.join("home").exists() && !s.join("tmp").exists()); let _ = std::fs::remove_dir_all(&s); } #[test] - fn unwritable_explicit_falls_through_to_the_environment_variable() { + fn environment_variable_is_used_without_an_explicit_directory() { let s = scratch(); - let bad = blocked(&s, "f1"); let good = s.join("from-env"); let env = [(DATA_DIR_ENV, good.clone().into_os_string())]; - let r = resolve_data_dir(Some(&bad), "tok", &env_of(&env), &s.join("tmp")); + let r = resolve_data_dir(None, "tok", &env_of(&env), &s.join("tmp")); assert_eq!(r.source, DataDirSource::Environment); assert_eq!(r.dir, Some(good)); - assert_eq!(r.skipped.len(), 1, "the rejected candidate is recorded"); let _ = std::fs::remove_dir_all(&s); } @@ -455,40 +658,63 @@ mod tests { fn platform_default_is_used_when_home_is_set() { let s = scratch(); let home = s.join("home"); - let env = [ - ("HOME", home.clone().into_os_string()), - ("XDG_DATA_HOME", home.clone().into_os_string()), - ("LOCALAPPDATA", home.clone().into_os_string()), - ]; - let r = resolve_data_dir(None, "tok-1", &env_of(&env), &s.join("tmp")); + let r = resolve_data_dir(None, "tok-1", &env_of(&home_env(&home)), &s.join("tmp")); assert_eq!(r.source, DataDirSource::PlatformDefault); assert!(r.dir.unwrap().starts_with(&home)); let _ = std::fs::remove_dir_all(&s); } #[test] - fn no_home_falls_back_to_the_temp_directory() { + fn no_home_falls_back_to_a_private_temp_directory() { let s = scratch(); let tmp = s.join("tmp"); let r = resolve_data_dir(None, "tok-1", &env_of(&[]), &tmp); assert_eq!(r.source, DataDirSource::TempFallback); - assert_eq!(r.dir, Some(tmp.join("bugsee").join("tok1"))); + let dir = r.dir.unwrap(); + assert_eq!(dir, tmp.join(private_temp_name()).join("tok1")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for d in [dir.as_path(), dir.parent().unwrap()] { + let mode = std::fs::metadata(d).unwrap().permissions().mode(); + assert_eq!(mode & 0o077, 0, "{} must be private", d.display()); + } + } let _ = std::fs::remove_dir_all(&s); } + #[cfg(unix)] #[test] - fn nothing_writable_resolves_to_none_and_lists_every_attempt() { + fn temp_fallback_refuses_a_symlink_planted_at_the_private_name() { let s = scratch(); - let env = [(DATA_DIR_ENV, blocked(&s, "f2").into_os_string())]; + let tmp = s.join("tmp"); + let elsewhere = s.join("elsewhere"); + std::fs::create_dir_all(&tmp).unwrap(); + std::fs::create_dir_all(&elsewhere).unwrap(); + std::os::unix::fs::symlink(&elsewhere, tmp.join(private_temp_name())).unwrap(); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &tmp); + assert!(r.dir.is_none(), "must not follow the symlink"); + assert_eq!(r.skipped.len(), 1); + assert!( + std::fs::read_dir(&elsewhere).unwrap().next().is_none(), + "nothing may be created through the symlink" + ); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn nothing_writable_resolves_to_none_and_lists_every_default_attempt() { + let s = scratch(); + let blocked_home = blocked(&s, "f3"); let r = resolve_data_dir( - Some(&blocked(&s, "f3")), + None, "tok", - &env_of(&env), + &env_of(&home_env(&blocked_home)), &blocked(&s, "f4"), ); assert_eq!(r.source, DataDirSource::None); assert!(r.dir.is_none()); - assert_eq!(r.skipped.len(), 3); + assert_eq!(r.skipped.len(), 2); let _ = std::fs::remove_dir_all(&s); } @@ -500,4 +726,25 @@ mod tests { assert_eq!(r.source, DataDirSource::TempFallback); let _ = std::fs::remove_dir_all(&s); } + + // Unix only: elsewhere the temp fallback IS the old location. + #[cfg(unix)] + #[test] + fn the_previous_default_location_is_offered_for_recovery_only_if_it_exists() { + let s = scratch(); + let tmp = s.join("tmp"); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &tmp); + assert!(r.legacy_dir.is_none(), "nothing to adopt yet"); + + let legacy = tmp.join("bugsee").join("tok"); + std::fs::create_dir_all(&legacy).unwrap(); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &tmp); + assert_eq!(r.legacy_dir, Some(legacy.clone())); + + // A host that chose its directory never used the legacy one. + let explicit = s.join("explicit"); + let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&[]), &tmp); + assert!(r.legacy_dir.is_none()); + let _ = std::fs::remove_dir_all(&s); + } } diff --git a/crates/bugsee/tests/fork_safety.rs b/crates/bugsee/tests/fork_safety.rs index 14c77eb..64365f0 100644 --- a/crates/bugsee/tests/fork_safety.rs +++ b/crates/bugsee/tests/fork_safety.rs @@ -518,3 +518,81 @@ fn a_panic_in_the_child_before_touching_the_sdk_does_not_write_into_the_parents_ let _ = std::fs::remove_dir_all(&data); let _ = std::fs::remove_dir_all(&out); } + +/// Fails transiently until `allow` exists, then behaves like [`FileTransport`]. +struct AllowTransport { + allow: PathBuf, + inner: Arc, +} + +impl Transport for AllowTransport { + fn register_session(&self, a: &str, b: &[u8]) -> Result { + self.inner.register_session(a, b) + } + fn create_issue(&self, a: &str, b: Option<&str>, c: &[u8]) -> Result { + if self.allow.exists() { + self.inner.create_issue(a, b, c) + } else { + Err(TransportError::Transient("offline".into())) + } + } + fn upload_bundle(&self, e: &str, zip: &[u8]) -> Result<(), TransportError> { + self.inner.upload_bundle(e, zip) + } +} + +#[test] +fn a_child_in_memory_mode_does_not_resend_the_parents_unsent_reports() { + let _serial = serial(); + let root = scratch("mem"); + let out = root.join("out"); + let allow = root.join("allow"); + // A directory that can never be created: the SDK runs in memory. + let blocker = root.join("blocker"); + std::fs::write(&blocker, b"x").unwrap(); + let guard = Bugsee::launch_with( + LaunchOptions::new("APP_TOKEN") + .data_dir(blocker.join("data")) + .native_crash_capture(false) + .with_transport(Arc::new(AllowTransport { + allow: allow.clone(), + inner: FileTransport::new(&out), + })), + ) + .expect("launch"); + + // Offline: the parent's report stays in its in-memory backlog. + Bugsee::capture_message(LogLevel::Error, "parent-held-report"); + // (A sleep, not a timed-out `flush`: a main thread that has just waited on a + // timed channel recv carries its parker state across `fork` and on macOS + // libdispatch traps the child.) + std::thread::sleep(Duration::from_millis(500)); + + let child = fork_child(|| { + Bugsee::capture_message(LogLevel::Error, "child-report"); + std::fs::write(&allow, b"1").unwrap(); + let ok = Bugsee::flush(Duration::from_secs(10)); + i32::from(!ok) + }); + assert!(exited_with( + wait_with_deadline(child, Duration::from_secs(30)), + 0 + )); + assert!(Bugsee::flush(Duration::from_secs(10))); + + let reports = delivered(&out); + assert_eq!( + mentions(&reports, "child-report"), + 1, + "{}", + summarize(&reports) + ); + assert_eq!( + mentions(&reports, "parent-held-report"), + 1, + "the child must not resend the parent's backlog: {}", + summarize(&reports) + ); + drop(guard); + let _ = std::fs::remove_dir_all(&root); +} diff --git a/crates/bugsee/tests/storage_fallback.rs b/crates/bugsee/tests/storage_fallback.rs index ba82295..32128d8 100644 --- a/crates/bugsee/tests/storage_fallback.rs +++ b/crates/bugsee/tests/storage_fallback.rs @@ -61,7 +61,9 @@ fn launch_degrades_through_the_resolution_order() { assert!(!status.may_be_ephemeral()); drop(guard); - // 2. An unusable explicit directory falls through to the environment variable. + // 2. A directory the host asked for is FINAL: unusable means memory mode, not + // some other location on disk (here the environment variable is writable + // and must stay untouched). let from_env = s.join("from-env"); std::env::set_var(DATA_DIR_ENV, &from_env); let guard = Bugsee::launch_with( @@ -71,20 +73,27 @@ fn launch_degrades_through_the_resolution_order() { ) .expect("launch"); let status = Bugsee::storage_status().expect("status"); + assert_eq!(status.mode, StorageMode::Memory); + assert!(status.explicit_rejected.is_some()); + assert!(!from_env.exists(), "must not fall back to another location"); + drop(guard); + + // 3. Without an explicit directory the environment variable is used. + let guard = + Bugsee::launch_with(LaunchOptions::new("TOKEN_A").with_transport(transport.clone())) + .expect("launch"); + let status = Bugsee::storage_status().expect("status"); assert_eq!(status.source, DataDirSource::Environment); assert_eq!(status.data_dir.as_deref(), Some(from_env.as_path())); - assert_eq!(status.skipped.len(), 1); drop(guard); std::env::remove_var(DATA_DIR_ENV); - // 3. Nothing usable anywhere: memory mode, and reports still go out. + // 4. Nothing usable anywhere (no home, temp dir unusable): memory mode, and + // reports still go out. let transport = Arc::new(MockTransport::default()); - let guard = Bugsee::launch_with( - LaunchOptions::new("TOKEN_A") - .data_dir(&blocked) - .with_transport(transport.clone()), - ) - .expect("an unwritable environment must not fail launch"); + let guard = + Bugsee::launch_with(LaunchOptions::new("TOKEN_A").with_transport(transport.clone())) + .expect("an unwritable environment must not fail launch"); let status = Bugsee::storage_status().expect("status"); assert_eq!(status.mode, StorageMode::Memory); assert_eq!(status.source, DataDirSource::None); From 01e337cf08ab960f57e0fcb331d17f39dbad3abf Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 14:34:47 +0500 Subject: [PATCH 03/52] fix: second review round for data-directory fallback (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fork child that falls back to memory clears the panic snapshot path and updates the published storage resolution (dir None, source None) - legacy location is adopted only if /bugsee and the token dir are real directories owned by us and not our own directory (canonicalised compare) - legacy queue is included in flush, gc'd for orphans, and retired from polling once recovery has run and its queue is empty - do not resolve temp_dir() unless a default location is actually needed - tests: fork memory-fallback, symlinked/aliased legacy dir, end-to-end legacy crashed session + queued report delivered once and cleaned up πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/runtime.rs | 38 +++++- crates/bugsee-core/tests/storage_failures.rs | 43 +++++-- crates/bugsee/src/api.rs | 20 ++- crates/bugsee/src/options.rs | 121 +++++++++++++++---- crates/bugsee/tests/fork_safety.rs | 62 ++++++++++ 5 files changed, 240 insertions(+), 44 deletions(-) diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 21b16a5..232e8f8 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -834,6 +834,10 @@ fn spawn_pipeline( let uploader_shared = Arc::clone(shared); let uploader_data_dir = data_dir.map(Path::to_path_buf); let uploader_legacy = legacy_data_dir.clone(); + // Set once the worker has finished recovering the legacy directory, so the + // uploader does not retire it while recovery is still about to fill its queue. + let legacy_ready = Arc::new(AtomicBool::new(legacy_data_dir.is_none())); + let uploader_ready = Arc::clone(&legacy_ready); let uploader = std::thread::Builder::new() .name("bugsee-uploader".into()) .spawn(move || { @@ -842,6 +846,7 @@ fn spawn_pipeline( uploader_shared, uploader_data_dir, uploader_legacy, + uploader_ready, backoff_base, fresh_launch, ) @@ -878,9 +883,11 @@ fn spawn_pipeline( // Best effort, once: sessions a previous default location left // behind. No generation of ours lives there, so none is skipped. if let Some(dir) = &legacy_data_dir { + queue::gc_orphans(dir); run_recovery(&worker_shared, dir, u64::MAX); } })); + legacy_ready.store(true, Ordering::SeqCst); let _ = worker_upload_tx.send(UploadMsg::Wake); worker_loop( rx, @@ -1348,11 +1355,12 @@ fn uploader_loop( shared: Arc, data_dir: Option, legacy_data_dir: Option, + legacy_ready: Arc, backoff_base: Duration, fresh_launch: bool, ) { let poll = Duration::from_secs(5); - let legacy = legacy_data_dir; + let mut legacy = legacy_data_dir; // A fresh launch retries any queued report immediately (force), ignoring // backoff scheduled by the prior session. A fork child skips this: the // parent's uploader is already working the shared queue. Guarded so a @@ -1383,7 +1391,12 @@ fn uploader_loop( } Ok(UploadMsg::Drain { deadline, ack }) => { let empty = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_until_empty(&shared, data_dir.as_deref(), backoff_base, deadline) + let dirs: Vec<&Path> = data_dir + .as_deref() + .into_iter() + .chain(legacy.as_deref()) + .collect(); + drain_until_empty(&shared, &dirs, backoff_base, deadline) })) .unwrap_or(false); let _ = ack.send(empty); @@ -1405,6 +1418,21 @@ fn uploader_loop( } Err(RecvTimeoutError::Disconnected) => break, } + retire_legacy(&mut legacy, &legacy_ready); + } +} + +/// Stop polling the legacy directory once recovery has run (it runs once per +/// launch, so nothing new can appear) and its queue is empty. +fn retire_legacy(legacy: &mut Option, ready: &AtomicBool) { + if !ready.load(Ordering::SeqCst) { + return; + } + if legacy + .as_deref() + .is_some_and(|dir| queue::list_pending(dir).is_empty()) + { + *legacy = None; } } @@ -1672,7 +1700,7 @@ fn backoff_delay(retry: u32, base: Duration) -> i64 { /// whether the queue ended up empty. fn drain_until_empty( shared: &Shared, - data_dir: Option<&Path>, + dirs: &[&Path], backoff_base: Duration, deadline: Duration, ) -> bool { @@ -1680,7 +1708,7 @@ fn drain_until_empty( loop { let now = epoch_ms(); drain_memory(shared, backoff_base, true, false, Some(start + deadline)); - if let Some(data_dir) = data_dir { + for &data_dir in dirs { for report in queue::list_pending(data_dir) { let Some(_claim) = queue::try_claim(&report) else { continue; // a peer is delivering it; the loop below waits it out @@ -1704,7 +1732,7 @@ fn drain_until_empty( } } } - let disk_empty = data_dir.is_none_or(|d| queue::list_pending(d).is_empty()); + let disk_empty = dirs.iter().all(|d| queue::list_pending(d).is_empty()); if disk_empty && shared.mem_queue.lock().is_empty() { return true; } diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index eb021de..d8ab57a 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -209,13 +209,16 @@ fn queue_write_failure_holds_the_report_in_memory() { } #[test] -fn legacy_directory_leftovers_are_delivered_once() { +fn legacy_directory_leftovers_are_delivered_once_and_cleaned_up() { + use bugsee_core::capture::record; use bugsee_core::model::report::TimeWindow; use bugsee_core::reporting::assemble_docs; let dir = TempDir::new(); let legacy = dir.path.join("legacy"); let data_dir = dir.path.join("data"); + + // A report an old build queued but never delivered... let meta = manual_upload_meta(); let env = bugsee_core::model::environment::Environment::detect_with( "t", @@ -226,23 +229,41 @@ fn legacy_directory_leftovers_are_delivered_once() { let stranded = assemble_docs(Vec::new(), window, &meta, &env, None, &[], "T", 1).unwrap(); bugsee_core::queue::enqueue(&legacy, &stranded).unwrap(); + // ...and a session of that build that died (marker left behind, owner gone). + let part = legacy.join("parts").join("1").join("0"); + std::fs::create_dir_all(&part).unwrap(); + let mut bytes = Vec::new(); + let payload = br#"{"timestamp":100,"level":1,"source":2,"message":"legacy crash line"}"#; + record::frame(100, payload, &mut bytes); + std::fs::write(part.join("log.part"), &bytes).unwrap(); + std::fs::create_dir_all(legacy.join("sessions")).unwrap(); + std::fs::write(legacy.join("sessions").join("1.alive"), "0").unwrap(); + std::fs::write(legacy.join("gen"), "1").unwrap(); + let transport = Arc::new(MockTransport::default()); let mut config = RecorderConfig::new(&data_dir, "T"); config.legacy_data_dir = Some(legacy.clone()); let recorder = Recorder::launch(config, transport.clone()).expect("launch"); + + // Recovery runs on the worker before the flush is acknowledged, and a flush + // drains the legacy queue too. assert!(recorder.flush(Duration::from_secs(10))); - // The startup drain is asynchronous; flush covers only the data dir queue. - let start = std::time::Instant::now(); - while transport.uploaded_bundles.lock().unwrap().is_empty() - && start.elapsed() < Duration::from_secs(10) - { - std::thread::sleep(Duration::from_millis(20)); - } - assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 1); + assert_eq!( + transport.uploaded_bundles.lock().unwrap().len(), + 2, + "the queued report and the recovered crash" + ); + assert!(bugsee_core::queue::list_pending(&legacy).is_empty()); assert!( - bugsee_core::queue::list_pending(&legacy).is_empty(), - "a delivered legacy report must be removed" + !legacy.join("sessions").join("1.alive").exists(), + "the recovered session's marker must be gone" ); + assert!(!legacy.join("parts").join("1").exists()); + + // Nothing is delivered a second time, however long we wait. + assert!(recorder.flush(Duration::from_secs(10))); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 2); } /// Blocks every delivery until released, so the backlog cannot drain. diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 54caa8e..a693e7f 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -96,7 +96,9 @@ impl Bugsee { .or_else(|| (!skipped.is_empty()).then(|| skipped.join("; "))) .unwrap_or_else(|| "no usable data directory".to_string()) }); - let data_dir = dir.unwrap_or_else(|| std::env::temp_dir().join("bugsee")); + // Memory mode never touches the path; an empty one avoids resolving a + // temp directory just to ignore it. + let data_dir = dir.unwrap_or_default(); if memory_only.is_none() && !skipped.is_empty() { static WARNED: AtomicBool = AtomicBool::new(false); bugsee_core::util::warn_once( @@ -586,9 +588,23 @@ fn revive_after_fork(epoch: u64) { #[cfg(feature = "panic")] if recorder.storage_mode() == StorageMode::Disk { bugsee_panic::set_snapshot_path(recorder.panic_info_path()); + } else { + // The inherited path points into the PARENT's live session; a panic here + // must not write its snapshot there. + bugsee_panic::clear_snapshot_path(); } - *RECORDER.lock() = Some(recorder); + // Lock order RECORDER -> RESOLUTION, as in `launch_with`/`storage_status`. + { + let mut slot = RECORDER.lock(); + if recorder.storage_mode() == StorageMode::Memory { + if let Some(resolved) = RESOLUTION.lock().as_mut() { + resolved.dir = None; + resolved.source = DataDirSource::None; + } + } + *slot = Some(recorder); + } *failures = 0; SEEN_FORK_EPOCH.store(epoch, Ordering::SeqCst); } diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index 757d084..e4de340 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -238,7 +238,7 @@ impl LaunchOptions { self.data_dir.as_deref(), &self.app_token, &|name| std::env::var_os(name), - &std::env::temp_dir(), + &std::env::temp_dir, ) } } @@ -304,7 +304,7 @@ pub(crate) fn resolve_data_dir( explicit: Option<&Path>, app_token: &str, env: &dyn Fn(&str) -> Option, - temp: &Path, + temp: &dyn Fn() -> PathBuf, ) -> DataDirResolution { let token = short_token(app_token); @@ -334,6 +334,8 @@ pub(crate) fn resolve_data_dir( }; } + // Only now: an explicit directory never needs the temp location. + let temp = temp(); let mut candidates: Vec<(DataDirSource, PathBuf, bool)> = Vec::new(); if let Some(base) = platform_data_base(env) { candidates.push(( @@ -363,8 +365,8 @@ pub(crate) fn resolve_data_dir( let legacy = temp.join("bugsee").join(&token); let legacy_dir = found .as_ref() - .filter(|(_, dir)| *dir != legacy) - .and_then(|_| trusted_existing_dir(&legacy).then_some(legacy)); + .filter(|(_, dir)| legacy_is_adoptable(&legacy, dir)) + .map(|_| legacy); match found { Some((source, dir)) => DataDirResolution { dir: Some(dir), @@ -485,6 +487,24 @@ fn owned_by_us(_meta: &std::fs::Metadata) -> bool { true } +/// Whether `legacy` (`/bugsee/`) may be recovered from. Both it AND +/// `/bugsee` must be real directories owned by us: a symlink at either +/// would let someone else's tree, or our own live data directory, be processed +/// (recovering our live session as a crash). Paths are compared canonicalised, +/// not as strings, so another spelling of our own directory is rejected too. +fn legacy_is_adoptable(legacy: &Path, chosen: &Path) -> bool { + let Some(parent) = legacy.parent() else { + return false; + }; + if !trusted_existing_dir(parent) || !trusted_existing_dir(legacy) { + return false; + } + match (legacy.canonicalize(), chosen.canonicalize()) { + (Ok(a), Ok(b)) => a != b, + _ => false, + } +} + /// Prove a file can be created in `dir`. `create_new` with an unpredictable name /// refuses to follow or reuse anything pre-planted at that path. fn probe_writable(dir: &Path) -> std::io::Result<()> { @@ -601,7 +621,7 @@ mod tests { let s = scratch(); let explicit = s.join("explicit"); let env = [(DATA_DIR_ENV, s.join("from-env").into_os_string())]; - let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&env), &s.join("tmp")); + let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&env), &|| s.join("tmp")); assert_eq!(r.source, DataDirSource::Explicit); assert_eq!(r.dir.as_deref(), Some(explicit.as_path())); assert!(r.skipped.is_empty() && r.explicit_rejected.is_none()); @@ -615,12 +635,9 @@ mod tests { let env = [(DATA_DIR_ENV, good_env.clone().into_os_string())]; let mut full_env = env.to_vec(); full_env.extend(home_env(&s.join("home"))); - let r = resolve_data_dir( - Some(&blocked(&s, "f1")), - "tok", - &env_of(&full_env), - &s.join("tmp"), - ); + let r = resolve_data_dir(Some(&blocked(&s, "f1")), "tok", &env_of(&full_env), &|| { + s.join("tmp") + }); assert!(r.dir.is_none()); assert_eq!(r.source, DataDirSource::None); assert!(r.explicit_rejected.is_some()); @@ -637,7 +654,7 @@ mod tests { let s = scratch(); let mut env = vec![(DATA_DIR_ENV, blocked(&s, "f2").into_os_string())]; env.extend(home_env(&s.join("home"))); - let r = resolve_data_dir(None, "tok", &env_of(&env), &s.join("tmp")); + let r = resolve_data_dir(None, "tok", &env_of(&env), &|| s.join("tmp")); assert!(r.dir.is_none() && r.explicit_rejected.is_some()); assert!(!s.join("home").exists() && !s.join("tmp").exists()); let _ = std::fs::remove_dir_all(&s); @@ -648,7 +665,7 @@ mod tests { let s = scratch(); let good = s.join("from-env"); let env = [(DATA_DIR_ENV, good.clone().into_os_string())]; - let r = resolve_data_dir(None, "tok", &env_of(&env), &s.join("tmp")); + let r = resolve_data_dir(None, "tok", &env_of(&env), &|| s.join("tmp")); assert_eq!(r.source, DataDirSource::Environment); assert_eq!(r.dir, Some(good)); let _ = std::fs::remove_dir_all(&s); @@ -658,7 +675,7 @@ mod tests { fn platform_default_is_used_when_home_is_set() { let s = scratch(); let home = s.join("home"); - let r = resolve_data_dir(None, "tok-1", &env_of(&home_env(&home)), &s.join("tmp")); + let r = resolve_data_dir(None, "tok-1", &env_of(&home_env(&home)), &|| s.join("tmp")); assert_eq!(r.source, DataDirSource::PlatformDefault); assert!(r.dir.unwrap().starts_with(&home)); let _ = std::fs::remove_dir_all(&s); @@ -668,7 +685,7 @@ mod tests { fn no_home_falls_back_to_a_private_temp_directory() { let s = scratch(); let tmp = s.join("tmp"); - let r = resolve_data_dir(None, "tok-1", &env_of(&[]), &tmp); + let r = resolve_data_dir(None, "tok-1", &env_of(&[]), &|| tmp.clone()); assert_eq!(r.source, DataDirSource::TempFallback); let dir = r.dir.unwrap(); assert_eq!(dir, tmp.join(private_temp_name()).join("tok1")); @@ -692,7 +709,7 @@ mod tests { std::fs::create_dir_all(&tmp).unwrap(); std::fs::create_dir_all(&elsewhere).unwrap(); std::os::unix::fs::symlink(&elsewhere, tmp.join(private_temp_name())).unwrap(); - let r = resolve_data_dir(None, "tok", &env_of(&[]), &tmp); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); assert!(r.dir.is_none(), "must not follow the symlink"); assert_eq!(r.skipped.len(), 1); assert!( @@ -706,12 +723,9 @@ mod tests { fn nothing_writable_resolves_to_none_and_lists_every_default_attempt() { let s = scratch(); let blocked_home = blocked(&s, "f3"); - let r = resolve_data_dir( - None, - "tok", - &env_of(&home_env(&blocked_home)), - &blocked(&s, "f4"), - ); + let r = resolve_data_dir(None, "tok", &env_of(&home_env(&blocked_home)), &|| { + blocked(&s, "f4") + }); assert_eq!(r.source, DataDirSource::None); assert!(r.dir.is_none()); assert_eq!(r.skipped.len(), 2); @@ -722,7 +736,7 @@ mod tests { fn an_empty_environment_variable_is_ignored() { let s = scratch(); let env = [(DATA_DIR_ENV, OsString::new())]; - let r = resolve_data_dir(None, "tok", &env_of(&env), &s.join("tmp")); + let r = resolve_data_dir(None, "tok", &env_of(&env), &|| s.join("tmp")); assert_eq!(r.source, DataDirSource::TempFallback); let _ = std::fs::remove_dir_all(&s); } @@ -733,18 +747,73 @@ mod tests { fn the_previous_default_location_is_offered_for_recovery_only_if_it_exists() { let s = scratch(); let tmp = s.join("tmp"); - let r = resolve_data_dir(None, "tok", &env_of(&[]), &tmp); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); assert!(r.legacy_dir.is_none(), "nothing to adopt yet"); let legacy = tmp.join("bugsee").join("tok"); std::fs::create_dir_all(&legacy).unwrap(); - let r = resolve_data_dir(None, "tok", &env_of(&[]), &tmp); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); assert_eq!(r.legacy_dir, Some(legacy.clone())); // A host that chose its directory never used the legacy one. let explicit = s.join("explicit"); - let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&[]), &tmp); + let r = resolve_data_dir(Some(&explicit), "tok", &env_of(&[]), &|| tmp.clone()); + assert!(r.legacy_dir.is_none()); + let _ = std::fs::remove_dir_all(&s); + } + #[cfg(unix)] + #[test] + fn a_symlinked_legacy_location_is_never_adopted() { + let s = scratch(); + let tmp = s.join("tmp"); + let other = s.join("other"); + std::fs::create_dir_all(other.join("tok")).unwrap(); + std::fs::create_dir_all(&tmp).unwrap(); + // `/bugsee` itself is a symlink to a tree that has our token dir. + std::os::unix::fs::symlink(&other, tmp.join("bugsee")).unwrap(); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); + assert!(r.dir.is_some() && r.legacy_dir.is_none()); + + // The token directory itself being a symlink is refused as well. + std::fs::remove_file(tmp.join("bugsee")).unwrap(); + std::fs::create_dir_all(tmp.join("bugsee")).unwrap(); + std::os::unix::fs::symlink(other.join("tok"), tmp.join("bugsee").join("tok")).unwrap(); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); assert!(r.legacy_dir.is_none()); let _ = std::fs::remove_dir_all(&s); } + + // The platform default must be `/bugsee/` to collide. + #[cfg(all(unix, not(target_vendor = "apple")))] + #[test] + fn the_legacy_location_is_skipped_when_it_is_our_own_directory_by_another_spelling() { + let s = scratch(); + let tmp = s.join("tmp"); + let legacy = tmp.join("bugsee").join("tok"); + std::fs::create_dir_all(&legacy).unwrap(); + // The platform default resolves to the same directory, spelled with `..`. + let base = tmp.join("elsewhere").join(".."); + std::fs::create_dir_all(tmp.join("elsewhere")).unwrap(); + let env = [ + ("HOME", base.clone().into_os_string()), + ("XDG_DATA_HOME", base.into_os_string()), + ]; + let r = resolve_data_dir(None, "tok", &env_of(&env), &|| tmp.clone()); + assert_eq!(r.source, DataDirSource::PlatformDefault); + assert!( + r.legacy_dir.is_none(), + "our live data directory must not be recovered as a legacy one" + ); + let _ = std::fs::remove_dir_all(&s); + } + + #[test] + fn an_explicit_directory_never_asks_for_the_temp_location() { + let s = scratch(); + let r = resolve_data_dir(Some(&s.join("explicit")), "tok", &env_of(&[]), &|| { + panic!("temp_dir must not be resolved eagerly") + }); + assert_eq!(r.source, DataDirSource::Explicit); + let _ = std::fs::remove_dir_all(&s); + } } diff --git a/crates/bugsee/tests/fork_safety.rs b/crates/bugsee/tests/fork_safety.rs index 64365f0..a27b0f5 100644 --- a/crates/bugsee/tests/fork_safety.rs +++ b/crates/bugsee/tests/fork_safety.rs @@ -519,6 +519,68 @@ fn a_panic_in_the_child_before_touching_the_sdk_does_not_write_into_the_parents_ let _ = std::fs::remove_dir_all(&out); } +/// A child that cannot open its own session falls back to memory. It must then +/// stop pointing the panic observer into the PARENT's live session, and its +/// storage status must say it has no directory. +#[test] +fn a_child_that_falls_back_to_memory_does_not_write_into_the_parents_session() { + let _serial = serial(); + let data = scratch("data"); + let out = scratch("out"); + let guard = Bugsee::launch_with( + LaunchOptions::new("APP_TOKEN") + .data_dir(&data) + .native_crash_capture(false) + .with_transport(FileTransport::new(&out)), + ) + .expect("launch"); + std::thread::sleep(Duration::from_millis(300)); + + // Make a NEW session impossible (the child's `Session::begin` needs + // `sessions/` to be a directory) while the parent's parts stay writable. + let sessions = data.join("sessions"); + let parked = data.join("sessions.parked"); + std::fs::rename(&sessions, &parked).unwrap(); + std::fs::write(&sessions, b"not a directory").unwrap(); + + let child = fork_child(|| { + Bugsee::capture_message(LogLevel::Warning, "child-starting"); // revives + let Some(status) = Bugsee::storage_status() else { + return 2; + }; + if status.mode != bugsee::StorageMode::Memory || status.data_dir.is_some() { + return 3; + } + let _ = std::thread::spawn(|| panic!("child-panic-in-memory-mode")).join(); + 0 + }); + assert!(exited_with( + wait_with_deadline(child, Duration::from_secs(30)), + 0 + )); + + let stray: Vec<_> = std::fs::read_dir(data.join("parts")) + .into_iter() + .flatten() + .flatten() + .map(|d| d.path().join("panic.info")) + .filter(|p| p.exists()) + .collect(); + assert!( + stray.is_empty(), + "the child wrote its panic into its parent's session: {stray:?}" + ); + // The parent is untouched. + let parent = Bugsee::storage_status().expect("status"); + assert_eq!(parent.mode, bugsee::StorageMode::Disk); + + let _ = std::fs::remove_file(&sessions); + let _ = std::fs::rename(&parked, &sessions); + drop(guard); + let _ = std::fs::remove_dir_all(&data); + let _ = std::fs::remove_dir_all(&out); +} + /// Fails transiently until `allow` exists, then behaves like [`FileTransport`]. struct AllowTransport { allow: PathBuf, From cc96a411bd60698b11d5cb560ab1fdc9d9a134d7 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:01:36 +0500 Subject: [PATCH 04/52] feat(bugsee): populate utc_offset, hardware model and Windows locale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds crates/bugsee/src/hostinfo.rs with per-platform lookups (libc on unix, hand-declared Win32 calls on Windows; no new dependencies) and threads the results through HostFacts into platform.utc_offset and hardware.model. Closes #16 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- PROGRESS.md | 21 +- crates/bugsee-core/src/model/environment.rs | 14 + crates/bugsee/Cargo.toml | 4 +- crates/bugsee/src/hostinfo.rs | 400 ++++++++++++++++++++ crates/bugsee/src/lib.rs | 2 + crates/bugsee/src/telemetry.rs | 11 +- 6 files changed, 438 insertions(+), 14 deletions(-) create mode 100644 crates/bugsee/src/hostinfo.rs diff --git a/PROGRESS.md b/PROGRESS.md index 6f6bbeb..03f5e8c 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -426,14 +426,19 @@ can fix. browser SDK is also the remaining producer exposed to the `platform.jailbreak` class of bug β€” the appserver fix protects it now, but its `EnvironmentPlatform` is still `{type, os, locale}`. -3. **Two environment fields remain unpopulated**, both needing a decision rather - than effort. `utc_offset`: the `time` crate refuses a local offset in a - multithreaded process (an SDK always is one), so it needs `localtime_r`/Win32 - and a small `unsafe` β€” which belongs in the host crate, not core. - `hardware.model`: no `sysinfo` API, per-platform work, and on desktop it - yields a mainboard string worth little until Phase 6 puts this SDK on mobile. - Windows `locale` is likewise unset β€” `GetUserDefaultLocaleName` needs a - binding or a crate, and this SDK still takes no new dependencies. +3. **Environment fields `utc_offset`, `hardware.model` and Windows `locale` are + now populated** (#16), in `crates/bugsee/src/hostinfo.rs` β€” the one place in + the host crate that calls the OS, so the `unsafe` stays out of core. No new + dependencies: `libc` (already in the tree) on unix, hand-declared + `kernel32`/`advapi32` calls on Windows. `utc_offset` is `localtime_r` + `tm_gmtoff` / `GetTimeZoneInformation` (the `time` crate refuses a local + offset in a multithreaded process). `hardware.model` is `sysctl hw.model` on + macOS/iOS, DMI `product_name` then the device-tree `model` on Linux, and the + SMBIOS `SystemProductName` registry value on Windows; firmware placeholders + ("To Be Filled By O.E.M.") are reported as absent, and Android reports none. + Windows `locale` falls back to `GetUserDefaultLocaleName` when the POSIX + variables are unset. Windows code is type-checked and clippy-clean but its + tests have not been run on real hardware. ### Decided against diff --git a/crates/bugsee-core/src/model/environment.rs b/crates/bugsee-core/src/model/environment.rs index 537b792..19c28da 100644 --- a/crates/bugsee-core/src/model/environment.rs +++ b/crates/bugsee-core/src/model/environment.rs @@ -233,6 +233,10 @@ pub struct HostFacts { pub disk_free: Option, /// `platform.locale`, in the `en_US` form the other SDKs report. pub locale: Option, + /// `platform.utc_offset`, minutes east of UTC when the facts were taken. + pub utc_offset: Option, + /// `hardware.model`, the machine's model string (e.g. `"MacBookPro18,3"`). + pub hardware_model: Option, } impl Environment { @@ -260,6 +264,8 @@ impl Environment { env.platform.disk_total = facts.disk_total; env.platform.disk_free = facts.disk_free; env.platform.locale = facts.locale.clone(); + env.platform.utc_offset = facts.utc_offset; + env.hardware.model = facts.hardware_model.clone(); env } @@ -374,6 +380,8 @@ mod tests { disk_total: Some(500000), disk_free: Some(120000), locale: Some("en_US".into()), + utc_offset: Some(-300), + hardware_model: Some("MacBookPro18,3".into()), }; let v = serde_json::to_value(Environment::detect_with( "1.0.0", @@ -389,6 +397,10 @@ mod tests { assert_eq!(v["platform"]["disk_total"], 500000); assert_eq!(v["platform"]["disk_free"], 120000); assert_eq!(v["platform"]["locale"], "en_US"); + // Negative: west of UTC. The sign convention is the contract's ("minutes + // east"), so pin it here rather than only in the host crate. + assert_eq!(v["platform"]["utc_offset"], -300); + assert_eq!(v["hardware"]["model"], "MacBookPro18,3"); } #[test] @@ -407,12 +419,14 @@ mod tests { "disk_total", "disk_free", "locale", + "utc_offset", ] { assert!( !platform.contains_key(key), "{key} should be omitted, not null" ); } + assert!(!v["hardware"].as_object().unwrap().contains_key("model")); } #[test] diff --git a/crates/bugsee/Cargo.toml b/crates/bugsee/Cargo.toml index bc971a1..68edf63 100644 --- a/crates/bugsee/Cargo.toml +++ b/crates/bugsee/Cargo.toml @@ -40,5 +40,7 @@ name = "crash_e2e_matrix" harness = false required-features = ["native"] -[target.'cfg(unix)'.dev-dependencies] +# Local UTC offset and the hardware model are libc calls on unix; `libc` is +# already in the tree via `bugsee-core` / `bugsee-native`, so this adds nothing. +[target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/crates/bugsee/src/hostinfo.rs b/crates/bugsee/src/hostinfo.rs new file mode 100644 index 0000000..2bff5a1 --- /dev/null +++ b/crates/bugsee/src/hostinfo.rs @@ -0,0 +1,400 @@ +// +// hostinfo.rs +// bugsee +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// + +//! Per-platform host facts that neither `std` nor `sysinfo` expose: the local +//! UTC offset, the hardware model, and the Windows user locale. +//! +//! This is the one place in the host crate that calls into the OS directly, so +//! the `unsafe` is confined here and kept to single FFI calls with no +//! invariants beyond their arguments. No binding crate is used: `libc` is +//! already in the tree, and the three Win32 calls are declared by hand (a +//! binding crate for three functions is a worse trade than 15 lines of +//! `extern`). Everything is best-effort and returns `None` rather than a guess. + +/// Minutes east of UTC for the local time zone right now. +/// +/// Not the `time` crate: it refuses to read the local offset once a process is +/// multithreaded (an SDK always is), because `localtime_r` can race a +/// concurrent `setenv("TZ")`. We accept that narrow race for a single +/// diagnostic number, which the crate's blanket refusal does not allow. +/// +/// Sampled once when the environment is built, so a DST change during a very +/// long run is not reflected in that run's reports. +pub(crate) fn utc_offset_minutes() -> Option { + imp::utc_offset_minutes() +} + +/// The machine's model string, e.g. `MacBookPro18,3`, `iPhone14,2`, +/// `ThinkPad X1 Carbon`, `Raspberry Pi 4 Model B`. +/// +/// On desktop Linux and Windows this is the firmware's product name, which on +/// white-box or DIY machines is often a placeholder; those are reported as +/// absent (see [`clean_model`]) rather than as a model. +pub(crate) fn hardware_model() -> Option { + imp::hardware_model() +} + +/// The user's locale from the OS settings, in the `en_US` form. Only Windows +/// has a setting beyond the POSIX environment variables the caller already +/// reads; elsewhere this is `None`. +pub(crate) fn locale() -> Option { + imp::locale() +} + +/// Trim, drop embedded NULs (device-tree strings are NUL-terminated), and +/// reject the placeholders firmware vendors ship when they never filled the +/// field in. Reporting "To Be Filled By O.E.M." as a hardware model would be +/// worse than reporting nothing. +fn clean_model(raw: &str) -> Option { + let value = raw.replace('\0', ""); + let value = value.trim(); + const PLACEHOLDERS: [&str; 8] = [ + "to be filled by o.e.m.", + "system product name", + "default string", + "not specified", + "not applicable", + "unknown", + "none", + "n/a", + ]; + if value.is_empty() || PLACEHOLDERS.iter().any(|p| value.eq_ignore_ascii_case(p)) { + return None; + } + Some(value.to_string()) +} + +#[cfg(unix)] +mod imp { + use super::clean_model; + + pub(super) fn utc_offset_minutes() -> Option { + // SAFETY: `localtime_r` writes only into the `tm` we hand it and reads + // the `time_t` by value; a zeroed `tm` is a valid out-parameter. + let gmtoff = unsafe { + let now = libc::time(std::ptr::null_mut()); + let mut tm: libc::tm = std::mem::zeroed(); + if libc::localtime_r(&now, &mut tm).is_null() { + return None; + } + tm.tm_gmtoff + }; + Some((gmtoff / 60) as i32) + } + + #[cfg(target_vendor = "apple")] + pub(super) fn hardware_model() -> Option { + // `hw.model` is the model identifier on both macOS ("MacBookPro18,3") + // and iOS ("iPhone14,2"). + let name = c"hw.model"; + let mut len: libc::size_t = 0; + // SAFETY: a null buffer asks for the required length only. + let rc = unsafe { + libc::sysctlbyname( + name.as_ptr(), + std::ptr::null_mut(), + &mut len, + std::ptr::null_mut(), + 0, + ) + }; + if rc != 0 || len == 0 { + return None; + } + let mut buf = vec![0u8; len]; + // SAFETY: `buf` is `len` bytes, the size the kernel just reported. + let rc = unsafe { + libc::sysctlbyname( + name.as_ptr(), + buf.as_mut_ptr().cast(), + &mut len, + std::ptr::null_mut(), + 0, + ) + }; + if rc != 0 { + return None; + } + buf.truncate(len); + clean_model(&String::from_utf8_lossy(&buf)) + } + + #[cfg(not(target_vendor = "apple"))] + pub(super) fn hardware_model() -> Option { + // DMI covers x86 servers and laptops; the device tree covers ARM + // boards (Raspberry Pi and friends), which have no DMI. Android exposes + // neither to an app, so it reports nothing here. + [ + "/sys/class/dmi/id/product_name", + "/sys/firmware/devicetree/base/model", + ] + .into_iter() + .find_map(|path| clean_model(&std::fs::read_to_string(path).ok()?)) + } + + pub(super) fn locale() -> Option { + None + } +} + +#[cfg(windows)] +mod imp { + use super::clean_model; + + // `std` already links kernel32; advapi32 is part of every Windows install. + #[repr(C)] + struct SystemTime([u16; 8]); + + #[repr(C)] + struct TimeZoneInformation { + bias: i32, + standard_name: [u16; 32], + standard_date: SystemTime, + standard_bias: i32, + daylight_name: [u16; 32], + daylight_date: SystemTime, + daylight_bias: i32, + } + + #[link(name = "kernel32")] + extern "system" { + fn GetTimeZoneInformation(info: *mut TimeZoneInformation) -> u32; + fn GetUserDefaultLocaleName(name: *mut u16, len: i32) -> i32; + } + + #[link(name = "advapi32")] + extern "system" { + fn RegGetValueW( + hkey: isize, + subkey: *const u16, + value: *const u16, + flags: u32, + kind: *mut u32, + data: *mut core::ffi::c_void, + len: *mut u32, + ) -> i32; + } + + const TIME_ZONE_ID_INVALID: u32 = 0xFFFF_FFFF; + const TIME_ZONE_ID_DAYLIGHT: u32 = 2; + const HKEY_LOCAL_MACHINE: isize = 0x8000_0002_u32 as i32 as isize; + const RRF_RT_REG_SZ: u32 = 0x0000_0002; + + /// Windows reports the *bias*: minutes WEST of UTC (`UTC = local + bias`), + /// plus a correction while daylight time is in force. The contract wants + /// minutes east, so negate. + pub(super) fn offset_from_bias( + bias: i32, + standard_bias: i32, + daylight_bias: i32, + id: u32, + ) -> i32 { + let adjust = if id == TIME_ZONE_ID_DAYLIGHT { + daylight_bias + } else { + standard_bias + }; + -(bias + adjust) + } + + pub(super) fn utc_offset_minutes() -> Option { + // SAFETY: `info` is a correctly laid-out, writable out-parameter. + let (id, info) = unsafe { + let mut info: TimeZoneInformation = std::mem::zeroed(); + let id = GetTimeZoneInformation(&mut info); + (id, info) + }; + if id == TIME_ZONE_ID_INVALID { + return None; + } + Some(offset_from_bias( + info.bias, + info.standard_bias, + info.daylight_bias, + id, + )) + } + + fn wide(s: &str) -> Vec { + s.encode_utf16().chain(std::iter::once(0)).collect() + } + + pub(super) fn hardware_model() -> Option { + // The SMBIOS product name Windows caches at boot. + let subkey = wide(r"HARDWARE\DESCRIPTION\System\BIOS"); + let value = wide("SystemProductName"); + let mut len: u32 = 0; + // SAFETY: a null buffer asks for the required size (in bytes) only. + let rc = unsafe { + RegGetValueW( + HKEY_LOCAL_MACHINE, + subkey.as_ptr(), + value.as_ptr(), + RRF_RT_REG_SZ, + std::ptr::null_mut(), + std::ptr::null_mut(), + &mut len, + ) + }; + if rc != 0 || len < 2 { + return None; + } + let mut buf = vec![0u16; (len as usize).div_ceil(2)]; + // SAFETY: `buf` holds at least `len` bytes, the size just reported. + let rc = unsafe { + RegGetValueW( + HKEY_LOCAL_MACHINE, + subkey.as_ptr(), + value.as_ptr(), + RRF_RT_REG_SZ, + std::ptr::null_mut(), + buf.as_mut_ptr().cast(), + &mut len, + ) + }; + if rc != 0 { + return None; + } + buf.truncate(len as usize / 2); + clean_model(&String::from_utf16_lossy(&buf)) + } + + /// `en-US` -> `en_US`; a script subtag (`zh-Hans-CN`) is dropped so the + /// value matches the language_REGION form the other SDKs send. + pub(super) fn locale_from_bcp47(tag: &str) -> Option { + let mut parts = tag.split('-'); + let lang = parts.next().filter(|l| !l.is_empty())?; + match parts.next_back() { + Some(region) => Some(format!("{lang}_{region}")), + None => Some(lang.to_string()), + } + } + + pub(super) fn locale() -> Option { + // LOCALE_NAME_MAX_LENGTH is 85, including the terminator. + let mut buf = [0u16; 85]; + // SAFETY: `buf` is writable for the `len` code units we pass. + let n = unsafe { GetUserDefaultLocaleName(buf.as_mut_ptr(), buf.len() as i32) }; + if n <= 1 { + return None; + } + locale_from_bcp47(&String::from_utf16_lossy(&buf[..n as usize - 1])) + } +} + +#[cfg(not(any(unix, windows)))] +mod imp { + pub(super) fn utc_offset_minutes() -> Option { + None + } + pub(super) fn hardware_model() -> Option { + None + } + pub(super) fn locale() -> Option { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn placeholder_and_blank_models_are_reported_as_absent() { + for raw in [ + "", + " ", + "\0", + "To Be Filled By O.E.M.", + "System Product Name", + "Default string", + "unknown", + ] { + assert_eq!(clean_model(raw), None, "{raw:?} must be absent"); + } + } + + #[test] + fn real_models_are_trimmed_and_nul_stripped() { + assert_eq!( + clean_model("MacBookPro18,3\n").as_deref(), + Some("MacBookPro18,3") + ); + // Device-tree `model` files end in a NUL. + assert_eq!( + clean_model("Raspberry Pi 4 Model B Rev 1.4\0").as_deref(), + Some("Raspberry Pi 4 Model B Rev 1.4") + ); + } + + #[test] + fn utc_offset_is_a_plausible_whole_minute_value() { + // Real offsets span UTC-12:00 to UTC+14:00. The exact value depends on + // the runner, so this pins "resolved and sane", not a number. + let off = utc_offset_minutes().expect("local offset must resolve on a desktop host"); + assert!((-12 * 60..=14 * 60).contains(&off), "implausible: {off}"); + } + + #[cfg(unix)] + #[test] + fn utc_offset_agrees_with_the_tz_environment() { + // `date +%z` is the independent oracle: same zone database, different + // code path. Compared only when `date` exists (it does on every CI + // image we run). + let Ok(out) = std::process::Command::new("date").arg("+%z").output() else { + return; + }; + let z = String::from_utf8_lossy(&out.stdout); + let z = z.trim(); + if z.len() != 5 { + return; + } + let sign = if z.starts_with('-') { -1 } else { 1 }; + let (h, m): (i32, i32) = (z[1..3].parse().unwrap(), z[3..5].parse().unwrap()); + assert_eq!(utc_offset_minutes(), Some(sign * (h * 60 + m))); + } + + #[cfg(target_vendor = "apple")] + #[test] + fn macos_reports_a_hardware_model() { + let model = hardware_model().expect("hw.model must resolve on Apple hosts"); + assert!(model.contains(','), "expected e.g. MacBookPro18,3: {model}"); + } + + #[cfg(windows)] + #[test] + fn windows_bias_is_negated_and_daylight_adjusted() { + use super::imp::offset_from_bias; + // US Eastern: bias 300 west; DST bias -60 -> UTC-4 while in force. + assert_eq!(offset_from_bias(300, 0, -60, 1), -300); + assert_eq!(offset_from_bias(300, 0, -60, 2), -240); + // India has no DST and a half-hour zone: UTC+5:30. + assert_eq!(offset_from_bias(-330, 0, 0, 0), 330); + } + + #[cfg(windows)] + #[test] + fn windows_locale_tags_map_to_the_posix_form() { + use super::imp::locale_from_bcp47; + assert_eq!(locale_from_bcp47("en-US").as_deref(), Some("en_US")); + assert_eq!(locale_from_bcp47("zh-Hans-CN").as_deref(), Some("zh_CN")); + assert_eq!(locale_from_bcp47("fr").as_deref(), Some("fr")); + assert_eq!(locale_from_bcp47(""), None); + } + + #[cfg(windows)] + #[test] + fn windows_resolves_locale_and_model_on_a_real_host() { + let locale = locale().expect("user locale must resolve on Windows"); + assert!(locale.len() >= 2, "{locale}"); + // Model may legitimately be absent on a VM with placeholder SMBIOS, so + // only assert it never comes back as a placeholder. + if let Some(m) = hardware_model() { + assert_eq!(clean_model(&m).as_deref(), Some(m.as_str())); + } + } +} diff --git a/crates/bugsee/src/lib.rs b/crates/bugsee/src/lib.rs index 867dde3..3c76b46 100644 --- a/crates/bugsee/src/lib.rs +++ b/crates/bugsee/src/lib.rs @@ -27,6 +27,8 @@ mod api; mod frames; +#[cfg(feature = "telemetry")] +mod hostinfo; mod options; mod perf; mod report; diff --git a/crates/bugsee/src/telemetry.rs b/crates/bugsee/src/telemetry.rs index 23d486e..4979386 100644 --- a/crates/bugsee/src/telemetry.rs +++ b/crates/bugsee/src/telemetry.rs @@ -87,7 +87,9 @@ pub fn host_facts(data_dir: &Path) -> HostFacts { memory_free: to_mb(sys.available_memory()), disk_total, disk_free, - locale: locale(), + locale: locale().or_else(crate::hostinfo::locale), + utc_offset: crate::hostinfo::utc_offset_minutes(), + hardware_model: crate::hostinfo::hardware_model(), } } @@ -136,10 +138,9 @@ fn disk_for(data_dir: &Path) -> (Option, Option) { /// present. Gating it off would also have meant gating this function's tests, /// losing coverage of platform-independent string logic for no benefit. /// -/// What Windows still lacks is the *fallback* when the environment says -/// nothing: `GetUserDefaultLocaleName` needs a Win32 binding or a crate, and -/// this SDK has taken no new dependencies. The field is optional, so such a -/// report simply omits it rather than carrying a wrong value. +/// When the environment says nothing, [`host_facts`] falls back to the +/// platform's own setting (`hostinfo::locale`, Windows only). The field is +/// optional, so a host with neither simply omits it. fn locale() -> Option { ["LC_ALL", "LC_MESSAGES", "LANG"] .into_iter() From a7410a4279764cdae164ed7c12e17ae7e62e34cb Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:08:41 +0500 Subject: [PATCH 05/52] fix(bugsee): address review of hostinfo (iOS model, tm_gmtoff gating, locale) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #16 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- PROGRESS.md | 2 +- crates/bugsee/Cargo.toml | 5 +- crates/bugsee/src/hostinfo.rs | 203 +++++++++++++++++++++++++++------- 3 files changed, 169 insertions(+), 41 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 03f5e8c..b010cc0 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -436,7 +436,7 @@ can fix. macOS/iOS, DMI `product_name` then the device-tree `model` on Linux, and the SMBIOS `SystemProductName` registry value on Windows; firmware placeholders ("To Be Filled By O.E.M.") are reported as absent, and Android reports none. - Windows `locale` falls back to `GetUserDefaultLocaleName` when the POSIX + Windows `locale` falls back to the first preferred UI language (`GetUserPreferredUILanguages`, then `GetUserDefaultLocaleName`) when the POSIX variables are unset. Windows code is type-checked and clippy-clean but its tests have not been run on real hardware. diff --git a/crates/bugsee/Cargo.toml b/crates/bugsee/Cargo.toml index 68edf63..e24104a 100644 --- a/crates/bugsee/Cargo.toml +++ b/crates/bugsee/Cargo.toml @@ -29,7 +29,7 @@ panic = ["dep:bugsee-panic"] # Native fatal-crash handler (signal/Mach exception) + next-launch recovery. native = ["dep:bugsee-native"] # System/process telemetry sampling (traces.system) via sysinfo. -telemetry = ["dep:sysinfo"] +telemetry = ["dep:sysinfo", "dep:libc"] # The crash matrix needs the REAL main thread: an uncaught panic must unwind out # of `main`, and the panic observer treats any other thread as "a thread died", @@ -43,4 +43,7 @@ required-features = ["native"] # Local UTC offset and the hardware model are libc calls on unix; `libc` is # already in the tree via `bugsee-core` / `bugsee-native`, so this adds nothing. [target.'cfg(unix)'.dependencies] +libc = { version = "0.2", optional = true } + +[target.'cfg(unix)'.dev-dependencies] libc = "0.2" diff --git a/crates/bugsee/src/hostinfo.rs b/crates/bugsee/src/hostinfo.rs index 2bff5a1..5234aa6 100644 --- a/crates/bugsee/src/hostinfo.rs +++ b/crates/bugsee/src/hostinfo.rs @@ -11,36 +11,44 @@ //! This is the one place in the host crate that calls into the OS directly, so //! the `unsafe` is confined here and kept to single FFI calls with no //! invariants beyond their arguments. No binding crate is used: `libc` is -//! already in the tree, and the three Win32 calls are declared by hand (a -//! binding crate for three functions is a worse trade than 15 lines of +//! already in the tree, and the handful of Win32 calls are declared by hand (a +//! binding crate for a few functions is a worse trade than a few lines of //! `extern`). Everything is best-effort and returns `None` rather than a guess. /// Minutes east of UTC for the local time zone right now. /// /// Not the `time` crate: it refuses to read the local offset once a process is -/// multithreaded (an SDK always is), because `localtime_r` can race a -/// concurrent `setenv("TZ")`. We accept that narrow race for a single -/// diagnostic number, which the crate's blanket refusal does not allow. +/// multithreaded (an SDK always is), because `localtime_r` reads `TZ` through +/// `getenv` and a concurrent `setenv` elsewhere in the process is a data race +/// that, in glibc, can be a use-after-free of `environ` β€” not merely a wrong +/// number. We accept that risk for one diagnostic value because this is called +/// exactly once, from `init` (`api.rs`), before the SDK starts its own threads. +/// Hosts that mutate the environment from other threads should call `init` +/// first. /// /// Sampled once when the environment is built, so a DST change during a very -/// long run is not reflected in that run's reports. +/// long run is not reflected in that run's reports. Unix targets whose `libc` +/// has no `tm_gmtoff` (illumos, Solaris, AIX, newlib, VxWorks) report `None`. pub(crate) fn utc_offset_minutes() -> Option { imp::utc_offset_minutes() } -/// The machine's model string, e.g. `MacBookPro18,3`, `iPhone14,2`, -/// `ThinkPad X1 Carbon`, `Raspberry Pi 4 Model B`. +/// The machine's model string, e.g. `MacBookPro18,3` (macOS), `iPhone14,2` +/// (iOS), `ThinkPad X1 Carbon` or `Raspberry Pi 4 Model B`. /// /// On desktop Linux and Windows this is the firmware's product name, which on /// white-box or DIY machines is often a placeholder; those are reported as -/// absent (see [`clean_model`]) rather than as a model. +/// absent (see [`clean_model`]) rather than as a model. Note that on some +/// vendors the field holds a part number rather than a marketing name. pub(crate) fn hardware_model() -> Option { imp::hardware_model() } -/// The user's locale from the OS settings, in the `en_US` form. Only Windows -/// has a setting beyond the POSIX environment variables the caller already -/// reads; elsewhere this is `None`. +/// The user's locale from the OS settings, in the `en_US` form, for when the +/// POSIX environment variables the caller already reads say nothing. Only +/// Windows has such a setting wired up. macOS GUI apps and iOS do not set +/// `LANG` either, so they currently report no locale; reading the preferred +/// language from CoreFoundation is a follow-up. pub(crate) fn locale() -> Option { imp::locale() } @@ -49,15 +57,19 @@ pub(crate) fn locale() -> Option { /// reject the placeholders firmware vendors ship when they never filled the /// field in. Reporting "To Be Filled By O.E.M." as a hardware model would be /// worse than reporting nothing. +#[cfg_attr(not(any(unix, windows)), allow(dead_code))] fn clean_model(raw: &str) -> Option { let value = raw.replace('\0', ""); let value = value.trim(); - const PLACEHOLDERS: [&str; 8] = [ + const PLACEHOLDERS: [&str; 11] = [ "to be filled by o.e.m.", "system product name", "default string", "not specified", "not applicable", + "all series", + "type1productconfigid", + "invalid", "unknown", "none", "n/a", @@ -68,14 +80,52 @@ fn clean_model(raw: &str) -> Option { Some(value.to_string()) } +/// `en-US` -> `en_US`. Keeps the language and the region only: a 4-letter +/// script (`sr-Latn-RS` -> `sr_RS`) or variant (`ca-ES-valencia` -> `ca_ES`) is +/// skipped, so the value matches the language_REGION form the other SDKs send. +/// The region is the first subtag after the language that is 2 letters or 3 +/// digits; a tag without one (`fr`, `sr-Latn`) yields the bare language. +/// Anything from a `_` on (`de-DE_phoneb`, a Windows sort-order suffix) is cut +/// first. +#[cfg_attr(not(windows), allow(dead_code))] +fn locale_from_bcp47(tag: &str) -> Option { + let tag = tag.split('_').next().unwrap_or(""); + let mut parts = tag.split('-'); + let lang = parts.next().filter(|l| !l.is_empty())?; + let region = parts.find(|p| { + (p.len() == 2 && p.bytes().all(|b| b.is_ascii_alphabetic())) + || (p.len() == 3 && p.bytes().all(|b| b.is_ascii_digit())) + }); + Some(match region { + Some(region) => format!("{lang}_{region}"), + None => lang.to_string(), + }) +} + #[cfg(unix)] mod imp { use super::clean_model; + // The OSes whose `libc::tm` carries `tm_gmtoff`. + #[cfg(any( + target_os = "linux", + target_os = "android", + target_vendor = "apple", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly" + ))] pub(super) fn utc_offset_minutes() -> Option { - // SAFETY: `localtime_r` writes only into the `tm` we hand it and reads - // the `time_t` by value; a zeroed `tm` is a valid out-parameter. + // SAFETY: `tzset` takes no arguments; `localtime_r` writes only into + // the `tm` we hand it and reads the `time_t` by value; a zeroed `tm` is + // a valid out-parameter. See the caller's note on the `TZ` race. let gmtoff = unsafe { + // POSIX, but not every `libc` crate target binds it. + extern "C" { + fn tzset(); + } + tzset(); let now = libc::time(std::ptr::null_mut()); let mut tm: libc::tm = std::mem::zeroed(); if libc::localtime_r(&now, &mut tm).is_null() { @@ -86,11 +136,38 @@ mod imp { Some((gmtoff / 60) as i32) } + #[cfg(not(any( + target_os = "linux", + target_os = "android", + target_vendor = "apple", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly" + )))] + pub(super) fn utc_offset_minutes() -> Option { + None + } + + /// macOS: `hw.model` is the model identifier ("MacBookPro18,3"). Every other + /// Apple OS: `hw.model` is the *board* id ("D63AP"), and the product + /// identifier ("iPhone14,2") is `hw.machine`, the same value iOS SDKs read + /// from `utsname.machine`. In the simulator `hw.machine` is the host's + /// architecture, so the simulated device's identifier is taken from + /// `SIMULATOR_MODEL_IDENTIFIER` first. #[cfg(target_vendor = "apple")] pub(super) fn hardware_model() -> Option { - // `hw.model` is the model identifier on both macOS ("MacBookPro18,3") - // and iOS ("iPhone14,2"). + #[cfg(not(target_os = "macos"))] + if let Some(sim) = std::env::var("SIMULATOR_MODEL_IDENTIFIER") + .ok() + .and_then(|v| clean_model(&v)) + { + return Some(sim); + } + #[cfg(target_os = "macos")] let name = c"hw.model"; + #[cfg(not(target_os = "macos"))] + let name = c"hw.machine"; let mut len: libc::size_t = 0; // SAFETY: a null buffer asks for the required length only. let rc = unsafe { @@ -164,6 +241,12 @@ mod imp { extern "system" { fn GetTimeZoneInformation(info: *mut TimeZoneInformation) -> u32; fn GetUserDefaultLocaleName(name: *mut u16, len: i32) -> i32; + fn GetUserPreferredUILanguages( + flags: u32, + count: *mut u32, + buf: *mut u16, + len: *mut u32, + ) -> i32; } #[link(name = "advapi32")] @@ -180,23 +263,26 @@ mod imp { } const TIME_ZONE_ID_INVALID: u32 = 0xFFFF_FFFF; + const TIME_ZONE_ID_STANDARD: u32 = 1; const TIME_ZONE_ID_DAYLIGHT: u32 = 2; + const MUI_LANGUAGE_NAME: u32 = 0x8; const HKEY_LOCAL_MACHINE: isize = 0x8000_0002_u32 as i32 as isize; const RRF_RT_REG_SZ: u32 = 0x0000_0002; /// Windows reports the *bias*: minutes WEST of UTC (`UTC = local + bias`), /// plus a correction while daylight time is in force. The contract wants - /// minutes east, so negate. + /// minutes east, so negate. `TIME_ZONE_ID_UNKNOWN` (no DST rules for the + /// zone) is just the bias. pub(super) fn offset_from_bias( bias: i32, standard_bias: i32, daylight_bias: i32, id: u32, ) -> i32 { - let adjust = if id == TIME_ZONE_ID_DAYLIGHT { - daylight_bias - } else { - standard_bias + let adjust = match id { + TIME_ZONE_ID_DAYLIGHT => daylight_bias, + TIME_ZONE_ID_STANDARD => standard_bias, + _ => 0, }; -(bias + adjust) } @@ -263,18 +349,45 @@ mod imp { clean_model(&String::from_utf16_lossy(&buf)) } - /// `en-US` -> `en_US`; a script subtag (`zh-Hans-CN`) is dropped so the - /// value matches the language_REGION form the other SDKs send. - pub(super) fn locale_from_bcp47(tag: &str) -> Option { - let mut parts = tag.split('-'); - let lang = parts.next().filter(|l| !l.is_empty())?; - match parts.next_back() { - Some(region) => Some(format!("{lang}_{region}")), - None => Some(lang.to_string()), + /// The user's first preferred UI language (`en-US`), which is what POSIX + /// `LC_MESSAGES` and Android's `Locale.getDefault()` correspond to. + fn ui_language() -> Option { + let mut count: u32 = 0; + let mut len: u32 = 0; + // SAFETY: a null buffer asks for the required length (in u16 units). + let ok = unsafe { + GetUserPreferredUILanguages( + MUI_LANGUAGE_NAME, + &mut count, + std::ptr::null_mut(), + &mut len, + ) + }; + if ok == 0 || len < 2 { + return None; + } + let mut buf = vec![0u16; len as usize]; + // SAFETY: `buf` holds `len` u16 units, the size just reported. + let ok = unsafe { + GetUserPreferredUILanguages(MUI_LANGUAGE_NAME, &mut count, buf.as_mut_ptr(), &mut len) + }; + if ok == 0 { + return None; } + // A double-NUL-terminated list; the first entry is the preferred one. + let first = buf.split(|&c| c == 0).next()?; + super::locale_from_bcp47(&String::from_utf16_lossy(first)) } + /// Prefers the UI language, falling back to the regional-format locale + /// (`GetUserDefaultLocaleName`) if the former cannot be read. The two can + /// differ (English UI with German number formats); the UI language is the + /// better match for what other platforms report as the locale. pub(super) fn locale() -> Option { + ui_language().or_else(regional_locale) + } + + fn regional_locale() -> Option { // LOCALE_NAME_MAX_LENGTH is 85, including the terminator. let mut buf = [0u16; 85]; // SAFETY: `buf` is writable for the `len` code units we pass. @@ -282,7 +395,7 @@ mod imp { if n <= 1 { return None; } - locale_from_bcp47(&String::from_utf16_lossy(&buf[..n as usize - 1])) + super::locale_from_bcp47(&String::from_utf16_lossy(&buf[..n as usize - 1])) } } @@ -313,6 +426,9 @@ mod tests { "System Product Name", "Default string", "unknown", + "All Series", + "Type1ProductConfigId", + "INVALID", ] { assert_eq!(clean_model(raw), None, "{raw:?} must be absent"); } @@ -358,7 +474,7 @@ mod tests { assert_eq!(utc_offset_minutes(), Some(sign * (h * 60 + m))); } - #[cfg(target_vendor = "apple")] + #[cfg(target_os = "macos")] #[test] fn macos_reports_a_hardware_model() { let model = hardware_model().expect("hw.model must resolve on Apple hosts"); @@ -374,16 +490,25 @@ mod tests { assert_eq!(offset_from_bias(300, 0, -60, 2), -240); // India has no DST and a half-hour zone: UTC+5:30. assert_eq!(offset_from_bias(-330, 0, 0, 0), 330); + // TIME_ZONE_ID_UNKNOWN ignores the (garbage-prone) standard bias. + assert_eq!(offset_from_bias(-330, 77, 0, 0), 330); } - #[cfg(windows)] #[test] - fn windows_locale_tags_map_to_the_posix_form() { - use super::imp::locale_from_bcp47; - assert_eq!(locale_from_bcp47("en-US").as_deref(), Some("en_US")); - assert_eq!(locale_from_bcp47("zh-Hans-CN").as_deref(), Some("zh_CN")); - assert_eq!(locale_from_bcp47("fr").as_deref(), Some("fr")); - assert_eq!(locale_from_bcp47(""), None); + fn bcp47_tags_map_to_the_posix_form() { + for (tag, expected) in [ + ("en-US", Some("en_US")), + ("zh-Hans-CN", Some("zh_CN")), + ("fr", Some("fr")), + ("sr-Latn", Some("sr")), + ("sr-Latn-RS", Some("sr_RS")), + ("ca-ES-valencia", Some("ca_ES")), + ("de-DE_phoneb", Some("de_DE")), + ("es-419", Some("es_419")), + ("", None), + ] { + assert_eq!(locale_from_bcp47(tag).as_deref(), expected, "{tag:?}"); + } } #[cfg(windows)] From 11226fbf1e3ae0179c3eac338a067dcb4358e135 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:26:53 +0500 Subject: [PATCH 06/52] fix(core): exactly-once recovery and queue draining across processes sharing a data dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several processes on one data directory could double-deliver (or lose) reports: - A session was recovered twice: a peer that listed it before another finished claimed it after the winner had discarded it, producing a phantom report. The marker is now re-checked after the claim is won. - Stale-claim takeover was remove-then-create, so two takers could each remove the other's fresh claim and both hold it. Takeover is now serialised by a guard file and re-validated under it; release only removes the claim's own content. - Claims and session markers identified owners by bare pid: a recycled pid (or a restarted container's pid 1) kept a dead owner's work blocked forever. Owners now carry a process start time (Linux /proc, macOS proc_pidinfo, Windows GetProcessTimes), with a 30 minute lease as a backstop. - On Windows process liveness always answered "dead", so live peers' claims and markers were taken over. Implemented via OpenProcess/GetExitCodeProcess. - queue::gc_orphans at a peer's launch reaped another process's in-flight enqueue (sidecars before the bundle rename), stranding the bundle and losing the report. Only files untouched for 60 s are reaped now. - Leftover claims of discarded sessions are swept. Adds tests/multiprocess.rs (real child processes racing; SIGKILLed holder) plus unit tests for claims, process identity and queue gc. Closes #14 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 4 +- crates/bugsee-core/src/claim.rs | 297 +++++++++++++++++++---- crates/bugsee-core/src/lib.rs | 1 + crates/bugsee-core/src/process.rs | 232 ++++++++++++++++++ crates/bugsee-core/src/queue.rs | 49 +++- crates/bugsee-core/src/recovery.rs | 59 +++-- crates/bugsee-core/src/session.rs | 4 +- crates/bugsee-core/tests/multiprocess.rs | 245 +++++++++++++++++++ 8 files changed, 811 insertions(+), 80 deletions(-) create mode 100644 crates/bugsee-core/src/process.rs create mode 100644 crates/bugsee-core/tests/multiprocess.rs diff --git a/DESIGN.md b/DESIGN.md index f2f03f6..90bc74b 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -197,7 +197,9 @@ Crash recovery needs the data directory to **outlive the process**: on a contain **Release note, default location moved.** Earlier builds defaulted to `/bugsee/`; the default is now the platform location (3) or the private temp fallback (4). When no directory was requested and the old location exists, is a real directory and is owned by the current user, the launch recovers its crashed sessions and delivers its queue once, best effort (delivery is asynchronous and `flush` does not wait for it). It is not claimed to be lossless: a legacy session whose process is still alive is skipped, and a legacy directory that fails the ownership check is left alone. Hosts that set `data_dir` were never using it. -**Unsafe.** The facade makes two `geteuid()` calls (unix) to name and vet the private temp directory, alongside core's single `kill(pid, 0)`. +**Unsafe.** The facade makes two `geteuid()` calls (unix) to name and vet the private temp directory. Core's process probes (`process.rs`) are `kill(pid, 0)` and `proc_pidinfo` (macOS), plus `OpenProcess`/`GetExitCodeProcess`/`GetProcessTimes` on Windows. + +**Several processes on one directory (#14).** Every pending item is owned by exactly one process at a time: a crashed session by `sessions/.claim`, a queued report by `queue/.claim`. A claim is a file created with `O_EXCL`/`CREATE_NEW` holding `pid start-time nonce`; it is released on drop and, if the holder dies, **taken over** once stale. Stale means: the owner pid is gone, or alive with a different start time (pid reuse; a container's restarted app is pid 1 again), or the file has no owner and is older than 5 s, or it is older than a 30 minute lease (wedged holder, or a platform with no start time). Takeover is serialised by a short-lived `.takeover` guard file (re-checked under the guard), so two takers cannot both remove the other's fresh claim; a guard older than 10 s is abandoned. No OS file lock is used: it would be tied to the open file description, which `fork` duplicates. After winning a session claim the marker is re-checked, because a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s, since a peer's in-flight `enqueue` looks like an orphan (reaping it lost the report). The `.alive` marker also records the start time, so a recycled pid no longer keeps a dead session unrecovered. Delivery is at-least-once: a holder that dies after uploading but before removing the item makes its successor upload it again. Tests: `bugsee-core/tests/multiprocess.rs` (real child processes; racing launchers, SIGKILLed holder) and the unit tests in `claim.rs`, `process.rs`, `queue.rs`. Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, a memory-mode case in `bugsee/tests/fork_safety.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised, since mounting one needs root; failures are injected by blocking path components and removing the directory mid-run. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index 7e01144..df40239 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -10,75 +10,178 @@ //! and its workers above all β€” and the work items in it (a crashed session to //! recover, a queued report to upload) must each be handled by exactly one of //! them, or the backend receives duplicates. A claim is a small file created -//! with `O_EXCL` that records its owner's pid. +//! with `O_EXCL` (`CREATE_NEW` on Windows) that records its owner as +//! `" "`. //! -//! A claim must never block the work forever if its owner dies, so a claim whose -//! owner is no longer alive is *stale* and is taken over. (An OS file lock would -//! be released on death automatically, but it is per-open-file-description, not -//! per-process, which makes it unreliable across `fork`.) +//! A claim must never block the work forever if its owner dies, so a stale claim +//! is taken over. It is stale when +//! +//! * its owner process is gone, or the pid now belongs to a *different* process +//! (the recorded start time no longer matches: pid reuse, or a container's +//! restarted app being pid 1 again), +//! * it has no readable owner and is older than [`UNOWNED_GRACE`] (its creator +//! died between creating the file and writing it), or +//! * it is older than [`MAX_LEASE`], a backstop for a holder that is alive but +//! wedged, or on a platform that cannot tell processes apart. +//! +//! Taking over is serialised by a short-lived guard file, so two processes that +//! both find the same claim stale cannot each remove the other's fresh claim and +//! both end up holding it. (An OS file lock would release on death automatically, +//! but it is per-open-file-description, not per-process, which makes it unreliable +//! across `fork`.) +//! +//! None of this runs on the crash path; it allocates freely. use std::io::Write; use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; +use crate::process; + /// A claim that has no readable owner is given this long before it is treated as -/// stale: its creator writes the pid a moment after creating the file. +/// stale: its creator writes the owner a moment after creating the file. const UNOWNED_GRACE: Duration = Duration::from_secs(5); +/// No claim is honoured for longer than this, whatever its owner's state. Work +/// items are short (one upload, one recovery), so a claim this old is wedged. +const MAX_LEASE: Duration = Duration::from_secs(30 * 60); + +/// The takeover guard is held for a few syscalls; one older than this was left +/// by a process that died inside the critical section. +const GUARD_TTL: Duration = Duration::from_secs(10); + /// Holds a claim; dropping it releases the claim. #[derive(Debug)] pub struct Claim { path: PathBuf, + /// Exactly what this claim wrote, so release never removes a successor's claim. + content: String, } impl Drop for Claim { fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); + if std::fs::read_to_string(&self.path).is_ok_and(|c| c == self.content) { + let _ = std::fs::remove_file(&self.path); + } } } /// Try to claim `path`. `None` if another live process (or another thread of this -/// one) holds it. +/// one) holds it, or if it cannot be claimed. pub fn try_claim(path: &Path) -> Option { - for _ in 0..2 { - match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(path) - { - Ok(mut file) => { - let _ = write!(file, "{}", std::process::id()); - return Some(Claim { - path: path.to_path_buf(), - }); - } - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { - if !is_stale(path) { - return None; - } - // Take over a claim whose owner is gone, then retry the create. - // Losing the race to another taker just means `create_new` fails - // again and we report the claim as held. - let _ = std::fs::remove_file(path); + let content = owner_record(); + match create(path, &content) { + Create::Done => Some(Claim { + path: path.to_path_buf(), + content, + }), + Create::Exists => take_over(path, content), + // Directory missing / unwritable: nothing can be claimed, and the + // caller's own I/O will fail more usefully than we can here. + Create::Failed => None, + } +} + +enum Create { + Done, + Exists, + Failed, +} + +/// Atomically create `path` holding `content` (one `write`, so the owner is +/// visible as soon as the file is). +fn create(path: &Path, content: &str) -> Create { + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + { + Ok(mut file) => { + let _ = file.write_all(content.as_bytes()); + Create::Done + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Create::Exists, + Err(_) => Create::Failed, + } +} + +/// Replace a stale claim with ours. Only one process at a time may do this: the +/// guard file excludes a second taker between our staleness check and our create, +/// which is exactly the window in which it could otherwise remove the claim we +/// just made. +fn take_over(path: &Path, content: String) -> Option { + let observed = std::fs::read_to_string(path).ok()?; + if !is_stale(path, &observed) { + return None; + } + let guard_path = sibling(path, "takeover"); + match create(&guard_path, "") { + Create::Done => {} + Create::Exists => { + // Held by a peer mid-takeover (it finishes in microseconds, and the + // work is retried on the next drain), or left by one that died. + if age(&guard_path).is_some_and(|a| a > GUARD_TTL) { + let _ = std::fs::remove_file(&guard_path); } - // Directory missing / unwritable: nothing can be claimed, and the - // caller's own I/O will fail more usefully than we can here. - Err(_) => return None, + return None; } + Create::Failed => return None, + } + let _guard = Guard(guard_path); + // Re-check under the guard: a peer may have completed its own takeover since + // we looked, and what is there now is a live claim, not the stale one. + if std::fs::read_to_string(path).ok().as_deref() != Some(observed.as_str()) { + return None; + } + let _ = std::fs::remove_file(path); + match create(path, &content) { + Create::Done => Some(Claim { + path: path.to_path_buf(), + content, + }), + // A fresh creator slipped in between our remove and create: it holds it. + Create::Exists | Create::Failed => None, } - None } -fn is_stale(path: &Path) -> bool { - let owner = std::fs::read_to_string(path).unwrap_or_default(); - match owner.trim().parse::() { - // Our own pid is a live owner: another thread of this process holds it. - Ok(pid) => pid != std::process::id() && !crate::recovery::process_is_alive(pid), - Err(_) => std::fs::metadata(path) - .and_then(|m| m.modified()) - .ok() - .and_then(|t| SystemTime::now().duration_since(t).ok()) - .is_some_and(|age| age > UNOWNED_GRACE), +struct Guard(PathBuf); +impl Drop for Guard { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +fn sibling(path: &Path, ext: &str) -> PathBuf { + let mut s = path.as_os_str().to_os_string(); + s.push("."); + s.push(ext); + PathBuf::from(s) +} + +/// `" "` for this process; the nonce makes every claim's +/// content unique, so "is this still my claim" is a content comparison. +fn owner_record() -> String { + let (pid, start) = + process::parse_token(&process::self_token()).unwrap_or((std::process::id(), None)); + let start = start.map_or_else(|| "-".to_string(), |s| s.to_string()); + format!("{pid} {start} {}", crate::util::random_hex(8)) +} + +fn age(path: &Path) -> Option { + let modified = std::fs::metadata(path).and_then(|m| m.modified()).ok()?; + SystemTime::now().duration_since(modified).ok() +} + +fn is_stale(path: &Path, content: &str) -> bool { + let age = age(path); + if age.is_some_and(|a| a > MAX_LEASE) { + return true; + } + match process::parse_token(content) { + // A live owner with the recorded identity (including another thread of + // this very process) holds it. + Some((pid, start)) => !process::owner_is_alive(pid, start), + None => age.is_some_and(|a| a > UNOWNED_GRACE), } } @@ -92,6 +195,16 @@ mod tests { p } + fn backdate(path: &Path, by: Duration) { + let f = std::fs::OpenOptions::new().write(true).open(path).unwrap(); + f.set_modified(SystemTime::now() - by).unwrap(); + } + + /// This process's own live owner record, as a peer would have written it. + fn live_owner() -> String { + format!("{} nonce", process::self_token()) + } + #[test] fn a_claim_is_exclusive_and_released_on_drop() { let dir = tmp(); @@ -109,25 +222,115 @@ mod tests { // A pid that cannot be alive: far above any real pid range. std::fs::write(&path, "4294967290").unwrap(); assert!(try_claim(&path).is_some(), "stale claim must be reclaimed"); + assert!(!sibling(&path, "takeover").exists(), "guard released"); } #[test] fn a_claim_held_by_a_live_process_is_respected() { let dir = tmp(); let path = dir.join("x.claim"); - // pid 1 is always alive on unix; on other platforms liveness is "dead". - std::fs::write(&path, "1").unwrap(); + std::fs::write(&path, live_owner()).unwrap(); + assert!(try_claim(&path).is_none()); + // Old-format owner (pid only), pid 1: alive on unix. #[cfg(unix)] + { + std::fs::write(&path, "1").unwrap(); + assert!(try_claim(&path).is_none()); + } + } + + #[test] + fn a_recycled_pid_does_not_keep_a_dead_owners_claim_alive() { + let Some(start) = process::start_time(std::process::id()) else { + return; // platform cannot tell processes apart + }; + let dir = tmp(); + let path = dir.join("x.claim"); + // Our own (live) pid, but with a start time that is not ours: the claim + // was written by an earlier process that had this pid. + std::fs::write( + &path, + format!("{} {} stale", std::process::id(), start.wrapping_add(1)), + ) + .unwrap(); + assert!(try_claim(&path).is_some(), "recycled pid must not block"); + } + + #[test] + fn a_freshly_created_unowned_claim_is_not_stolen_but_an_old_one_is() { + let dir = tmp(); + let path = dir.join("x.claim"); + std::fs::write(&path, "").unwrap(); // creator has not written its owner yet assert!(try_claim(&path).is_none()); - #[cfg(not(unix))] + backdate(&path, UNOWNED_GRACE + Duration::from_secs(5)); + assert!(try_claim(&path).is_some(), "creator died before writing"); + } + + #[test] + fn a_wedged_holder_cannot_hold_a_claim_past_the_lease() { + let dir = tmp(); + let path = dir.join("x.claim"); + std::fs::write(&path, live_owner()).unwrap(); + assert!(try_claim(&path).is_none()); + backdate(&path, MAX_LEASE + Duration::from_secs(60)); assert!(try_claim(&path).is_some()); } #[test] - fn a_freshly_created_unowned_claim_is_not_stolen() { + fn releasing_a_claim_never_removes_its_successors() { + let dir = tmp(); + let path = dir.join("x.claim"); + let first = try_claim(&path).unwrap(); + // First is judged wedged and replaced behind its back. + std::fs::write(&path, live_owner()).unwrap(); + drop(first); + assert!(path.exists(), "the successor's claim must survive"); + } + + #[test] + fn an_abandoned_takeover_guard_does_not_block_forever() { let dir = tmp(); let path = dir.join("x.claim"); - std::fs::write(&path, "").unwrap(); // creator has not written its pid yet + std::fs::write(&path, "4294967290").unwrap(); + let guard = sibling(&path, "takeover"); + std::fs::write(&guard, "").unwrap(); + // A taker is (apparently) mid-takeover: back off. assert!(try_claim(&path).is_none()); + // It died there long ago: clear the guard, then succeed. + backdate(&guard, GUARD_TTL + Duration::from_secs(5)); + assert!( + try_claim(&path).is_none(), + "clearing the guard is its own step" + ); + assert!(try_claim(&path).is_some()); + } + + #[test] + fn racing_takers_of_one_stale_claim_yield_exactly_one_holder() { + for _ in 0..200 { + let dir = tmp(); + let path = dir.join("x.claim"); + std::fs::write(&path, "4294967290").unwrap(); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(8)); + let handles: Vec<_> = (0..8) + .map(|_| { + let (path, barrier) = (path.clone(), barrier.clone()); + std::thread::spawn(move || { + barrier.wait(); + let claim = try_claim(&path); + // Keep the claim held until every thread has tried. + barrier.wait(); + claim.is_some() + }) + }) + .collect(); + let holders = handles + .into_iter() + .map(|h| h.join().unwrap()) + .filter(|held| *held) + .count(); + assert_eq!(holders, 1, "exactly one taker may win"); + let _ = std::fs::remove_dir_all(&dir); + } } } diff --git a/crates/bugsee-core/src/lib.rs b/crates/bugsee-core/src/lib.rs index cdbc126..e187ece 100644 --- a/crates/bugsee-core/src/lib.rs +++ b/crates/bugsee-core/src/lib.rs @@ -23,6 +23,7 @@ pub mod errors; pub mod fork; pub mod model; pub mod panic_info; +pub(crate) mod process; pub mod queue; pub mod recovery; pub mod reporting; diff --git a/crates/bugsee-core/src/process.rs b/crates/bugsee-core/src/process.rs new file mode 100644 index 0000000..c37f583 --- /dev/null +++ b/crates/bugsee-core/src/process.rs @@ -0,0 +1,232 @@ +// +// process.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! Process identity and liveness, for deciding whether the owner recorded in a +//! claim or a liveness marker is still the process that wrote it. +//! +//! A bare pid is not an identity: pids are recycled, and in a container the +//! restarted app is routinely pid 1 again. An *owner token* therefore pairs the +//! pid with the process's **start time** (an opaque, stable per-process number), +//! so a recycled pid is told apart from the original owner. Where the platform +//! offers no start time the token carries the pid alone and liveness degrades to +//! "a process with that pid exists". + +use std::sync::OnceLock; + +/// Whether a process with `pid` currently exists. +/// +/// Unix: `kill(pid, 0)`; `0` (permitted) or `EPERM` (exists, not permitted) both +/// mean alive, `ESRCH` means gone. Windows: the process can be opened and has not +/// exited. A pid-reuse false positive is closed by [`owner_is_alive`]'s start +/// time comparison, not here. +#[cfg(unix)] +pub(crate) fn is_alive(pid: u32) -> bool { + if pid == 0 { + return false; + } + // SAFETY: `kill` with signal 0 performs only an error/permission check and + // has no side effects; it is safe to call with any pid value. + let ret = unsafe { libc::kill(pid as libc::pid_t, 0) }; + ret == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) +} + +#[cfg(windows)] +pub(crate) fn is_alive(pid: u32) -> bool { + win::with_process(pid, |handle| { + let mut code = 0u32; + // SAFETY: `handle` is a valid open process handle for the closure's duration. + let ok = unsafe { win::GetExitCodeProcess(handle, &mut code) }; + ok == 0 || code == win::STILL_ACTIVE + }) + // Access denied means the process exists but belongs to someone else. + .unwrap_or_else(|denied| denied) +} + +#[cfg(not(any(unix, windows)))] +pub(crate) fn is_alive(_pid: u32) -> bool { + false +} + +/// An opaque per-process start time for `pid`, or `None` if it cannot be read +/// (platform without support, process gone, or not permitted). Only ever compared +/// for equality against a value read the same way. +#[cfg(target_os = "linux")] +pub(crate) fn start_time(pid: u32) -> Option { + let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + // `pid (comm) state ...`: comm may contain spaces and parens, so split after + // the LAST ')'. `starttime` is field 22; `state` (field 3) is index 0 here. + let rest = &stat[stat.rfind(')')? + 1..]; + rest.split_whitespace().nth(19)?.parse().ok() +} + +#[cfg(target_os = "macos")] +pub(crate) fn start_time(pid: u32) -> Option { + // SAFETY: an all-zero `proc_bsdinfo` is a valid plain-data value, and + // `proc_pidinfo` writes at most `size_of` bytes into it. + let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; + let size = std::mem::size_of::() as libc::c_int; + let n = unsafe { + libc::proc_pidinfo( + pid as libc::c_int, + libc::PROC_PIDTBSDINFO, + 0, + (&mut info as *mut libc::proc_bsdinfo).cast(), + size, + ) + }; + (n == size).then(|| info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec) +} + +#[cfg(windows)] +pub(crate) fn start_time(pid: u32) -> Option { + win::with_process(pid, |handle| { + let mut ft = [win::FileTime::default(); 4]; + // SAFETY: `handle` is valid; the four out-pointers point into `ft`. + let [creation, exit, kernel, user] = &mut ft; + let ok = unsafe { win::GetProcessTimes(handle, creation, exit, kernel, user) }; + (ok != 0).then(|| (u64::from(ft[0].high) << 32) | u64::from(ft[0].low)) + }) + .ok() + .flatten() +} + +#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] +pub(crate) fn start_time(_pid: u32) -> Option { + None +} + +/// This process's owner token, `""` or `" "`. Cached, but keyed +/// by pid: a `fork` child inherits the cache and must not reuse the parent's token. +pub(crate) fn self_token() -> String { + static CACHE: OnceLock<(u32, String)> = OnceLock::new(); + let pid = std::process::id(); + let (cached_pid, token) = CACHE.get_or_init(|| (pid, make_token(pid))); + if *cached_pid == pid { + token.clone() + } else { + make_token(pid) + } +} + +fn make_token(pid: u32) -> String { + match start_time(pid) { + Some(start) => format!("{pid} {start}"), + None => pid.to_string(), + } +} + +/// Parse an owner token (`""` or `" "`; trailing fields, such as +/// a claim's nonce, are ignored). `None` if no pid can be read. +pub(crate) fn parse_token(text: &str) -> Option<(u32, Option)> { + let mut it = text.split_whitespace(); + let pid = it.next()?.parse().ok()?; + let start = it.next().and_then(|s| s.parse().ok()); + Some((pid, start)) +} + +/// Whether the process an owner token names is still alive *and is the same +/// process that wrote the token*. With a recorded start time, a live process +/// whose start time differs is a recycled pid, i.e. the owner is gone. +pub(crate) fn owner_is_alive(pid: u32, recorded_start: Option) -> bool { + if !is_alive(pid) { + return false; + } + match (recorded_start, start_time(pid)) { + (Some(recorded), Some(now)) => recorded == now, + // Cannot compare (old-format token, or start time unreadable): the pid + // exists, so give the owner the benefit of the doubt. + _ => true, + } +} + +#[cfg(windows)] +mod win { + //! The few kernel32 calls needed, declared directly: kernel32 is always + //! linked, so this avoids a dependency for four functions. + use std::ffi::c_void; + + pub type Handle = *mut c_void; + pub const STILL_ACTIVE: u32 = 259; + const PROCESS_QUERY_LIMITED_INFORMATION: u32 = 0x1000; + const ERROR_ACCESS_DENIED: i32 = 5; + + #[repr(C)] + #[derive(Clone, Copy, Default)] + pub struct FileTime { + pub low: u32, + pub high: u32, + } + + extern "system" { + fn OpenProcess(access: u32, inherit: i32, pid: u32) -> Handle; + fn CloseHandle(handle: Handle) -> i32; + pub fn GetExitCodeProcess(handle: Handle, code: *mut u32) -> i32; + pub fn GetProcessTimes( + handle: Handle, + creation: *mut FileTime, + exit: *mut FileTime, + kernel: *mut FileTime, + user: *mut FileTime, + ) -> i32; + } + + /// Run `f` with a query handle to `pid`. `Err(true)` if the process exists but + /// access was denied, `Err(false)` if it cannot be opened (gone). + pub fn with_process(pid: u32, f: impl FnOnce(Handle) -> T) -> Result { + if pid == 0 { + return Err(false); + } + // SAFETY: plain Win32 calls; the handle is closed on every path. + unsafe { + let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); + if handle.is_null() { + let denied = + std::io::Error::last_os_error().raw_os_error() == Some(ERROR_ACCESS_DENIED); + return Err(denied); + } + let out = f(handle); + CloseHandle(handle); + Ok(out) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn own_process_is_alive_with_a_stable_start_time() { + assert!(is_alive(std::process::id())); + let a = start_time(std::process::id()); + assert_eq!(a, start_time(std::process::id())); + #[cfg(any(target_os = "linux", target_os = "macos", windows))] + assert!(a.is_some(), "start time must be readable on this platform"); + } + + #[test] + fn a_pid_with_the_wrong_start_time_is_a_recycled_pid() { + let me = std::process::id(); + if let Some(start) = start_time(me) { + assert!(owner_is_alive(me, Some(start))); + assert!(!owner_is_alive(me, Some(start.wrapping_add(1)))); + } + assert!(owner_is_alive(me, None), "no recorded start: pid alone"); + assert!(!owner_is_alive(4_294_967_290, Some(1))); + } + + #[test] + fn tokens_round_trip() { + assert_eq!(parse_token("12 345"), Some((12, Some(345)))); + assert_eq!(parse_token("12"), Some((12, None))); + assert_eq!(parse_token("12 345 deadbeef"), Some((12, Some(345)))); + assert_eq!(parse_token("junk"), None); + assert_eq!(parse_token(""), None); + let (pid, start) = parse_token(&self_token()).unwrap(); + assert_eq!(pid, std::process::id()); + assert_eq!(start, start_time(pid)); + } +} diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index 2ff3efe..11e5254 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -226,6 +226,11 @@ pub fn blacklist_add(data_dir: &Path, sigs: &[String]) { /// crash-atomic across its multiple files). Without this they accumulate /// unbounded. Call once at startup on the worker thread, before any enqueue, so /// it never races this process's own queue writes. +/// +/// A peer sharing the directory may be mid-`enqueue` right now (sidecars written, +/// bundle not yet renamed into place), and its in-flight files look exactly like +/// orphans. Reaping them would strand the bundle without its `.req`, losing the +/// report, so only files untouched for [`ORPHAN_GRACE`] are reaped. pub fn gc_orphans(data_dir: &Path) { let dir = queue_dir(data_dir); let read = match std::fs::read_dir(&dir) { @@ -245,15 +250,30 @@ pub fn gc_orphans(data_dir: &Path) { .or_else(|| name.strip_suffix(".meta")) .or_else(|| name.strip_suffix(".endpoint")) .or_else(|| name.strip_suffix(".claim")) + .or_else(|| name.strip_suffix(".claim.takeover")) .or_else(|| name.strip_suffix(".tmp")); if let Some(bundle_name) = bundle_name { - if !dir.join(bundle_name).exists() { + if !dir.join(bundle_name).exists() && untouched_for(&path, ORPHAN_GRACE) { let _ = std::fs::remove_file(&path); } } } } +/// How long an orphan-looking queue file must sit untouched before it is reaped +/// (see [`gc_orphans`]); far longer than any `enqueue` takes. +const ORPHAN_GRACE: std::time::Duration = std::time::Duration::from_secs(60); + +/// Whether `path` was last modified at least `grace` ago. Unknown or future-dated +/// times count as recent: the conservative answer never disturbs a live peer. +fn untouched_for(path: &Path, grace: std::time::Duration) -> bool { + std::fs::metadata(path) + .and_then(|m| m.modified()) + .ok() + .and_then(|t| t.elapsed().ok()) + .is_some_and(|age| age >= grace) +} + /// Whether any of `sigs` is locally blacklisted (report should be suppressed). pub fn any_blacklisted(data_dir: &Path, sigs: &[String]) -> bool { if sigs.is_empty() { @@ -262,3 +282,30 @@ pub fn any_blacklisted(data_dir: &Path, sigs: &[String]) -> bool { let set = load_blacklist(data_dir); sigs.iter().any(|s| set.contains(s)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gc_spares_a_peers_in_flight_enqueue_and_reaps_old_orphans() { + let data = std::env::temp_dir().join(format!("bugsee-qgc-{}", crate::util::random_hex(8))); + let dir = queue_dir(&data); + std::fs::create_dir_all(&dir).unwrap(); + // A peer mid-enqueue: sidecars written, bundle not yet renamed into place. + let fresh = dir.join("a.bundle.zip.req"); + std::fs::write(&fresh, "{}").unwrap(); + let old = dir.join("b.bundle.zip.req"); + std::fs::write(&old, "{}").unwrap(); + let f = std::fs::OpenOptions::new().write(true).open(&old).unwrap(); + f.set_modified(std::time::SystemTime::now() - ORPHAN_GRACE * 2) + .unwrap(); + drop(f); + + gc_orphans(&data); + + assert!(fresh.exists(), "an in-flight enqueue must not be reaped"); + assert!(!old.exists(), "an old orphan must be reaped"); + let _ = std::fs::remove_dir_all(&data); + } +} diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index ca21356..0262252 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -97,11 +97,13 @@ pub fn find_pending(data_dir: &Path, current_generation: u64) -> Vec() { + } else if let Some((pid, start)) = crate::process::parse_token(owner) { // Skip a marker still owned by a live *other* process. Our own pid is // never skipped (a live process cannot share our pid, so a marker - // bearing it is a dead prior incarnation β€” recover it). - if pid != self_pid && process_is_alive(pid) { + // bearing it is a dead prior incarnation β€” recover it). The marker + // may carry the owner's start time, which tells a recycled pid (a + // dead owner whose pid something else now uses) from the owner. + if pid != self_pid && crate::process::owner_is_alive(pid, start) { continue; } } @@ -146,30 +148,6 @@ fn marker_is_fresh(marker: &Path) -> bool { .unwrap_or(true) } -/// Whether a process with `pid` currently exists. -/// -/// Uses `kill(pid, 0)`, which sends no signal and only probes existence: `0` -/// (permitted) or `EPERM` (exists, not permitted) both mean *alive*; `ESRCH` -/// means gone. A rare pid-reuse false-positive can defer recovering a real crash -/// to a later launch β€” acceptable vs. corrupting a running peer's live session. -#[cfg(unix)] -pub(crate) fn process_is_alive(pid: u32) -> bool { - if pid == 0 { - return false; - } - // SAFETY: `kill` with signal 0 performs only an error/permission check and - // has no side effects; it is safe to call with any pid value. - let ret = unsafe { libc::kill(pid as libc::pid_t, 0) }; - ret == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) -} - -/// No portable liveness probe off-unix: treat as dead so recovery still proceeds -/// (single-process is the common case; a live shared-dir peer is a unix concern). -#[cfg(not(unix))] -pub(crate) fn process_is_alive(_pid: u32) -> bool { - false -} - /// Build the crash report for a pending session. Precedence: a *fresh* aborting /// panic (`SIGABRT` + a recent panic snapshot) correlates into one managed event; /// any other native signal yields the thin native variant; a lone panic snapshot @@ -348,13 +326,24 @@ pub fn bump_attempts(data_dir: &Path, pending: &PendingSession) -> u32 { /// Claim a pending session for recovery, so that of several processes sharing the /// data directory exactly one recovers it. The claim lives beside the liveness -/// marker and is released when the returned guard drops. +/// marker and is released when the returned guard drops. `None` if a peer holds +/// it or has already recovered the session. pub fn claim_pending(data_dir: &Path, pending: &PendingSession) -> Option { - crate::claim::try_claim( + let claim = crate::claim::try_claim( &data_dir .join("sessions") .join(format!("{}.claim", pending.generation)), - ) + )?; + // The listing that produced `pending` may be stale: a peer that held this + // claim before us can have recovered and discarded the session in between + // (its claim released, the marker gone). Recovering it now would deliver a + // phantom second report for it. The marker is removed first by `discard`, so + // its absence, re-checked while holding the claim, is conclusive. + data_dir + .join("sessions") + .join(format!("{}.alive", pending.generation)) + .exists() + .then_some(claim) } /// Reap leftover on-disk state from generations that no longer have a liveness @@ -388,6 +377,16 @@ pub fn gc_orphans(data_dir: &Path, current_generation: u64) { } } } + // A claim left by a holder that died after the session was discarded. + // Taking it (which succeeds only if it is stale) and dropping it + // removes it; a live holder's claim is left alone. + if let Some(gen_str) = name.strip_suffix(".claim") { + if let Ok(generation) = gen_str.parse::() { + if generation < current_generation && !alive(generation) { + drop(crate::claim::try_claim(&entry.path())); + } + } + } } } } diff --git a/crates/bugsee-core/src/session.rs b/crates/bugsee-core/src/session.rs index b4751b0..5a81692 100644 --- a/crates/bugsee-core/src/session.rs +++ b/crates/bugsee-core/src/session.rs @@ -46,7 +46,9 @@ impl Session { .open(&marker) { Ok(mut f) => { - let _ = write!(f, "{}", std::process::id()); + // " ": the start time lets recovery tell this + // process from a later one that reuses the pid. + let _ = write!(f, "{}", crate::process::self_token()); break candidate; } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { diff --git a/crates/bugsee-core/tests/multiprocess.rs b/crates/bugsee-core/tests/multiprocess.rs new file mode 100644 index 0000000..348f50c --- /dev/null +++ b/crates/bugsee-core/tests/multiprocess.rs @@ -0,0 +1,245 @@ +// +// multiprocess.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! Several processes sharing one data directory (a prefork server's workers, a CLI +//! run beside a daemon): every pending report is delivered exactly once, and a +//! holder that dies mid-work never blocks the others (issue #14). +//! +//! The "processes" are real: this test binary re-executes itself in a child role +//! selected by environment variables. + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use bugsee_core::transport::{Transport, TransportError}; +use bugsee_core::{queue, recovery, AssembledReport, Recorder, RecorderConfig}; + +const ROLE: &str = "BUGSEE_MP_ROLE"; +const DIR: &str = "BUGSEE_MP_DIR"; +const LOG: &str = "BUGSEE_MP_LOG"; + +/// A transport that records each upload by appending a line to a shared log file +/// (one `write` under `O_APPEND`, atomic across processes). +struct LogTransport { + log: PathBuf, + lock: Mutex<()>, +} +impl Transport for LogTransport { + fn register_session(&self, _: &str, _: &[u8]) -> Result { + Ok("t".into()) + } + fn create_issue(&self, _: &str, _: Option<&str>, _: &[u8]) -> Result { + Ok("endpoint".into()) + } + fn upload_bundle(&self, _: &str, zip: &[u8]) -> Result<(), TransportError> { + let _g = self.lock.lock().unwrap(); + // Give a racing peer a real window to double-deliver. + std::thread::sleep(Duration::from_millis(20)); + let mut f = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&self.log) + .unwrap(); + // Queued test items carry a readable name; recovered crashes are binary + // ZIPs, logged as a tag (each recovered session yields one). + let id = match std::str::from_utf8(zip) { + Ok(text) if text.starts_with("queued-") => text.to_string(), + _ => "recovered-crash".to_string(), + }; + // One `write` call: `writeln!` would split text and newline. + f.write_all(format!("{id}\n").as_bytes()).unwrap(); + Ok(()) + } +} + +/// Child entry point. A no-op in a normal test run. +#[test] +fn child_entry() { + let Ok(role) = std::env::var(ROLE) else { + return; + }; + let dir = PathBuf::from(std::env::var(DIR).unwrap()); + match role.as_str() { + "worker" => { + // Spin until the parent releases every worker at once. + let go = dir.join("GO"); + while !go.exists() { + std::thread::sleep(Duration::from_millis(1)); + } + let transport = Arc::new(LogTransport { + log: PathBuf::from(std::env::var(LOG).unwrap()), + lock: Mutex::new(()), + }); + let rec = Recorder::launch(RecorderConfig::new(&dir, "tok"), transport).unwrap(); + rec.flush(Duration::from_secs(3)); + } + "holder" => { + // Hold every queue item and every recoverable session, then wait to + // be killed. + let mut held = Vec::new(); + for q in queue::list_pending(&dir) { + held.push(queue::try_claim(&q).expect("claim queue item")); + } + for p in recovery::find_pending(&dir, u64::MAX) { + held.push(recovery::claim_pending(&dir, &p).expect("claim session")); + } + // Signal readiness through the filesystem: libtest owns stdout. + std::fs::write(dir.join("READY.tmp"), held.len().to_string()).unwrap(); + std::fs::rename(dir.join("READY.tmp"), dir.join("READY")).unwrap(); + std::thread::sleep(Duration::from_secs(120)); + } + other => panic!("unknown role {other}"), + } +} + +struct TempDir(PathBuf); +impl TempDir { + fn new() -> Self { + let p = + std::env::temp_dir().join(format!("bugsee-mp-{}", bugsee_core::util::random_hex(8))); + std::fs::create_dir_all(&p).unwrap(); + TempDir(p) + } +} +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn spawn(role: &str, dir: &Path, log: &Path) -> Child { + Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "child_entry", "--nocapture"]) + .env(ROLE, role) + .env(DIR, dir) + .env(LOG, log) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap() +} + +/// `queued` reports in the queue and `crashed` dead sessions awaiting recovery. +fn seed(dir: &Path, queued: usize, crashed: u64) { + for i in 0..queued { + queue::enqueue( + dir, + &AssembledReport { + bundle_name: format!("{}.bundle.zip", bugsee_core::util::random_hex(10)), + zip: format!("queued-{i}").into_bytes(), + request_json: br#"{"signatures":[]}"#.to_vec(), + }, + ) + .unwrap(); + } + let sessions = dir.join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + for g in 1..=crashed { + // "0" is the portable "owner gone" sentinel. + std::fs::write(sessions.join(format!("{g}.alive")), "0").unwrap(); + std::fs::create_dir_all(dir.join("parts").join(g.to_string())).unwrap(); + } + std::fs::write(dir.join("gen"), crashed.to_string()).unwrap(); +} + +fn delivered(log: &Path) -> Vec { + std::fs::read_to_string(log) + .map(|s| s.lines().map(String::from).collect()) + .unwrap_or_default() +} + +/// Every file under `dir`, relative, for failure messages. +fn tree(dir: &Path) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + for e in std::fs::read_dir(&d).into_iter().flatten().flatten() { + let p = e.path(); + if p.is_dir() { + stack.push(p); + } else { + out.push(p.strip_prefix(dir).unwrap().display().to_string()); + } + } + } + out.sort(); + out +} + +fn run_workers(dir: &Path, log: &Path, n: usize) { + let kids: Vec = (0..n).map(|_| spawn("worker", dir, log)).collect(); + std::thread::sleep(Duration::from_millis(300)); // all spinning on GO + std::fs::write(dir.join("GO"), "").unwrap(); + for mut k in kids { + let _ = k.wait(); + } +} + +#[test] +fn racing_launchers_deliver_each_pending_report_exactly_once() { + // Repeated: the double-delivery windows are narrow, so one round can pass by luck. + for round in 0..8 { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + std::fs::create_dir_all(&dir).unwrap(); + let log = tmp.0.join("delivered.log"); + seed(&dir, 6, 4); + + run_workers(&dir, &log, 5); + + let got = delivered(&log); + // 6 queued reports + 4 recovered crashes, no more, no fewer. + assert_eq!( + got.len(), + 10, + "round {round}, delivery log: {got:?}, left on disk: {:?}", + tree(&dir) + ); + for i in 0..6 { + let n = got.iter().filter(|l| **l == format!("queued-{i}")).count(); + assert_eq!(n, 1, "round {round}: queued-{i} delivered {n} times"); + } + assert_eq!(got.iter().filter(|l| *l == "recovered-crash").count(), 4); + assert!(queue::list_pending(&dir).is_empty()); + } +} + +#[test] +fn a_holder_killed_mid_work_does_not_block_the_others() { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + std::fs::create_dir_all(&dir).unwrap(); + let log = tmp.0.join("delivered.log"); + seed(&dir, 4, 3); + + let mut holder = spawn("holder", &dir, &log); + let ready = dir.join("READY"); + let deadline = Instant::now() + Duration::from_secs(20); + while !ready.exists() { + assert!(Instant::now() < deadline, "holder never reported ready"); + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(std::fs::read_to_string(&ready).unwrap(), "7"); + + // While the holder lives, nothing it holds is delivered by anyone else. + run_workers(&dir, &log, 2); + assert!(delivered(&log).is_empty(), "live holder's work was taken"); + std::fs::remove_file(dir.join("GO")).unwrap(); + + // SIGKILL / TerminateProcess: no destructor runs, its claim files survive. + holder.kill().unwrap(); + let _ = holder.wait(); + + let deadline = Instant::now() + Duration::from_secs(20); + run_workers(&dir, &log, 3); + while delivered(&log).len() < 7 && Instant::now() < deadline { + run_workers(&dir, &log, 1); + } + assert_eq!(delivered(&log).len(), 7, "log: {:?}", delivered(&log)); +} From faf434472ca6e9d9974b25be6faf95ff97297a1c Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:27:49 +0500 Subject: [PATCH 07/52] feat: define the shutdown/flush contract; opt-in SIGTERM/SIGINT flush MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Document per exit path what is persisted, delivered and recovered (Bugsee::stop docs, DESIGN.md section 8). - Bugsee::shutdown(timeout) = flush then stop. A stop during a flush now aborts the flush so the clean-stop acknowledgement bound holds. - Bugsee::install_termination_flush: opt-in, chaining SIGTERM/SIGINT hook (self-pipe; flush on a normal thread; re-raises with the default disposition so the exit status is unchanged) and a Windows console control handler. - LaunchOptions::flush_at_exit: atexit-based clean stop for deliberate process::exit. - Subprocess tests for every exit path. Closes #8 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 16 +- crates/bugsee-core/src/runtime.rs | 11 +- crates/bugsee/src/api.rs | 112 ++++++++- crates/bugsee/src/lib.rs | 1 + crates/bugsee/src/options.rs | 25 ++ crates/bugsee/src/termination.rs | 365 +++++++++++++++++++++++++++ crates/bugsee/tests/shutdown.rs | 403 ++++++++++++++++++++++++++++++ 7 files changed, 930 insertions(+), 3 deletions(-) create mode 100644 crates/bugsee/src/termination.rs create mode 100644 crates/bugsee/tests/shutdown.rs diff --git a/DESIGN.md b/DESIGN.md index f2f03f6..c411abf 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -201,6 +201,20 @@ Crash recovery needs the data directory to **outlive the process**: on a contain Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, a memory-mode case in `bugsee/tests/fork_safety.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised, since mounting one needs root; failures are injected by blocking path components and removing the directory mid-run. +**Shutdown and flush contract (issue #8).** `flush(timeout)` persists everything captured before the call and tries to upload every queued report; it returns `true` only if the outbound queue is empty, leaves the SDK running, and is idempotent. `stop()` (and the launch guard's `Drop`) persists, then ends the session by removing the liveness marker; it never uploads, so an undelivered report stays in the durable queue and goes out next launch (disk mode; in memory mode `flush` is the only guarantee). `shutdown(timeout)` is `flush` then `stop`. A `stop` arriving during a flush aborts the flush instead of waiting out its deadline, so the 2 s acknowledgement bound holds and the session still ends cleanly. The marker is intentionally left when a thread does not acknowledge in time, so a wedged shutdown is recovered, not lost. None of it is async-signal-safe; in a `fork` child it acts on the child's own revived recorder. + +| Exit path | Marker | Reports | Next launch | +|---|---|---|---| +| return from `main` / `stop` / `shutdown` | removed | persisted, queue intact | no abnormal-exit; queue delivered once | +| `process::exit` / `exit()` | left (default) | as already written | one abnormal-exit report; queue delivered once | +| same, with `LaunchOptions::flush_at_exit` | removed by an `atexit` handler | persisted | as the first row | +| `SIGTERM`/`SIGINT` + `Bugsee::install_termination_flush` | removed if the signal would have terminated the process | flushed, bounded | as the first row; host handler still called | +| `SIGTERM`/`SIGINT` without the hook, `SIGKILL`, OOM, power loss | left | records appended as drained survive | one abnormal-exit report | + +*Decision: `exit` without `stop` stays an abnormal exit.* From inside the process an `exit` is indistinguishable from a runtime tearing down after a fault the host swallowed, so the default errs towards reporting; hosts that exit on purpose opt in with `flush_at_exit` (or call `shutdown`). *Decision: the signal hook is opt-in and chaining.* The handler only writes the signal number to a non-blocking self-pipe and then calls the previously installed handler; a watcher thread does the flush. With a default previous disposition it runs `shutdown`, restores `SIG_DFL` and re-raises, so the exit status is unchanged; a second signal during the shutdown terminates at once. With a host handler or `SIG_IGN` it flushes only and the host decides. A `fork` child inherits the handler but not the watcher: it chains/terminates as if the hook were absent until `install_termination_flush` is called again there. Windows: a console control handler (Ctrl+C/Break flush only, since an earlier handler may swallow them; close/logoff/shutdown flush and stop) that returns `FALSE` to pass the event on; service-control stop is not a console event and needs a `shutdown` call from the host's stop handler. **Not compiled or run on Windows** (a macOS cross-check stops at the C dependencies of `zstd-sys`/`ring`); the Windows half was reviewed by reading only. + +Tests: `bugsee/tests/shutdown.rs` (a real child process per exit path, then a relaunch asserting exactly-once delivery and the abnormal-exit count). + **Generations:** each launch bumps `gen`; next-launch recovery reads the **previous** gen's parts before advancing. **Snapshot (no seal, no copy):** pin `[start,end]` (`end`=snapshot instant, `start`=`endβˆ’window`) + **hard-link current parts including the active one** into the report dir (`PartStore::snapshot_into`, after a `flush`). At export, read linked parts and include only entries with `start ≀ timestamp ≀ end` (monotonic order β†’ early-stop at the tail; the self-terminating `RecordIter` cleanly ignores any torn trailing write on the active part). Bounds become `manifest.time.start/end`. Cost: 2 timestamps + O(#parts) `link` syscalls. Fallback to copy on FAT/exFAT/cross-volume (capability-probed once). Reflink (APFS `clonefile` / Linux `FICLONE`) a noted future tier. @@ -269,7 +283,7 @@ That last case previously produced **no report whatsoever** β€” the SDK shut dow | Report/upload worker | 1 | `reports/`, `queue/` | export β†’ bundle (zstd-93) β†’ 3-step upload β†’ retry | | Crash-time context | any thread | β€” | panic observer / native handler: touch only preallocated `PanicSnapshot` + minidump | -**Crash-time isolation rule:** the fault path never enqueues/allocates/locks β€” it writes only to fixed preallocated storage + a file-based crash marker (worker may be dead/frozen). **Back-pressure:** the depth-capped channel drops the newest entry when full (never blocks a producer); **reports** are bounded by a separate in-flight counter (heavier per message, so a report storm drops-newest **non-crash** reports over the cap while never dropping a crash); the worker holds no large in-RAM buffer (it frames one record at a time and `write_all`s it), so on-disk growth is bounded by `max_bytes`/`max_window` eviction rather than RSS. **Async-optional:** capture worker is always a `std` thread; only the uploader can become a task on the host runtime (`async` feature). **Shutdown:** guard `Drop` β†’ `flush(timeout)`. +**Crash-time isolation rule:** the fault path never enqueues/allocates/locks β€” it writes only to fixed preallocated storage + a file-based crash marker (worker may be dead/frozen). **Back-pressure:** the depth-capped channel drops the newest entry when full (never blocks a producer); **reports** are bounded by a separate in-flight counter (heavier per message, so a report storm drops-newest **non-crash** reports over the cap while never dropping a crash); the worker holds no large in-RAM buffer (it frames one record at a time and `write_all`s it), so on-disk growth is bounded by `max_bytes`/`max_window` eviction rather than RSS. **Async-optional:** capture worker is always a `std` thread; only the uploader can become a task on the host runtime (`async` feature). **Shutdown:** guard `Drop` β†’ persist + clean stop (see Β§8, "Shutdown and flush contract"); `Bugsee::shutdown(timeout)` = flush then stop. ## 12. Crash-time safety, best-effort flush & edge cases diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 232e8f8..3f04ad0 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -243,6 +243,10 @@ struct Shared { max_report_queued: usize, /// Whether telemetry sampling is paused (mirrors the facade's pause state). paused: AtomicBool, + /// Set when the recorder is being stopped: a flush in progress gives up at + /// once, so the stop is acknowledged inside its bound (and the session can + /// end cleanly) instead of waiting out the flush's whole deadline. + stopping: AtomicBool, /// Reports that could not be queued on disk, delivered from memory. An entry /// stays here until its delivery is settled, so the length is the real backlog. mem_queue: ForkLock>, @@ -350,6 +354,7 @@ impl Recorder { report_queued: AtomicUsize::new(0), max_report_queued: DEFAULT_MAX_REPORT_QUEUED, paused: AtomicBool::new(false), + stopping: AtomicBool::new(false), mem_queue: ForkLock::new(Vec::new()), next_mem_id: AtomicU64::new(0), degraded: AtomicBool::new(false), @@ -626,6 +631,7 @@ impl Drop for Recorder { // A thread wedged in a host callback with no timeout is DETACHED (its // JoinHandle dropped un-joined) rather than hanging the stopping thread // forever. (Called only after the facade released its global lock.) + self.shared.stopping.store(true, Ordering::SeqCst); let (tx, rx) = channel(); let worker_acked = self.tx.send(Msg::Stop(tx)).is_ok() && rx.recv_timeout(Duration::from_secs(2)).is_ok(); @@ -1706,6 +1712,9 @@ fn drain_until_empty( ) -> bool { let start = Instant::now(); loop { + if shared.stopping.load(Ordering::SeqCst) { + return false; // stop wins; what is queued stays queued + } let now = epoch_ms(); drain_memory(shared, backoff_base, true, false, Some(start + deadline)); for &data_dir in dirs { @@ -1727,7 +1736,7 @@ fn drain_until_empty( now, false, ); - if start.elapsed() >= deadline { + if start.elapsed() >= deadline || shared.stopping.load(Ordering::SeqCst) { break; } } diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index a693e7f..5b8bfa9 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -228,6 +228,8 @@ impl Bugsee { bugsee_panic::set_report_from_hook(options.report_panics_from_hook); } + register_exit_flush(options.flush_at_exit); + Ok(LaunchGuard { _private: () }) } @@ -256,7 +258,30 @@ impl Bugsee { }) } - /// Stop the SDK and flush pending work (dropping the recorder joins the worker). + /// Stop the SDK: persist everything captured so far, then end the session + /// cleanly. Idempotent, and a no-op when not launched. Never uploads. + /// + /// # The shutdown contract + /// + /// | Exit path | Captured data and queued reports | Next launch | + /// |---|---|---| + /// | `stop`/`shutdown`, or the launch guard dropped (return from `main`) | Everything captured is persisted; a report not yet uploaded stays in the durable queue | No abnormal-exit report; queued reports are delivered once | + /// | `exit()` / `process::exit` | As above only with [`LaunchOptions::flush_at_exit`]; otherwise whatever the worker had already written | With the option: as above. Without: one abnormal-exit report | + /// | `SIGTERM`/`SIGINT` with [`Bugsee::install_termination_flush`] | Flushed (bounded), then a clean stop if the signal would have terminated the process | As the first row. The host's own handler is still called | + /// | `SIGTERM`/`SIGINT` without the hook, `SIGKILL`, OOM kill, power loss | Records are appended as the worker drains them, so completed records and queued reports survive; the last few in flight may not | One abnormal-exit report, carrying the recovered window | + /// | Fatal signal, aborting panic, unwinding panic out of `main` | See the crash capture docs | One crash report | + /// + /// Nothing is reported twice: a stopped session's marker is gone, and a + /// recovered one is consumed when its report is queued. In memory mode + /// (see [`StorageMode`]) nothing survives the process, so `flush` before + /// exiting is the only delivery guarantee. + /// + /// Bounds: `stop` waits at most 2 s each for the capture worker and the + /// uploader to acknowledge. If one does not (a host callback wedged), the + /// marker is deliberately left, so the session is recovered rather than + /// lost. It is not async-signal-safe and must not be called from a signal + /// handler (use [`Bugsee::install_termination_flush`]). In a `fork` child + /// it acts on that child's own recorder. pub fn stop() { check_fork(); #[cfg(feature = "native")] @@ -272,6 +297,56 @@ impl Bugsee { drop(recorder); } + /// Flush, then stop: the one call to make before a deliberate exit. + /// + /// Waits up to `timeout` for captured work to be persisted and the upload + /// queue to drain ([`Bugsee::flush`]), then [`Bugsee::stop`]s, which ends + /// the session cleanly whatever the flush returned. Returns the flush + /// result: `true` if every report was delivered. `false` is not a loss β€” + /// undelivered reports stay in the durable queue for the next launch (in + /// disk mode) β€” and also covers "not launched". Idempotent. Total time is + /// bounded by `timeout` plus the bounds documented on `stop`. + pub fn shutdown(timeout: Duration) -> bool { + let flushed = Self::flush(timeout); + Self::stop(); + flushed + } + + /// Opt in to flushing on `SIGTERM`/`SIGINT` (unix) or console control + /// events (Windows: Ctrl+C, Ctrl+Break, close, logoff, shutdown). + /// + /// Containers and process managers stop a service with `SIGTERM` and kill it + /// after a grace period; without this the process dies with the signal's + /// default action and the next launch reports an abnormal termination. It is + /// never installed by default and it never takes the signal from you: + /// + /// - **Unix.** The handler only writes to a self-pipe, then calls whatever + /// handler was installed before it. A thread does the flush. If the + /// previous disposition was the default (terminate), it runs + /// [`Bugsee::shutdown`] with `timeout`, restores the default and re-raises + /// the signal, so the exit status is unchanged and only delayed; a second + /// signal meanwhile terminates at once. If the host had its own handler (or + /// ignores the signal) the SDK only flushes, and the host decides when to + /// exit and ends the session by dropping the guard / calling `stop`. Call + /// this **after** installing your own handler, or make yours chain: a + /// handler installed later that does not chain replaces this one. + /// Signals covered: `SIGTERM`, `SIGINT`. A `fork` child inherits the + /// handler but not its thread; it terminates by default as usual, and + /// calling this again in the child re-arms it there. + /// - **Windows.** A console control handler that flushes (and, for close, + /// logoff and shutdown, which end the process regardless, also stops) + /// then returns `FALSE` so the next handler still runs. Windows gives + /// close/logoff/shutdown handlers only a few seconds: use a short + /// `timeout`. A service-control stop is not a console event and is not + /// covered; call [`Bugsee::shutdown`] from your stop handler. + /// + /// Idempotent; a second call only updates `timeout`. Works before `launch` + /// (it flushes nothing until there is something to flush). Choose `timeout` + /// below your orchestrator's grace period (Kubernetes defaults to 30 s). + pub fn install_termination_flush(timeout: Duration) -> io::Result<()> { + crate::termination::install(timeout) + } + /// Whether the SDK is launched and not paused. pub fn is_active() -> bool { check_fork(); @@ -455,6 +530,13 @@ impl Bugsee { } /// Block until pending work drains, or `timeout` elapses. + /// + /// Persists everything captured before the call and tries to upload every + /// queued report; returns `true` only if the outbound queue is then empty + /// (`false` on timeout, a failing transport, or when not launched). + /// Idempotent, callable from any thread, and it leaves the SDK running. A + /// `false` loses nothing in disk mode: the reports stay queued. Not + /// async-signal-safe. See [`Bugsee::stop`] for the per-exit-path contract. pub fn flush(timeout: Duration) -> bool { // Off-lock: flush blocks; must not hold the mutex across the wait. Self::handle().map(|h| h.flush(timeout)).unwrap_or(false) @@ -481,6 +563,34 @@ impl Bugsee { } } +/// Exit-time flush timeout in milliseconds, `0` when not requested. +static EXIT_FLUSH_MS: AtomicU64 = AtomicU64::new(0); + +extern "C" { + fn atexit(callback: extern "C" fn()) -> std::os::raw::c_int; +} + +extern "C" fn flush_at_exit() { + let ms = EXIT_FLUSH_MS.load(Ordering::SeqCst); + if ms > 0 { + let _ = std::panic::catch_unwind(|| Bugsee::shutdown(Duration::from_millis(ms))); + } +} + +/// Arm (or, with `None`, disarm) the `atexit` handler. libc cannot unregister +/// it, so it is registered once per process and consults the timeout. +fn register_exit_flush(timeout: Option) { + static REGISTERED: AtomicBool = AtomicBool::new(false); + let ms = timeout.map_or(0, |t| t.as_millis().clamp(1, u64::MAX as u128) as u64); + EXIT_FLUSH_MS.store(ms, Ordering::SeqCst); + if ms > 0 && !REGISTERED.swap(true, Ordering::SeqCst) { + // SAFETY: registering a plain `extern "C"` function. + unsafe { + atexit(flush_at_exit); + } + } +} + /// If this process was `fork()`ed since the SDK last looked, rebuild what the /// fork did not carry over. Cheap on the hot path: one atomic load. /// diff --git a/crates/bugsee/src/lib.rs b/crates/bugsee/src/lib.rs index ce610ef..1d84b57 100644 --- a/crates/bugsee/src/lib.rs +++ b/crates/bugsee/src/lib.rs @@ -32,6 +32,7 @@ mod perf; mod report; #[cfg(feature = "telemetry")] mod telemetry; +mod termination; #[cfg(feature = "net")] pub mod transport; diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index e4de340..03b057b 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -39,6 +39,7 @@ pub struct LaunchOptions { pub(crate) before_send: Option, pub(crate) before_breadcrumb: Option, pub(crate) sample_rate: f64, + pub(crate) flush_at_exit: Option, } impl LaunchOptions { @@ -63,6 +64,7 @@ impl LaunchOptions { before_send: None, before_breadcrumb: None, sample_rate: 1.0, + flush_at_exit: None, } } @@ -122,6 +124,29 @@ impl LaunchOptions { self } + /// Treat `std::process::exit` (and any other libc `exit`) as a deliberate + /// stop: at exit, flush for up to `timeout` and end the session cleanly + /// (default: off). + /// + /// Without it, a process that calls `exit` while the SDK is still launched + /// leaves its liveness marker behind and is reported on the next launch as + /// an abnormal termination, because nothing in the process can tell that + /// `exit` from a death. Enable this when your application ends through + /// `exit` on purpose. Returning from `main` needs no option: dropping the + /// launch guard already stops cleanly. + /// + /// It registers a libc `atexit` handler, so it does not run on `abort`, a + /// fatal signal or `_exit` β€” those stay abnormal terminations, as they + /// should. Caveat: a panic that unwinds out of `main` while the launch + /// guard has been leaked (`mem::forget`, a `static`) would be treated as a + /// deliberate exit; keep the guard as a local in `main`. + /// + /// See [`crate::Bugsee::stop`] for exactly what is guaranteed. + pub fn flush_at_exit(mut self, timeout: Duration) -> Self { + self.flush_at_exit = Some(timeout); + self + } + /// Set the on-disk data directory, taking precedence over everything else. /// /// The directory holds the capture window, the liveness marker and the retry diff --git a/crates/bugsee/src/termination.rs b/crates/bugsee/src/termination.rs new file mode 100644 index 0000000..476aa7a --- /dev/null +++ b/crates/bugsee/src/termination.rs @@ -0,0 +1,365 @@ +// +// termination.rs +// bugsee +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// + +//! Opt-in flush on `SIGTERM`/`SIGINT` (unix) or console control events +//! (Windows). See [`crate::Bugsee::install_termination_flush`]. +//! +//! # Unix +//! +//! The signal handler does only async-signal-safe things: it writes one byte +//! (the signal number) to a self-pipe, then **chains to whatever handler was +//! installed before ours**. A watcher thread reads the pipe and does the real +//! work on an ordinary thread, where taking locks, allocating and joining +//! threads are all fine: +//! +//! - the previous disposition was the default (terminate): `shutdown` (flush, +//! then a clean stop, because the exit is deliberate), restore the default +//! disposition and re-raise the signal, so the process ends exactly as it +//! would have β€” same exit status β€” only later; +//! - the previous disposition was a host handler, or "ignore": flush only. The +//! host decides whether and how to exit; it ends the session cleanly by +//! dropping the launch guard (or calling `stop`) on its way out. +//! +//! A second signal while the default-disposition shutdown is still running +//! terminates immediately: an operator pressing Ctrl+C twice is not made to +//! wait out the flush timeout. +//! +//! # Windows +//! +//! A console control handler. Windows runs it on a thread of its own, so it +//! flushes in place, then returns `FALSE` so the next handler (the host's, or +//! the default) still runs. + +use std::io; +use std::time::Duration; + +/// Install the hook, or refresh its timeout if it is already installed. +pub(crate) fn install(timeout: Duration) -> io::Result<()> { + imp::install(timeout) +} + +#[cfg(unix)] +mod imp { + use std::cell::UnsafeCell; + use std::io; + use std::mem::MaybeUninit; + use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, Ordering}; + use std::sync::Mutex; + use std::time::Duration; + + use libc::{c_int, c_void}; + + use crate::Bugsee; + + const SIGNALS: [c_int; 2] = [libc::SIGTERM, libc::SIGINT]; + + /// The dispositions in force before ours, indexed like [`SIGNALS`]. + /// + /// Written only by `install`, before the `sigaction` that arms the handler, + /// and read by the handler afterwards β€” the same discipline the native crash + /// handler uses for its table. + struct Previous(UnsafeCell<[MaybeUninit; 2]>); + // SAFETY: see above; there is no concurrent write once the handler is armed. + unsafe impl Sync for Previous {} + static PREVIOUS: Previous = Previous(UnsafeCell::new([MaybeUninit::uninit(); 2])); + + /// Write end of the self-pipe; `-1` until installed. + static PIPE_W: AtomicI32 = AtomicI32::new(-1); + /// The process that owns the watcher thread. A `fork` child inherits the + /// handler but not the thread. + static OWNER_PID: AtomicU32 = AtomicU32::new(0); + static TIMEOUT_MS: AtomicU64 = AtomicU64::new(0); + /// Set by the watcher once a terminating shutdown has begun. + static SHUTTING_DOWN: AtomicBool = AtomicBool::new(false); + /// Serialises `install` (never taken by the handler). + static INSTALL: Mutex = Mutex::new(false); + + fn index(sig: c_int) -> Option { + SIGNALS.iter().position(|s| *s == sig) + } + + /// The disposition that was in force before ours for `SIGNALS[idx]`. + /// + /// # Safety + /// The table must have been filled in (the handler only runs after that). + unsafe fn previous(idx: usize) -> libc::sigaction { + unsafe { (*PREVIOUS.0.get())[idx].assume_init() } + } + + fn is_default(prev: &libc::sigaction) -> bool { + prev.sa_sigaction == libc::SIG_DFL + } + + /// Restore the default disposition and send ourselves `sig`. Every call is + /// async-signal-safe. + fn die(sig: c_int) { + // SAFETY: plain syscalls on a zeroed, filled-in struct. + unsafe { + let mut dfl: libc::sigaction = core::mem::zeroed(); + dfl.sa_sigaction = libc::SIG_DFL; + libc::sigaction(sig, &dfl, core::ptr::null_mut()); + let mut set: libc::sigset_t = core::mem::zeroed(); + libc::sigemptyset(&mut set); + libc::sigaddset(&mut set, sig); + libc::pthread_sigmask(libc::SIG_UNBLOCK, &set, core::ptr::null_mut()); + libc::raise(sig); + } + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + unsafe fn errno_location() -> *mut c_int { + unsafe { libc::__errno_location() } + } + #[cfg(not(any(target_os = "linux", target_os = "android")))] + unsafe fn errno_location() -> *mut c_int { + unsafe { libc::__error() } + } + + /// Async-signal-safe: atomics, `getpid`, `write`, `sigaction`, `raise`, and a + /// call into the previous handler. + extern "C" fn handler(sig: c_int, info: *mut libc::siginfo_t, ctx: *mut c_void) { + // SAFETY: errno access and the calls below are async-signal-safe; the + // table is filled in before this handler is armed. + unsafe { + let saved_errno = *errno_location(); + let Some(idx) = index(sig) else { return }; + let prev = previous(idx); + let prev_default = is_default(&prev); + let owner = libc::getpid() as u32 == OWNER_PID.load(Ordering::Relaxed); + + if !owner { + // A fork child: no watcher thread here. Behave as if we were not + // installed. + if prev_default { + die(sig); + } + } else { + if prev_default && SHUTTING_DOWN.load(Ordering::SeqCst) { + die(sig); + } + let fd = PIPE_W.load(Ordering::Relaxed); + if fd >= 0 { + let byte = sig as u8; + // A full pipe means wake-ups are already pending: ignore. + libc::write(fd, (&byte as *const u8).cast(), 1); + } + } + + if !prev_default && prev.sa_sigaction != libc::SIG_IGN { + if prev.sa_flags & libc::SA_SIGINFO != 0 { + let f: extern "C" fn(c_int, *mut libc::siginfo_t, *mut c_void) = + core::mem::transmute(prev.sa_sigaction); + f(sig, info, ctx); + } else { + let f: extern "C" fn(c_int) = core::mem::transmute(prev.sa_sigaction); + f(sig); + } + } + *errno_location() = saved_errno; + } + } + + fn watcher(read_fd: c_int) { + let mut byte = 0u8; + loop { + // SAFETY: reading one byte into a local. + let n = unsafe { libc::read(read_fd, (&mut byte as *mut u8).cast(), 1) }; + if n == 0 { + return; + } + if n < 0 { + if io::Error::last_os_error().kind() == io::ErrorKind::Interrupted { + continue; + } + return; + } + let sig = byte as c_int; + let Some(idx) = index(sig) else { continue }; + // SAFETY: the table is filled in before the pipe can carry anything. + let prev = unsafe { previous(idx) }; + let timeout = Duration::from_millis(TIMEOUT_MS.load(Ordering::Relaxed)); + if is_default(&prev) { + // The process is about to end because of this signal: finish + // cleanly, then let it end the way it would have. + SHUTTING_DOWN.store(true, Ordering::SeqCst); + let _ = std::panic::catch_unwind(|| Bugsee::shutdown(timeout)); + die(sig); + // Still here: the signal did not kill us (blocked or ignored + // elsewhere). Carry on serving later signals. + SHUTTING_DOWN.store(false, Ordering::SeqCst); + } else { + let _ = std::panic::catch_unwind(|| Bugsee::flush(timeout)); + } + } + } + + /// Create the pipe and start the watcher; record the new owner. + fn start_watcher() -> io::Result<()> { + let mut fds = [0 as c_int; 2]; + // SAFETY: `fds` has room for the two descriptors `pipe` writes. + if unsafe { libc::pipe(fds.as_mut_ptr()) } != 0 { + return Err(io::Error::last_os_error()); + } + let (r, w) = (fds[0], fds[1]); + // SAFETY: fcntl on descriptors this function owns. + unsafe { + for fd in [r, w] { + libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC); + } + // The handler must never block on a full pipe. + let flags = libc::fcntl(w, libc::F_GETFL); + libc::fcntl(w, libc::F_SETFL, flags | libc::O_NONBLOCK); + } + let spawned = std::thread::Builder::new() + .name("bugsee-termination".into()) + .spawn(move || watcher(r)); + if let Err(e) = spawned { + // SAFETY: closing the descriptors created above. + unsafe { + libc::close(r); + libc::close(w); + } + return Err(e); + } + // The old write end (a fork child's inherited copy, or none) is left + // open on purpose: closing a descriptor the handler may be using races. + OWNER_PID.store(std::process::id(), Ordering::SeqCst); + PIPE_W.store(w, Ordering::SeqCst); + Ok(()) + } + + pub(super) fn install(timeout: Duration) -> io::Result<()> { + let mut installed = INSTALL.lock().unwrap_or_else(|e| e.into_inner()); + TIMEOUT_MS.store( + timeout.as_millis().min(u64::MAX as u128) as u64, + Ordering::Relaxed, + ); + if *installed { + // Already armed. In a fork child the watcher thread is gone: start one. + if OWNER_PID.load(Ordering::SeqCst) != std::process::id() { + SHUTTING_DOWN.store(false, Ordering::SeqCst); + start_watcher()?; + } + return Ok(()); + } + + // Record what is in force now, BEFORE arming. + for (i, sig) in SIGNALS.iter().enumerate() { + // SAFETY: querying with a null new-action fills the old one. + unsafe { + let mut cur: libc::sigaction = core::mem::zeroed(); + if libc::sigaction(*sig, core::ptr::null(), &mut cur) != 0 { + return Err(io::Error::last_os_error()); + } + (*PREVIOUS.0.get())[i] = MaybeUninit::new(cur); + } + } + start_watcher()?; + + for (i, sig) in SIGNALS.iter().enumerate() { + // SAFETY: arming our handler with a zeroed + filled struct. + unsafe { + let prev = previous(i); + let mut sa: libc::sigaction = core::mem::zeroed(); + sa.sa_sigaction = handler as *const () as usize; + // Keep the restart semantics the host chose; with no host + // handler there is nothing to preserve, and EINTR on every + // blocking call of every thread would only be a side effect of + // ours. + let host_handler = !is_default(&prev) && prev.sa_sigaction != libc::SIG_IGN; + sa.sa_flags = libc::SA_SIGINFO + | if !host_handler || prev.sa_flags & libc::SA_RESTART != 0 { + libc::SA_RESTART + } else { + 0 + }; + libc::sigemptyset(&mut sa.sa_mask); + if libc::sigaction(*sig, &sa, core::ptr::null_mut()) != 0 { + return Err(io::Error::last_os_error()); + } + } + } + *installed = true; + Ok(()) + } +} + +#[cfg(windows)] +mod imp { + use std::io; + use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + use std::sync::Mutex; + use std::time::Duration; + + use crate::Bugsee; + + const CTRL_CLOSE_EVENT: u32 = 2; + const CTRL_LOGOFF_EVENT: u32 = 5; + const CTRL_SHUTDOWN_EVENT: u32 = 6; + + type HandlerRoutine = unsafe extern "system" fn(u32) -> i32; + + #[link(name = "kernel32")] + extern "system" { + fn SetConsoleCtrlHandler(handler: Option, add: i32) -> i32; + } + + static TIMEOUT_MS: AtomicU64 = AtomicU64::new(0); + static ARMED: AtomicBool = AtomicBool::new(false); + static INSTALL: Mutex<()> = Mutex::new(()); + + /// Runs on a thread Windows creates for the event, so blocking is fine. + unsafe extern "system" fn on_event(kind: u32) -> i32 { + let timeout = Duration::from_millis(TIMEOUT_MS.load(Ordering::Relaxed)); + let _ = std::panic::catch_unwind(|| match kind { + // Windows ends the process once the handlers return, whatever they + // answer: this exit is deliberate, so finish cleanly. + CTRL_CLOSE_EVENT | CTRL_LOGOFF_EVENT | CTRL_SHUTDOWN_EVENT => { + Bugsee::shutdown(timeout); + } + // Ctrl+C / Ctrl+Break: a handler registered before ours may swallow + // it and keep running, so only flush. The host ends the session by + // dropping the launch guard. + _ => { + Bugsee::flush(timeout); + } + }); + // FALSE: pass the event on to the next handler (the host's, or the + // default one). We never swallow it. + 0 + } + + pub(super) fn install(timeout: Duration) -> io::Result<()> { + let _lock = INSTALL.lock().unwrap_or_else(|e| e.into_inner()); + TIMEOUT_MS.store( + timeout.as_millis().min(u64::MAX as u128) as u64, + Ordering::Relaxed, + ); + if ARMED.load(Ordering::SeqCst) { + return Ok(()); + } + // SAFETY: registering a handler routine with the documented signature. + if unsafe { SetConsoleCtrlHandler(Some(on_event), 1) } == 0 { + return Err(io::Error::last_os_error()); + } + ARMED.store(true, Ordering::SeqCst); + Ok(()) + } +} + +#[cfg(not(any(unix, windows)))] +mod imp { + use std::io; + use std::time::Duration; + + pub(super) fn install(_: Duration) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "termination hooks are not supported on this platform", + )) + } +} diff --git a/crates/bugsee/tests/shutdown.rs b/crates/bugsee/tests/shutdown.rs new file mode 100644 index 0000000..3a8c54c --- /dev/null +++ b/crates/bugsee/tests/shutdown.rs @@ -0,0 +1,403 @@ +// +// shutdown.rs +// bugsee +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! The shutdown contract, per exit path (see `Bugsee::stop`): what is delivered +//! by the dying process, what is recovered on the next launch, and that nothing +//! is reported twice or lost. +//! +//! Each case runs a real child process (this same test binary, re-entered +//! through `child_entry`), ends it the way the case says, then relaunches the SDK +//! in this process over the same data directory. + +#![cfg(unix)] + +use std::io::{Cursor, Read}; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use bugsee::core::transport::{Transport, TransportError}; +use bugsee::core::MockTransport; +use bugsee::{Bugsee, LaunchOptions, LogLevel}; +use serde_json::Value; +use zip::ZipArchive; + +const KIND_ENV: &str = "BUGSEE_SHUTDOWN_KIND"; +const DIR_ENV: &str = "BUGSEE_SHUTDOWN_DIR"; +const FAIL_ENV: &str = "BUGSEE_SHUTDOWN_FAIL_UPLOADS"; +const PENDING: &str = "pending-report-marker"; + +static SERIAL: Mutex<()> = Mutex::new(()); + +/// Writes every uploaded bundle to a file so the parent can see what a child +/// delivered; optionally fails every upload (an unreachable backend). +struct FileTransport { + dir: PathBuf, + fail: bool, + n: AtomicUsize, +} + +impl Transport for FileTransport { + fn register_session(&self, _: &str, _: &[u8]) -> Result { + Ok("token".into()) + } + fn create_issue(&self, _: &str, _: Option<&str>, _: &[u8]) -> Result { + if self.fail { + return Err(TransportError::Transient("backend unreachable".into())); + } + Ok("https://uploads.example/x".into()) + } + fn upload_bundle(&self, _: &str, zip: &[u8]) -> Result<(), TransportError> { + let n = self.n.fetch_add(1, Ordering::SeqCst); + std::fs::write( + self.dir + .join("out") + .join(format!("{}-{n}.zip", std::process::id())), + zip, + ) + .map_err(|e| TransportError::Transient(e.to_string())) + } +} + +fn crash_json(zip: &[u8]) -> Value { + let mut archive = ZipArchive::new(Cursor::new(zip)).expect("bundle is a zip"); + let mut entry = archive.by_name("crash.json").expect("crash.json present"); + let mut raw = Vec::new(); + entry.read_to_end(&mut raw).unwrap(); + serde_json::from_slice(&raw).unwrap() +} + +fn delivered_by_child(dir: &Path) -> Vec { + std::fs::read_dir(dir.join("out")) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| crash_json(&std::fs::read(e.path()).unwrap())) + .collect() +} + +// --------------------------------------------------------------------------- +// The child. +// --------------------------------------------------------------------------- + +static HOST_HANDLER_RAN: AtomicBool = AtomicBool::new(false); + +extern "C" fn host_handler(_: libc::c_int) { + HOST_HANDLER_RAN.store(true, Ordering::SeqCst); +} + +fn install_host_handler(sig: libc::c_int) { + // SAFETY: a zeroed sigaction filled in with a plain handler. + unsafe { + let mut sa: libc::sigaction = std::mem::zeroed(); + sa.sa_sigaction = host_handler as *const () as usize; + libc::sigemptyset(&mut sa.sa_mask); + assert_eq!(libc::sigaction(sig, &sa, std::ptr::null_mut()), 0); + } +} + +/// Not a test by itself: the body of every child. A plain run does nothing. +#[test] +fn child_entry() { + let Ok(kind) = std::env::var(KIND_ENV) else { + return; + }; + let dir = PathBuf::from(std::env::var(DIR_ENV).unwrap()); + let fail = std::env::var(FAIL_ENV).is_ok(); + let ready = || std::fs::write(dir.join("ready"), b"1").unwrap(); + + if kind == "sigterm_host_handler" { + install_host_handler(libc::SIGTERM); + } + let mut options = LaunchOptions::new("APP_TOKEN") + .data_dir(dir.join("data")) + .native_crash_capture(false) + .with_transport(Arc::new(FileTransport { + dir: dir.clone(), + fail, + n: AtomicUsize::new(0), + })); + if kind == "exit_flush" { + options = options.flush_at_exit(Duration::from_secs(2)); + } + let guard = Bugsee::launch_with(options).expect("launch"); + if kind.starts_with("sigterm") && kind != "sigterm_no_hook" { + Bugsee::install_termination_flush(Duration::from_secs(2)).expect("install"); + // Idempotent. + Bugsee::install_termination_flush(Duration::from_secs(2)).expect("install twice"); + } + Bugsee::capture_message(LogLevel::Error, PENDING); + + match kind.as_str() { + // The signal must be what persists the report: no flush beforehand. + "sigterm" | "sigterm_host_handler" => { + ready(); + if kind == "sigterm" { + std::thread::sleep(Duration::from_secs(60)); + std::process::exit(3); // the signal should have ended us + } + let start = Instant::now(); + while !HOST_HANDLER_RAN.load(Ordering::SeqCst) { + assert!(start.elapsed() < Duration::from_secs(60), "no signal"); + std::thread::sleep(Duration::from_millis(10)); + } + std::fs::write(dir.join("host-handler-ran"), b"1").unwrap(); + // The host decides to exit: a normal return path. + drop(guard); + std::process::exit(0); + } + // Reports are durably queued before the process ends the given way. + "sigterm_no_hook" | "kill" | "exit_plain" | "exit_flush" | "return" => { + Bugsee::flush(Duration::from_secs(1)); + ready(); + match kind.as_str() { + "exit_plain" | "exit_flush" => std::process::exit(0), + "return" => { + drop(guard); + std::process::exit(0); + } + _ => std::thread::sleep(Duration::from_secs(60)), + } + } + other => panic!("unknown kind {other}"), + } +} + +// --------------------------------------------------------------------------- +// The parent. +// --------------------------------------------------------------------------- + +struct Outcome { + status: std::process::ExitStatus, + stderr: String, + dir: PathBuf, + child_delivered: Vec, +} + +fn run_child(kind: &str, fail_uploads: bool, signal: Option) -> Outcome { + let dir = std::env::temp_dir().join(format!( + "bugsee-shutdown-{kind}-{}", + bugsee::core::util::random_hex(6) + )); + std::fs::create_dir_all(dir.join("out")).unwrap(); + let mut cmd = Command::new(std::env::current_exe().unwrap()); + cmd.args(["child_entry", "--exact", "--nocapture", "--test-threads=1"]) + .env(KIND_ENV, kind) + .env(DIR_ENV, &dir) + .stdout(Stdio::null()) + .stderr(Stdio::piped()); + if fail_uploads { + cmd.env(FAIL_ENV, "1"); + } + let mut child = cmd.spawn().expect("spawn"); + + let start = Instant::now(); + while !dir.join("ready").exists() { + assert!( + child.try_wait().unwrap().is_none(), + "child died before it was ready" + ); + assert!(start.elapsed() < Duration::from_secs(30), "child not ready"); + std::thread::sleep(Duration::from_millis(20)); + } + if let Some(sig) = signal { + // SAFETY: signalling a child this test spawned. + unsafe { libc::kill(child.id() as libc::pid_t, sig) }; + } + let start = Instant::now(); + let status = loop { + if let Some(s) = child.try_wait().unwrap() { + break s; + } + if start.elapsed() > Duration::from_secs(30) { + let _ = child.kill(); + panic!("child did not end after the signal"); + } + std::thread::sleep(Duration::from_millis(20)); + }; + let mut stderr = String::new(); + if let Some(mut e) = child.stderr.take() { + let _ = e.read_to_string(&mut stderr); + } + let child_delivered = delivered_by_child(&dir); + Outcome { + status, + stderr, + dir, + child_delivered, + } +} + +/// Launch over the child's data directory and collect what recovery delivers. +fn relaunch(dir: &Path) -> Vec { + let mock = Arc::new(MockTransport::default()); + let guard = Bugsee::launch_with( + LaunchOptions::new("APP_TOKEN") + .data_dir(dir.join("data")) + .native_crash_capture(false) + .with_transport(mock.clone()), + ) + .expect("relaunch"); + std::thread::sleep(Duration::from_millis(1500)); + Bugsee::flush(Duration::from_secs(10)); + let out = mock + .uploaded_bundles + .lock() + .unwrap() + .iter() + .map(|b| crash_json(b)) + .collect(); + drop(guard); + out +} + +fn count(reports: &[Value], needle: &str) -> usize { + reports + .iter() + .filter(|r| r.to_string().contains(needle)) + .count() +} + +const ABNORMAL: &str = "terminated abnormally"; + +/// Run one case and return `(pending, abnormal)` counts over everything +/// delivered by the child and by the first relaunch, after checking that a +/// second relaunch delivers nothing more. +fn run_case( + kind: &str, + fail_uploads: bool, + signal: Option, +) -> (Outcome, usize, usize) { + let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let out = run_child(kind, fail_uploads, signal); + let recovered = relaunch(&out.dir); + let again = relaunch(&out.dir); + assert!( + again.is_empty(), + "[{kind}] reported again on the following launch: {again:?}" + ); + let mut all = out.child_delivered.clone(); + all.extend(recovered); + let pending = count(&all, PENDING); + let abnormal = count(&all, ABNORMAL); + (out, pending, abnormal) +} + +#[test] +fn returning_from_main_is_a_clean_stop() { + let (out, pending, abnormal) = run_case("return", true, None); + assert_eq!(out.status.code(), Some(0), "{}", out.stderr); + assert_eq!(pending, 1, "the queued report is delivered exactly once"); + assert_eq!(abnormal, 0, "a deliberate stop is not a crash"); +} + +#[test] +fn exit_without_stop_is_an_abnormal_exit_unless_opted_in() { + // Default: nothing in the process can tell `exit` from a death, so it is + // reported as abnormal. The queued report is still not lost. + let (out, pending, abnormal) = run_case("exit_plain", true, None); + assert_eq!(out.status.code(), Some(0), "{}", out.stderr); + assert_eq!(pending, 1); + assert_eq!(abnormal, 1); + + // `flush_at_exit`: the same exit is deliberate. + let (out, pending, abnormal) = run_case("exit_flush", true, None); + assert_eq!(out.status.code(), Some(0), "{}", out.stderr); + assert_eq!(pending, 1); + assert_eq!(abnormal, 0); +} + +#[test] +fn sigkill_loses_neither_the_queue_nor_the_abnormal_exit() { + let (out, pending, abnormal) = run_case("kill", true, Some(libc::SIGKILL)); + assert_eq!(out.status.signal(), Some(libc::SIGKILL)); + assert_eq!(pending, 1); + assert_eq!(abnormal, 1); +} + +#[test] +fn sigterm_without_the_hook_is_untouched() { + let (out, pending, abnormal) = run_case("sigterm_no_hook", true, Some(libc::SIGTERM)); + assert_eq!(out.status.signal(), Some(libc::SIGTERM)); + assert_eq!(pending, 1); + assert_eq!(abnormal, 1); +} + +#[test] +fn sigterm_with_the_hook_flushes_then_dies_of_the_signal() { + // The backend is unreachable, so the report can only survive in the queue, + // and only if the signal made the SDK persist it before the process ended. + let (out, pending, abnormal) = run_case("sigterm", true, Some(libc::SIGTERM)); + assert_eq!( + out.status.signal(), + Some(libc::SIGTERM), + "the process must still die of the signal, not exit: {:?} {}", + out.status, + out.stderr + ); + assert_eq!(pending, 1, "delivered or queued, exactly once"); + assert_eq!(abnormal, 0, "a handled SIGTERM is a deliberate stop"); +} + +#[test] +fn sigterm_with_the_hook_delivers_when_the_backend_is_up() { + let (out, pending, abnormal) = run_case("sigterm", false, Some(libc::SIGTERM)); + assert_eq!(out.status.signal(), Some(libc::SIGTERM)); + assert_eq!( + count(&out.child_delivered, PENDING), + 1, + "delivered pre-exit" + ); + assert_eq!(pending, 1); + assert_eq!(abnormal, 0); +} + +#[test] +fn the_hosts_own_sigterm_handler_still_runs_and_decides() { + let (out, pending, abnormal) = run_case("sigterm_host_handler", true, Some(libc::SIGTERM)); + assert!( + out.dir.join("host-handler-ran").exists(), + "the previous handler was swallowed: {}", + out.stderr + ); + assert_eq!( + out.status.code(), + Some(0), + "the host chose to exit normally; the SDK must not kill it: {:?} {}", + out.status, + out.stderr + ); + assert_eq!(pending, 1); + assert_eq!(abnormal, 0); +} + +#[test] +fn shutdown_is_idempotent() { + let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let dir = std::env::temp_dir().join(format!( + "bugsee-shutdown-idem-{}", + bugsee::core::util::random_hex(6) + )); + let mock = Arc::new(MockTransport::default()); + let guard = Bugsee::launch_with( + LaunchOptions::new("APP_TOKEN") + .data_dir(&dir) + .native_crash_capture(false) + .with_transport(mock.clone()), + ) + .expect("launch"); + Bugsee::capture_message(LogLevel::Error, PENDING); + assert!(Bugsee::shutdown(Duration::from_secs(10)), "drained"); + assert!(!Bugsee::is_launched()); + // Everything after the first call is a no-op, including the guard's Drop. + assert!(!Bugsee::shutdown(Duration::from_secs(1))); + Bugsee::stop(); + drop(guard); + assert_eq!(mock.uploaded_bundles.lock().unwrap().len(), 1); + let _ = std::fs::remove_dir_all(&dir); +} From 5ec98a8c7edb5b4ec8902842a5a80e15b2a1e7fa Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:07:51 +0500 Subject: [PATCH 08/52] feat: general PII scrubber and report rate limiting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a regex-free scrubber (emails, token shapes, keyed secrets, card-like numbers, home-dir usernames; allow/deny/sensitive_keys; off switch) applied at report assembly over every JSON section after before_send, and token bucket rate limiting (per-signature + global, crash floor, bounded memory) applied before the durable queue, surfaced via DropReason::RateLimited and a bugsee.reports_suppressed attribute. Both on by default, opt-out via LaunchOptions::disable_scrubbing / disable_rate_limiting. Closes #11 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- PROGRESS.md | 28 +- crates/bugsee-core/src/lib.rs | 2 + crates/bugsee-core/src/ratelimit.rs | 414 ++++ crates/bugsee-core/src/reporting.rs | 71 +- crates/bugsee-core/src/runtime.rs | 95 +- crates/bugsee-core/src/scrub.rs | 1782 +++++++++++++++++ .../bugsee-core/tests/scrub_ratelimit_e2e.rs | 381 ++++ crates/bugsee-core/tests/storage_failures.rs | 2 +- crates/bugsee-reqwest/src/lib.rs | 44 +- crates/bugsee/src/api.rs | 7 + crates/bugsee/src/options.rs | 41 + crates/bugsee/tests/facade_rate_limit.rs | 53 + crates/bugsee/tests/facade_scrub.rs | 80 + 13 files changed, 2951 insertions(+), 49 deletions(-) create mode 100644 crates/bugsee-core/src/ratelimit.rs create mode 100644 crates/bugsee-core/src/scrub.rs create mode 100644 crates/bugsee-core/tests/scrub_ratelimit_e2e.rs create mode 100644 crates/bugsee/tests/facade_rate_limit.rs create mode 100644 crates/bugsee/tests/facade_scrub.rs diff --git a/PROGRESS.md b/PROGRESS.md index 6f6bbeb..e8c54e7 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -34,9 +34,34 @@ in [`DESIGN.md`](./DESIGN.md); the wire contract is `bugsee/report-bundle-struct | 2 β€” Handled errors + caught panics | βœ… | `catch_unwind` guards, panic snapshot, error/cause chains. Follow-ups: `tracing` Layer + `log` adapter done. Subsystem quarantine still open. | | 3 β€” Fatal + next-launch + correlation | βœ… (core) | Native handler, session generations + liveness markers, next-launch recovery, panic↔SIGABRT correlation, client-side native dedup signature from frame offsets. **New:** Windows native capture β€” SEH/VEH handler, PE CodeView module map, marker + frame capture β€” so all three desktop platforms report fatal crashes. Open: best-effort in-flight flush, full out-of-process minidump. | | 4 β€” Integrations & APM | βœ… (core) | APM transactions/spans, sysinfo telemetry, reqwest network capture. **New:** the environment is actually populated β€” OS version, kernel, memory, disk, locale, and app identity (`package_id`/`version`/`build`) via `LaunchOptions::app_version`. Open: sessions / release-health. | -| 5 β€” Hardening | βœ… (core) | Durable retry queue (backoff, retry cap, blacklist), `before_send`/`before_breadcrumb`, event sampling. **New:** `on_report_dropped` β€” delivery failure is observable at last; `flush()` alone never could distinguish delivered from discarded. Open: general PII scrubber, rate limits. | +| 5 β€” Hardening | βœ… (core) | Durable retry queue (backoff, retry cap, blacklist), `before_send`/`before_breadcrumb`, event sampling. **New:** `on_report_dropped` β€” delivery failure is observable at last; `flush()` alone never could distinguish delivered from discarded. **New:** general PII scrubber (`bugsee_core::scrub`, on by default, `disable_scrubbing()`) and token-bucket rate limiting (`bugsee_core::ratelimit`, on by default, `disable_rate_limiting()`, `DropReason::RateLimited`). | | 6 β€” Mobile/FFI | βœ… (core) | `bugsee-ffi` C ABI (`include/bugsee.h`). Open: actual iOS/Android target builds + Swift/Kotlin wrappers. | +## Behaviour change: PII scrubbing and rate limiting are ON by default (issue #11) + +There is no CHANGELOG file in this repo, so this is the migration note. Upgrading +changes what leaves the process: + +- **Scrubbing (default on).** Every report is scrubbed at assembly, after + `before_send`: capture channels, `crash.json`, summary/description/labels/ + attributes, `request.json` (environment) and the session-registration + environment. Emails, known token shapes, PEM private keys, URL passwords, + `password=`-style values, card numbers and home-directory user names become + markers (`[email]`, `[token]`, `[redacted]`, `[card]`, `[user]`). The reporter's + `email` identity and the dedup `signatures` are never touched; attachments and + minidumps are opaque and not scrubbed. Tune with `LaunchOptions::scrub(...)` + (`allow`/`deny`/`sensitive_keys`), disable with `disable_scrubbing()`. +- **Rate limiting (default on).** Non-fatal errors: per signature burst 5, +1/10 s; + global burst 60, +1/s. Crashes bypass those and use a crash floor (burst 20, + +1/6 s, never below 1); manual bug reports are exempt. Drops surface as + `DropReason::RateLimited` and as the `bugsee.reports_suppressed` attribute on + the next admitted non-crash report; `Recorder::suppressed_reports()` gives the + running total. Tune with `LaunchOptions::rate_limit(...)`, disable with + `disable_rate_limiting()`. +- Apps that relied on raw values appearing in reports (an email in a log line, a + token-looking attribute) must opt out or add `allow` entries. +- `bugsee-ffi` does not expose these options yet (follow-up); it gets the defaults. + ## Recent work β€” the symbol pipeline, end to end (`f224188`, `3e2f9bf`, `51f4dc7`) Closing the last gap between "a Rust crash arrives" and "a Rust crash is @@ -516,7 +541,6 @@ Rust and are worth evaluating before building one. - Phase 2: subsystem quarantine (auto-disable a repeatedly-faulting integration). - Phase 3: best-effort in-flight flush at crash; full out-of-process minidump (desktop) β€” see the backend note below, which is the same machinery. - Phase 4: sessions / release-health. -- Phase 5: general PII scrubber, rate limits. - Phase 6: iOS/Android target builds; Swift/Kotlin language wrappers. - CI: ~~cross-platform matrix~~ **done**; loom/miri/fuzz jobs still open β€” fuzzing is newly justified now that `pe_debug_id` parses headers of arbitrary third-party diff --git a/crates/bugsee-core/src/lib.rs b/crates/bugsee-core/src/lib.rs index cdbc126..9837753 100644 --- a/crates/bugsee-core/src/lib.rs +++ b/crates/bugsee-core/src/lib.rs @@ -24,9 +24,11 @@ pub mod fork; pub mod model; pub mod panic_info; pub mod queue; +pub mod ratelimit; pub mod recovery; pub mod reporting; pub mod runtime; +pub mod scrub; pub mod session; pub mod signature; pub mod transport; diff --git a/crates/bugsee-core/src/ratelimit.rs b/crates/bugsee-core/src/ratelimit.rs new file mode 100644 index 0000000..bc87500 --- /dev/null +++ b/crates/bugsee-core/src/ratelimit.rs @@ -0,0 +1,414 @@ +// +// ratelimit.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// + +//! Report-volume limiter: token buckets that stop a noisy error loop from +//! flooding the backend or the local disk. +//! +//! Applied in the capture worker after `before_send`/sampling and **before** any +//! disk work (snapshot, ZIP, durable queue). A rejected report costs one lock +//! acquisition and a hash lookup; the signature key is only copied (one small +//! allocation) the first time a signature is seen, never on a repeat or a +//! rejection of a known signature. +//! +//! * **Errors** (`IssueType::Error`) must pass both a *per-signature* bucket +//! (so one hot failure cannot starve the rest) and a *global* bucket. +//! * **Crashes** are never subject to those buckets. They draw from a separate, +//! small *crash floor* bucket that exists only to bound a pathological +//! in-process crash-report loop; its burst is large enough that a real crash +//! is always delivered. +//! * **Manual bug reports** (`IssueType::Bug`) are user-initiated and exempt. +//! +//! Memory is bounded: at most [`RateLimitConfig::max_signatures`] per-signature +//! buckets are tracked; beyond that, unseen signatures share one overflow +//! bucket. Suppressed reports are counted and the count is handed to the next +//! admitted *non-crash* report (crashes never consume it) so the backend sees +//! that volume was cut; if that report then fails to assemble the caller returns +//! the count with [`RateLimiter::restore_suppressed`]. + +use std::collections::HashMap; +use std::time::{Duration, Instant}; + +use crate::fork::ForkLock; +use crate::model::enums::IssueType; + +/// A token bucket shape: `burst` tokens available at once, one more every +/// `refill_every`. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Rate { + pub burst: u32, + pub refill_every: Duration, +} + +impl Rate { + pub const fn new(burst: u32, refill_every: Duration) -> Self { + Rate { + burst, + refill_every, + } + } +} + +/// Rate-limit configuration. The default is on, with limits generous enough +/// that a normal app never sees them. +/// +/// Non-exhaustive so options can be added without a breaking change; start from +/// [`RateLimitConfig::default`] and use the chainable setters (or assign fields). +#[non_exhaustive] +#[derive(Debug, Clone)] +pub struct RateLimitConfig { + /// Master switch. `false` admits everything. + pub enabled: bool, + /// All non-crash error reports together. Default: burst 60, 1 per second. + pub global: Rate, + /// Error reports sharing one signature. Default: burst 5, 1 per 10 s. + pub per_signature: Rate, + /// Crash floor. Default: burst 20, 1 per 6 s. + pub crash_floor: Rate, + /// Distinct signatures tracked before overflow sharing. Default 256. + pub max_signatures: usize, +} + +impl Default for RateLimitConfig { + fn default() -> Self { + RateLimitConfig { + enabled: true, + global: Rate::new(60, Duration::from_secs(1)), + per_signature: Rate::new(5, Duration::from_secs(10)), + crash_floor: Rate::new(20, Duration::from_secs(6)), + max_signatures: 256, + } + } +} + +impl RateLimitConfig { + /// A configuration that never limits. + pub fn disabled() -> Self { + RateLimitConfig { + enabled: false, + ..Default::default() + } + } + + /// Set the global (all non-crash errors) bucket. + pub fn global(mut self, rate: Rate) -> Self { + self.global = rate; + self + } + + /// Set the per-signature bucket. + pub fn per_signature(mut self, rate: Rate) -> Self { + self.per_signature = rate; + self + } + + /// Set the crash-floor bucket (its burst is clamped to at least 1). + pub fn crash_floor(mut self, rate: Rate) -> Self { + self.crash_floor = rate; + self + } + + /// Set how many distinct signatures are tracked. + pub fn max_signatures(mut self, n: usize) -> Self { + self.max_signatures = n; + self + } +} + +/// The verdict for one report. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Decision { + /// Deliver. Carries how many reports were suppressed since the last + /// admitted non-crash one (to summarise on this report; always 0 for a + /// crash). + Allow { suppressed_before: u64 }, + /// Drop: a bucket is empty. + Deny, +} + +struct Bucket { + tokens: f64, + last: Instant, +} + +impl Bucket { + fn full(rate: Rate, now: Instant) -> Self { + Bucket { + tokens: rate.burst as f64, + last: now, + } + } + + fn refill(&mut self, rate: Rate, now: Instant) { + let dt = now.saturating_duration_since(self.last); + self.last = now; + if rate.refill_every.is_zero() { + self.tokens = rate.burst as f64; + } else { + let add = dt.as_secs_f64() / rate.refill_every.as_secs_f64(); + self.tokens = (self.tokens + add).min(rate.burst as f64); + } + } + + fn has(&self) -> bool { + self.tokens >= 1.0 + } + + fn is_full(&self, rate: Rate) -> bool { + self.tokens >= rate.burst as f64 + } +} + +struct State { + global: Bucket, + crash: Bucket, + overflow: Bucket, + sigs: HashMap, + suppressed_since_allow: u64, + suppressed_total: u64, +} + +/// The limiter. `Send + Sync`; its lock takes part in the `fork()` protocol. +pub struct RateLimiter { + cfg: RateLimitConfig, + state: ForkLock, +} + +impl RateLimiter { + pub fn new(mut cfg: RateLimitConfig) -> Self { + // A crash must always be deliverable: the floor never rounds to zero. + cfg.crash_floor.burst = cfg.crash_floor.burst.max(1); + let now = Instant::now(); + let state = State { + global: Bucket::full(cfg.global, now), + crash: Bucket::full(cfg.crash_floor, now), + overflow: Bucket::full(cfg.per_signature, now), + sigs: HashMap::new(), + suppressed_since_allow: 0, + suppressed_total: 0, + }; + RateLimiter { + cfg, + state: ForkLock::new(state), + } + } + + /// Decide a report of `kind` with dedup signature `signature` (`""` when it + /// has none) arriving now. + pub fn check(&self, kind: IssueType, signature: &str) -> Decision { + self.check_at(kind, signature, Instant::now()) + } + + /// [`check`](Self::check) with an explicit clock, for tests. + pub fn check_at(&self, kind: IssueType, signature: &str, now: Instant) -> Decision { + if !self.cfg.enabled || kind == IssueType::Bug { + return Decision::Allow { + suppressed_before: 0, + }; + } + let cfg = &self.cfg; + let mut st = self.state.lock(); + let st = &mut *st; + let allowed = if kind == IssueType::Crash { + st.crash.refill(cfg.crash_floor, now); + if st.crash.has() { + st.crash.tokens -= 1.0; + true + } else { + false + } + } else { + st.global.refill(cfg.global, now); + // Make room for a new signature before borrowing its bucket. + if !st.sigs.contains_key(signature) && st.sigs.len() >= cfg.max_signatures { + let rate = cfg.per_signature; + st.sigs.retain(|_, b| { + b.refill(rate, now); + !b.is_full(rate) + }); + } + // Look up without allocating; only a first-seen signature is copied. + if !st.sigs.contains_key(signature) && st.sigs.len() < cfg.max_signatures { + st.sigs + .insert(signature.to_owned(), Bucket::full(cfg.per_signature, now)); + } + let bucket = match st.sigs.get_mut(signature) { + Some(b) => b, + None => &mut st.overflow, + }; + bucket.refill(cfg.per_signature, now); + // Spend from both buckets or neither. + if bucket.has() && st.global.has() { + bucket.tokens -= 1.0; + st.global.tokens -= 1.0; + true + } else { + false + } + }; + if allowed { + // A crash never consumes the summary: it belongs to the error + // stream that was cut. + let suppressed_before = if kind == IssueType::Crash { + 0 + } else { + std::mem::take(&mut st.suppressed_since_allow) + }; + Decision::Allow { suppressed_before } + } else { + st.suppressed_since_allow += 1; + st.suppressed_total += 1; + Decision::Deny + } + } + + /// Give back a suppressed count that was handed out by [`Decision::Allow`] + /// but could not be delivered, so the next admitted report carries it. + pub fn restore_suppressed(&self, n: u64) { + self.state.lock().suppressed_since_allow += n; + } + + /// Total reports suppressed since launch. + pub fn suppressed_total(&self) -> u64 { + self.state.lock().suppressed_total + } + + /// Number of per-signature buckets currently tracked (bounded). + pub fn tracked_signatures(&self) -> usize { + self.state.lock().sigs.len() + } + + pub(crate) fn fork_lock(&self) { + self.state.fork_lock(); + } + + pub(crate) fn fork_unlock(&self) { + self.state.fork_unlock(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn limiter() -> RateLimiter { + RateLimiter::new(RateLimitConfig { + global: Rate::new(10, Duration::from_secs(1)), + per_signature: Rate::new(3, Duration::from_secs(10)), + crash_floor: Rate::new(4, Duration::from_secs(60)), + max_signatures: 4, + ..Default::default() + }) + } + + fn allowed(d: Decision) -> bool { + matches!(d, Decision::Allow { .. }) + } + + #[test] + fn per_signature_burst_then_deny_then_refill() { + let l = limiter(); + let t0 = Instant::now(); + // Another signature is unaffected. + assert!(allowed(l.check_at(IssueType::Error, "b", t0))); + let n = (0..10) + .filter(|_| allowed(l.check_at(IssueType::Error, "a", t0))) + .count(); + assert_eq!(n, 3); + // 10 s later one token is back, and the summary reports the 7 drops. + let d = l.check_at(IssueType::Error, "a", t0 + Duration::from_secs(10)); + assert_eq!( + d, + Decision::Allow { + suppressed_before: 7 + } + ); + assert_eq!(l.suppressed_total(), 7); + } + + #[test] + fn global_limit_applies_across_signatures() { + let l = RateLimiter::new(RateLimitConfig { + global: Rate::new(10, Duration::from_secs(1)), + max_signatures: 1000, + ..Default::default() + }); + let t0 = Instant::now(); + let n = (0..100) + .filter(|i| allowed(l.check_at(IssueType::Error, &format!("s{i}"), t0))) + .count(); + assert_eq!(n, 10); + } + + #[test] + fn crashes_bypass_error_buckets_and_bugs_are_exempt() { + let l = limiter(); + let t0 = Instant::now(); + for _ in 0..100 { + l.check_at(IssueType::Error, "x", t0); + } + // Errors are exhausted globally, yet crashes up to the floor pass. + for _ in 0..4 { + assert!(allowed(l.check_at(IssueType::Crash, "x", t0))); + } + assert!(!allowed(l.check_at(IssueType::Crash, "x", t0))); + for _ in 0..50 { + assert!(allowed(l.check_at(IssueType::Bug, "x", t0))); + } + } + + #[test] + fn memory_is_bounded_under_signature_churn() { + let l = limiter(); + let t0 = Instant::now(); + for i in 0..10_000 { + l.check_at(IssueType::Error, &format!("sig{i}"), t0); + } + assert!(l.tracked_signatures() <= 4); + } + + #[test] + fn crash_floor_never_rounds_to_zero_and_crashes_leave_the_summary_alone() { + let l = RateLimiter::new(RateLimitConfig { + crash_floor: Rate::new(0, Duration::from_secs(60)), + per_signature: Rate::new(1, Duration::from_secs(60)), + ..Default::default() + }); + let t0 = Instant::now(); + assert!(allowed(l.check_at(IssueType::Error, "a", t0))); + assert_eq!(l.check_at(IssueType::Error, "a", t0), Decision::Deny); + assert_eq!( + l.check_at(IssueType::Crash, "a", t0), + Decision::Allow { + suppressed_before: 0 + } + ); + // The summary is still pending for the next error, and survives a restore. + assert_eq!( + l.check_at(IssueType::Error, "b", t0), + Decision::Allow { + suppressed_before: 1 + } + ); + l.restore_suppressed(1); + assert_eq!( + l.check_at(IssueType::Error, "c", t0), + Decision::Allow { + suppressed_before: 1 + } + ); + } + + #[test] + fn disabled_admits_everything() { + let l = RateLimiter::new(RateLimitConfig::disabled()); + let t0 = Instant::now(); + for _ in 0..1000 { + assert!(allowed(l.check_at(IssueType::Error, "a", t0))); + } + } +} diff --git a/crates/bugsee-core/src/reporting.rs b/crates/bugsee-core/src/reporting.rs index a91844c..42c3167 100644 --- a/crates/bugsee-core/src/reporting.rs +++ b/crates/bugsee-core/src/reporting.rs @@ -18,6 +18,7 @@ use crate::capture::{export_report, ChannelDocument}; use crate::model::enums::{IssueType, Severity}; use crate::model::environment::Environment; use crate::model::report::{FileDescriptor, IssueRequest, Manifest, Source, TimeWindow}; +use crate::scrub::Scrubber; use crate::util::{iso8601_ms, random_hex}; /// A user attachment carried by a report. @@ -98,6 +99,33 @@ pub fn assemble_with_extras( extra_files: &[ExtraFile], app_token: &str, created_on_ms: i64, +) -> std::io::Result { + assemble_scrubbed( + report_dir, + window, + meta, + env, + crash_json, + extra_files, + app_token, + created_on_ms, + None, + ) +} + +/// [`assemble_with_extras`] with the PII `scrubber` applied (see +/// [`assemble_docs`]). +#[allow(clippy::too_many_arguments)] +pub fn assemble_scrubbed( + report_dir: &Path, + window: TimeWindow, + meta: &ReportMeta, + env: &Environment, + crash_json: Option>, + extra_files: &[ExtraFile], + app_token: &str, + created_on_ms: i64, + scrubber: Option<&Scrubber>, ) -> std::io::Result { let docs = export_report(report_dir, window.start, window.end)?; assemble_docs( @@ -109,6 +137,7 @@ pub fn assemble_with_extras( extra_files, app_token, created_on_ms, + scrubber, ) } @@ -116,6 +145,13 @@ pub fn assemble_with_extras( /// instead of a snapshot directory. With an empty `docs` it builds a bundle that /// needs no file system at all: the in-memory delivery path used when the data /// directory is unusable (read-only, full, gone). +/// +/// This is the single choke point every delivery path (disk, in-memory, +/// recovery) goes through, so the PII `scrubber` is applied here: each capture +/// channel, `crash.json`, the metadata (summary, description, labels, +/// attributes) and `request.json` (environment). Attachments and `extra_files` +/// are opaque binary and untouched; so are the reporter's own `email` and the +/// dedup `signatures`. #[allow(clippy::too_many_arguments)] pub fn assemble_docs( docs: Vec, @@ -126,11 +162,24 @@ pub fn assemble_docs( extra_files: &[ExtraFile], app_token: &str, created_on_ms: i64, + scrubber: Option<&Scrubber>, ) -> std::io::Result { + let scrubber = scrubber.filter(|s| s.is_enabled()); + let scrubbed_meta; + let meta = match scrubber { + Some(s) => { + scrubbed_meta = s.scrub_meta(meta); + &scrubbed_meta + } + None => meta, + }; let mut files: Vec = Vec::new(); let mut entries: Vec = Vec::new(); - for doc in docs { + for mut doc in docs { + if let Some(clean) = scrubber.and_then(|s| s.scrub_envelope(&doc.bytes)) { + doc.bytes = clean; + } let filename = format!("{}.{}.json", random_hex(8), doc.channel); // The manifest `type` is the SHARED base type per the cross-SDK contract // (`events.user`/`events.system` β†’ `events`, `traces.user`/`traces.system` @@ -144,7 +193,10 @@ pub fn assemble_docs( entries.push(BundleEntry::new(filename, doc.bytes)); } - if let Some(bytes) = crash_json { + if let Some(mut bytes) = crash_json { + if let Some(clean) = scrubber.and_then(|s| s.scrub_json_bytes(&bytes)) { + bytes = clean; + } files.push(FileDescriptor::capture("crash.json", "crash")); entries.push(BundleEntry::new("crash.json", bytes)); } @@ -185,7 +237,20 @@ pub fn assemble_docs( created_on: iso8601_ms(created_on_ms), environment: env.clone(), }; - let request_json = serde_json::to_vec(&request)?; + let mut request_json = serde_json::to_vec(&request)?; + if let Some(s) = scrubber { + // The environment (paths, host names) is scrubbed; the reporter's own + // `email` is an identity they supplied on purpose and is kept. + if let Ok(mut v) = serde_json::to_value(&request) { + let email = v.as_object_mut().and_then(|m| m.remove("email")); + if s.scrub_value(&mut v) { + if let (Some(m), Some(e)) = (v.as_object_mut(), email) { + m.insert("email".into(), e); + } + request_json = serde_json::to_vec(&v)?; + } + } + } entries.push(BundleEntry::new(".apptoken", app_token.as_bytes().to_vec())); entries.push(BundleEntry::new("request.json", request_json.clone())); diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 232e8f8..c7f4f2a 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -117,6 +117,12 @@ pub enum DropReason { /// full, so it was dropped rather than queued. Only seen with an unusable /// data directory. StorageUnavailable, + /// Suppressed by the [`RateLimiter`](crate::ratelimit::RateLimiter) before + /// it was assembled (a noisy error loop). Fires once per suppressed report + /// from the capture worker thread, so keep the hook cheap. The running + /// count is attached to the next admitted non-crash report as the + /// `bugsee.reports_suppressed` attribute. + RateLimited, } /// Called when a report is abandoned. See [`DropReason`]. @@ -168,6 +174,10 @@ pub struct RecorderConfig { /// Fraction of non-fatal (`error`) reports to keep, in `[0, 1]`. Crashes are /// never sampled out. pub sample_rate: f64, + /// PII scrubber applied to every report at assembly. On by default. + pub scrub: crate::scrub::ScrubConfig, + /// Report-volume limits applied before the durable queue. On by default. + pub rate_limit: crate::ratelimit::RateLimitConfig, /// Max in-flight capture entries before new ones are dropped (back-pressure). pub max_queued_entries: usize, /// `Some(reason)` skips the disk entirely and runs in [`StorageMode::Memory`]. @@ -197,6 +207,8 @@ impl RecorderConfig { on_report_dropped: None, before_breadcrumb: None, sample_rate: 1.0, + scrub: Default::default(), + rate_limit: Default::default(), max_queued_entries: DEFAULT_MAX_QUEUED, memory_only: None, legacy_data_dir: None, @@ -234,6 +246,8 @@ struct Shared { on_report_dropped: Option, before_breadcrumb: Option, sample_rate: f64, + scrubber: Option, + limiter: crate::ratelimit::RateLimiter, /// In-flight capture entries queued to the worker (back-pressure counter). queued: AtomicUsize, max_queued: usize, @@ -260,8 +274,10 @@ impl ForkLockable for Shared { self.scope.fork_lock(); self.session.fork_lock(); self.mem_queue.fork_lock(); + self.limiter.fork_lock(); } fn fork_unlock(&self) { + self.limiter.fork_unlock(); self.mem_queue.fork_unlock(); self.session.fork_unlock(); self.scope.fork_unlock(); @@ -333,7 +349,19 @@ impl Recorder { &config.host_facts, &config.app_identity, ); - let environment_json = serde_json::to_vec(&env).unwrap_or_default(); + let scrubber = config + .scrub + .enabled + .then(|| crate::scrub::Scrubber::new(config.scrub.clone())); + // The session-registration body carries the environment too (paths, + // host facts), so it is scrubbed like every report's request.json. + let mut environment_json = serde_json::to_vec(&env).unwrap_or_default(); + if let Some(clean) = scrubber + .as_ref() + .and_then(|s| s.scrub_json_bytes(&environment_json)) + { + environment_json = clean; + } let shared = Arc::new(Shared { scope: ForkLock::new(Scope::default()), env, @@ -345,6 +373,8 @@ impl Recorder { on_report_dropped: config.on_report_dropped, before_breadcrumb: config.before_breadcrumb, sample_rate: config.sample_rate, + scrubber, + limiter: crate::ratelimit::RateLimiter::new(config.rate_limit), queued: AtomicUsize::new(0), max_queued: config.max_queued_entries.max(1), report_queued: AtomicUsize::new(0), @@ -583,6 +613,11 @@ impl Recorder { self.storage } + /// Reports suppressed by the rate limiter since launch. + pub fn suppressed_reports(&self) -> u64 { + self.shared.limiter.suppressed_total() + } + /// Whether a disk write has failed since launch (disk full, permission /// denied, directory removed). Reports still go out, rebuilt without capture /// context or held in memory, but crash recovery can no longer be relied on. @@ -957,7 +992,7 @@ fn recover_one(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSess merge_scope(shared, &mut built.meta); let window = TimeWindow { start, end }; - let delivered = reporting::assemble_with_extras( + let delivered = reporting::assemble_scrubbed( &pending.parts_dir, window, &built.meta, @@ -966,6 +1001,7 @@ fn recover_one(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSess &built.extra_files, &shared.app_token, end, + shared.scrubber.as_ref(), ) .and_then(|assembled| queue::enqueue(data_dir, &assembled)) .is_ok(); @@ -1168,15 +1204,20 @@ fn deliver_snapshot( return false; // handle's Drop reclaims the snapshot dir } } + let Some(carried) = admit(shared, &mut meta) else { + return false; + }; // The snapshot dir is reclaimed by `handle`'s Drop on return (any path). - let assembled = reporting::assemble( + let assembled = reporting::assemble_scrubbed( &handle.dir, handle.window, &meta, &shared.env, None, + &[], &shared.app_token, handle.window.end, + shared.scrubber.as_ref(), ) // The snapshot became unreadable (disk gone): still send what we know. .or_else(|_| { @@ -1185,7 +1226,10 @@ fn deliver_snapshot( }); match assembled { Ok(assembled) => persist_or_hold(shared, data_dir, assembled), - Err(_) => false, + Err(_) => { + shared.limiter.restore_suppressed(carried); + false + } } } @@ -1221,6 +1265,7 @@ fn assemble_in_memory( &[], &shared.app_token, window.end, + shared.scrubber.as_ref(), ) } @@ -1247,6 +1292,31 @@ fn hold_in_memory(shared: &Shared, report: AssembledReport) { } } +/// Apply the rate limiter. On admission, folds the count of reports suppressed +/// since the last admitted non-crash one into `meta` (so the backend can see +/// volume was cut) and returns that count, which the caller must hand back via +/// `limiter.restore_suppressed` if the report then fails to assemble. `None` +/// means the report was suppressed. +fn admit(shared: &Shared, meta: &mut ReportMeta) -> Option { + use crate::ratelimit::Decision; + let sig = meta.signatures.first().map_or("", String::as_str); + match shared.limiter.check(meta.issue_type, sig) { + Decision::Allow { suppressed_before } => { + if suppressed_before > 0 { + meta.attrs.insert( + "bugsee.reports_suppressed".into(), + Value::from(suppressed_before), + ); + } + Some(suppressed_before) + } + Decision::Deny => { + notify_dropped(shared, DropReason::RateLimited); + None + } + } +} + fn process_report( store: Option<&mut PartStore>, shared: &Shared, @@ -1280,6 +1350,11 @@ fn process_report( } } + // Rate limit before any disk work (snapshot / ZIP / durable queue). + let Some(carried) = admit(shared, &mut meta) else { + return Ok(()); + }; + let window_start = window_end.saturating_sub(caps.max_window_ms); let window = TimeWindow { start: window_start, @@ -1302,7 +1377,13 @@ fn process_report( if has_disk { shared.degraded.store(true, Ordering::Relaxed); } - assemble_in_memory(shared, &meta, window, crash_json)? + match assemble_in_memory(shared, &meta, window, crash_json) { + Ok(assembled) => assembled, + Err(e) => { + shared.limiter.restore_suppressed(carried); + return Err(e); + } + } } }; @@ -1338,14 +1419,16 @@ fn assemble_from_disk( store .snapshot_into(&handle.dir, window.start, window.end) .ok()?; - reporting::assemble( + reporting::assemble_scrubbed( &handle.dir, window, meta, &shared.env, crash_json, + &[], &shared.app_token, window.end, + shared.scrubber.as_ref(), ) .ok() } diff --git a/crates/bugsee-core/src/scrub.rs b/crates/bugsee-core/src/scrub.rs new file mode 100644 index 0000000..d72fde6 --- /dev/null +++ b/crates/bugsee-core/src/scrub.rs @@ -0,0 +1,1782 @@ +// +// scrub.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// + +//! General-purpose PII / secret scrubber applied to everything that goes into a +//! report bundle (capture channels, `crash.json`, report metadata, environment). +//! +//! It is a small hand-written, regex-free matcher (no new dependency) that +//! recognises: +//! +//! * **emails** (`user@example.com` -> `[email]`), +//! * **tokens / keys** with a well-known shape (JWTs, `sk_live_...`, `ghp_...`, +//! `AKIA...`, `xox?-...`, `hf_...`, `Bearer ...`, PEM private-key blocks) and +//! the password in a URL's userinfo (`redis://:pw@host`) -> `[token]` / +//! `[redacted]`, +//! * **keyed secrets**: the value after `password=`, `"api_key": "..."`, +//! `Authorization: ...` and similar -> `[redacted]`; and, in structured JSON, +//! every string under an object key that names a secret, +//! * **card-like numbers** (real card layouts only, Luhn-valid, plausible +//! issuer) -> `[card]`, +//! * **home-directory usernames** in paths (`/Users/alice/...`, +//! `/home/alice/...`, `C:\Users\alice\...`) -> `[user]`. +//! +//! # Where it runs +//! +//! At report **assembly** time (the "exporter"), on the capture worker thread, +//! *after* `before_send`, so values a host callback adds are scrubbed too. It is +//! never run on a fault path and never on the caller's thread. Parts on the +//! local disk are not rewritten; only what leaves the process is scrubbed. +//! Binary payloads (user attachments, minidumps) are opaque and untouched. +//! +//! # Cost +//! +//! Each detector makes one forward pass over a string with no backtracking, and +//! the pathological-input tests bound the whole thing to a few passes (the +//! detectors share run caches so adversarial input such as `sk-sk-sk-...` or +//! `password=password=...` cannot go quadratic). Clean strings are returned +//! borrowed (no allocation); a string is copied only when something matched. +//! Measured in release on an Apple-silicon laptop +//! (`cargo test -p bugsee-core --release scrub_throughput -- --ignored --nocapture`): +//! about 0.4 us for an 80-byte clean string (~5 ns/byte), and about 135 us for a +//! 100-record, 13 KB log channel (~10 ns/byte including the record split). Exporting a +//! capture channel splits it into records and only fully parses a record when a +//! cheap byte pre-check says it could contain something; see +//! [`Scrubber::scrub_envelope`]. A full 100k-entry window is a fraction of a +//! second of worker-thread time per report, which the rate limiter bounds. +//! +//! # Extension points +//! +//! [`ScrubConfig::allow`] keeps specific matched values, [`ScrubConfig::deny`] +//! always redacts literal substrings, and [`ScrubConfig::sensitive_keys`] adds +//! key names whose values are treated as secrets. Each detector can be turned +//! off, and [`ScrubConfig::disabled`] switches the whole scrubber off. +//! +//! # Key matching +//! +//! [`is_sensitive_key`] is the one credential-key matcher; `bugsee-reqwest`'s +//! URL/header/body sanitiser uses it too. In *structured* data (JSON keys, +//! headers, query parameters) it is a normalised substring test. In *free +//! text* (`name=value`, `name: value`) a looser test would redact ordinary +//! prose, so only unambiguous names (`password`, `token`, ...) match as a +//! suffix, and the ambiguous ones (`key`, `auth`, `session`, `signature`, ...) +//! additionally need `=` or a quoted key, and a value that looks like a secret +//! (8+ characters including a digit). + +use std::borrow::Cow; + +use serde_json::Value; + +const EMAIL: &str = "[email]"; +const TOKEN: &str = "[token]"; +const CARD: &str = "[card]"; +const USER: &str = "[user]"; +const REDACTED: &str = "[redacted]"; +const MARKERS: &[&str] = &[EMAIL, TOKEN, CARD, USER, REDACTED]; + +/// What the scrubber does. The default scrubs everything built in. +/// +/// Non-exhaustive so options can be added without a breaking change; build with +/// [`ScrubConfig::default`] and the chainable setters, or assign fields. +#[non_exhaustive] +#[derive(Debug, Clone)] +pub struct ScrubConfig { + /// Master switch. `false` makes the scrubber a no-op. + pub enabled: bool, + /// Redact email addresses. + pub emails: bool, + /// Redact well-known token / key shapes, PEM private keys, `Bearer` + /// credentials and URL userinfo passwords. + pub tokens: bool, + /// Redact `key=value` / `"key": "value"` pairs whose key names a secret + /// (and strings under such keys in structured JSON). + pub keyed: bool, + /// Redact Luhn-valid card-like numbers. + pub cards: bool, + /// Redact the user name in home-directory paths. + pub paths: bool, + /// Matched values left alone. A match is kept when its text equals an entry, + /// or when the entry starts with `@` and the (email) match ends with it, so + /// `"@example.com"` keeps every example.com address. + pub allow: Vec, + /// Literal substrings that are always redacted (e.g. an internal host name + /// or a customer id). + pub deny: Vec, + /// Extra key names treated as secrets, in addition to the built-ins. + /// Compared case-insensitively, ignoring `_`/`-`/spaces; a substring match + /// in structured data, a suffix match in free text. + pub sensitive_keys: Vec, +} + +impl Default for ScrubConfig { + fn default() -> Self { + ScrubConfig { + enabled: true, + emails: true, + tokens: true, + keyed: true, + cards: true, + paths: true, + allow: Vec::new(), + deny: Vec::new(), + sensitive_keys: Vec::new(), + } + } +} + +impl ScrubConfig { + /// A configuration with the scrubber switched off entirely. + pub fn disabled() -> Self { + ScrubConfig { + enabled: false, + ..Default::default() + } + } + + /// Keep matches equal to `value` (or, for `@domain`, emails at that domain). + pub fn allow(mut self, value: impl Into) -> Self { + self.allow.push(value.into()); + self + } + + /// Always redact the literal substring `value`. + pub fn deny(mut self, value: impl Into) -> Self { + self.deny.push(value.into()); + self + } + + /// Treat values under the key name `name` as secrets. + pub fn sensitive_key(mut self, name: impl Into) -> Self { + self.sensitive_keys.push(name.into()); + self + } + + /// Enable/disable email redaction. + pub fn emails(mut self, on: bool) -> Self { + self.emails = on; + self + } + + /// Enable/disable token / key redaction. + pub fn tokens(mut self, on: bool) -> Self { + self.tokens = on; + self + } + + /// Enable/disable keyed-secret redaction. + pub fn keyed(mut self, on: bool) -> Self { + self.keyed = on; + self + } + + /// Enable/disable card-number redaction. + pub fn cards(mut self, on: bool) -> Self { + self.cards = on; + self + } + + /// Enable/disable home-directory user-name redaction. + pub fn paths(mut self, on: bool) -> Self { + self.paths = on; + self + } +} + +/// Credential-bearing key stems for **structured** data (normalised: lowercase +/// alphanumerics). Substring match, so `clientSecret`, `x-session-id`, +/// `refresh_token` and `AccountKey` are all covered. +/// +/// `key` on its own is deliberately NOT a substring stem (it would redact +/// `keyboard`, `monkey`, `keyword`); it, `sig`, `pin` and `otp` are matched +/// exactly instead. Compound `...key` names that matter are listed (`apikey`, +/// `accountkey`, `accesskey`, `privatekey`). +const STEMS: &[&str] = &[ + "password", + "passwd", + "passphrase", + "pwd", + "secret", + "token", + "apikey", + "accountkey", + "accesskey", + "privatekey", + "auth", + "session", + "cookie", + "credential", + "signature", + "csrf", + "xsrf", + "bearer", + "jwt", +]; + +/// Names matched only exactly (too ambiguous to match as substrings). +const EXACT: &[&str] = &["key", "sig", "pin", "otp"]; + +/// Keys that look like a stem but are SDK-owned structure that must survive +/// (`signatures` is the crash dedup list). +const KEY_EXCEPTIONS: &[&str] = &["signatures"]; + +/// Free-text names that are unambiguous: matched as a suffix. +const TEXT_STRONG: &[&str] = &[ + "password", + "passwd", + "passphrase", + "pwd", + "secret", + "token", + "apikey", + "accountkey", + "accesskey", + "privatekey", + "authorization", + "credential", + "credentials", + "cookie", + "csrf", + "xsrf", + "jwt", + "bearer", + "sessionid", + "sessiontoken", +]; + +/// Free-text names that are also ordinary words; see the module docs. +const TEXT_AMBIGUOUS_SUFFIX: &[&str] = &["auth", "session", "signature"]; + +/// Normalise `name` (lowercase alphanumerics only) into a small stack buffer, +/// falling back to the heap for very long keys. +fn with_normalized(name: &str, f: impl FnOnce(&[u8]) -> R) -> R { + let mut buf = [0u8; 64]; + let mut n = 0; + for c in name.bytes().filter(u8::is_ascii_alphanumeric) { + if n == buf.len() { + let all: Vec = name + .bytes() + .filter(u8::is_ascii_alphanumeric) + .map(|c| c.to_ascii_lowercase()) + .collect(); + return f(&all); + } + buf[n] = c.to_ascii_lowercase(); + n += 1; + } + f(&buf[..n]) +} + +fn contains(hay: &[u8], needle: &[u8]) -> bool { + needle.is_empty() || hay.windows(needle.len()).any(|w| w == needle) +} + +/// Whether a key (JSON object key, header name, query/form parameter) names a +/// value that must be redacted. The single credential-key matcher shared by the +/// scrubber and `bugsee-reqwest`. +pub fn is_sensitive_key(name: &str) -> bool { + with_normalized(name, |n| { + !KEY_EXCEPTIONS.iter().any(|e| n == e.as_bytes()) + && (EXACT.iter().any(|e| n == e.as_bytes()) + || STEMS.iter().any(|s| contains(n, s.as_bytes()))) + }) +} + +/// Prefixed token shapes: `(prefix, min tail length, tail needs a digit)`. The +/// tail is a run of `[A-Za-z0-9_-]`. +const TOKEN_PREFIXES: &[(&str, usize, bool)] = &[ + ("sk_live_", 8, false), + ("sk_test_", 8, false), + ("rk_live_", 8, false), + ("rk_test_", 8, false), + ("whsec_", 16, false), + ("ghp_", 20, false), + ("gho_", 20, false), + ("ghu_", 20, false), + ("ghs_", 20, false), + ("ghr_", 20, false), + ("github_pat_", 20, false), + ("glpat-", 15, false), + ("xoxb-", 10, false), + ("xoxp-", 10, false), + ("xoxa-", 10, false), + ("xoxs-", 10, false), + ("xoxr-", 10, false), + ("sk-ant-", 20, false), + ("sk-", 20, true), + ("hf_", 20, false), + ("AKIA", 16, false), + ("ASIA", 16, false), + ("AIza", 30, false), + ("SG.", 20, false), + ("npm_", 30, false), + ("shpat_", 20, false), +]; + +/// First bytes of every shape `tokens` can match (a word-start filter). +const TOKEN_FIRST: &[u8] = b"srgxwhASnebB"; + +struct Span { + start: usize, + end: usize, + repl: &'static str, +} + +/// A configured scrubber. Cheap to share (`Arc`) and `Send + Sync`. +#[derive(Debug, Clone)] +pub struct Scrubber { + cfg: ScrubConfig, + /// `cfg.sensitive_keys`, normalised once. + extra_keys: Vec, +} + +impl Default for Scrubber { + fn default() -> Self { + Scrubber::new(ScrubConfig::default()) + } +} + +/// Which free-text key class a name belongs to. +#[derive(PartialEq, Eq, Clone, Copy)] +enum KeyClass { + None, + Strong, + Ambiguous, +} + +impl Scrubber { + pub fn new(cfg: ScrubConfig) -> Self { + let extra_keys = cfg + .sensitive_keys + .iter() + .map(|k| { + k.bytes() + .filter(u8::is_ascii_alphanumeric) + .map(|c| c.to_ascii_lowercase() as char) + .collect::() + }) + .filter(|k| !k.is_empty()) + .collect(); + Scrubber { cfg, extra_keys } + } + + /// Whether this scrubber does anything. + pub fn is_enabled(&self) -> bool { + self.cfg.enabled + } + + /// Scrub a string. Borrowed (no allocation) when nothing matched. + pub fn scrub_str<'a>(&self, s: &'a str) -> Cow<'a, str> { + if !self.cfg.enabled || s.is_empty() { + return Cow::Borrowed(s); + } + let mut spans: Vec = Vec::new(); + if self.cfg.emails { + self.emails(s, &mut spans); + } + if self.cfg.tokens { + self.tokens(s, &mut spans); + self.pem_blocks(s, &mut spans); + self.url_userinfo(s, &mut spans); + } + if self.cfg.keyed { + self.keyed(s, &mut spans); + } + if self.cfg.cards { + self.cards(s, &mut spans); + } + if self.cfg.paths { + self.paths(s, &mut spans); + } + for needle in &self.cfg.deny { + if needle.is_empty() { + continue; + } + for (at, m) in s.match_indices(needle.as_str()) { + spans.push(Span { + start: at, + end: at + m.len(), + repl: REDACTED, + }); + } + } + if spans.is_empty() { + return Cow::Borrowed(s); + } + spans.sort_by(|a, b| a.start.cmp(&b.start).then(b.end.cmp(&a.end))); + let mut out = String::with_capacity(s.len()); + let mut at = 0; + for sp in &spans { + if sp.end <= at { + continue; // fully inside an earlier span + } + if sp.start >= at { + out.push_str(&s[at..sp.start]); + out.push_str(sp.repl); + } + // A partial overlap extends the earlier redaction: nothing of the + // later span's tail may survive. + at = sp.end; + } + out.push_str(&s[at..]); + Cow::Owned(out) + } + + /// Scrub every string in `v` in place, and every string under an object key + /// that names a secret. Numbers and booleans are never rewritten. Returns + /// whether anything changed. + pub fn scrub_value(&self, v: &mut Value) -> bool { + if !self.cfg.enabled { + return false; + } + match v { + Value::String(s) => match self.scrub_str(s) { + Cow::Borrowed(_) => false, + Cow::Owned(n) => { + *s = n; + true + } + }, + Value::Array(a) => a.iter_mut().fold(false, |c, x| self.scrub_value(x) | c), + Value::Object(m) => { + let mut changed = false; + for (k, x) in m.iter_mut() { + if self.cfg.keyed && self.structured_key_is_sensitive(k) { + changed |= self.redact_strings(x); + } else { + changed |= self.scrub_value(x); + } + } + changed + } + _ => false, + } + } + + /// Replace every string leaf under `v` (a secret-keyed value) by a marker. + fn redact_strings(&self, v: &mut Value) -> bool { + match v { + Value::String(s) => { + if s.is_empty() || s == REDACTED || self.allowed(s) { + false + } else { + *s = REDACTED.to_string(); + true + } + } + Value::Array(a) => a.iter_mut().fold(false, |c, x| self.redact_strings(x) | c), + Value::Object(m) => m + .values_mut() + .fold(false, |c, x| self.redact_strings(x) | c), + _ => false, + } + } + + /// Scrub a serialized JSON document. `None` when nothing changed (the + /// caller keeps its original bytes). Unparseable input is scrubbed as text. + pub fn scrub_json_bytes(&self, bytes: &[u8]) -> Option> { + if !self.cfg.enabled { + return None; + } + match serde_json::from_slice::(bytes) { + Ok(mut v) => { + if self.scrub_value(&mut v) { + serde_json::to_vec(&v).ok() + } else { + None + } + } + Err(_) => match self.scrub_str(&String::from_utf8_lossy(bytes)) { + Cow::Borrowed(_) => None, + Cow::Owned(n) => Some(n.into_bytes()), + }, + } + } + + /// Scrub a capture-channel document of the form `{"...":[rec,rec,...]}` + /// (the export envelope). The array is split into records without parsing, + /// and a record is only parsed and rewritten when [`might_match`] says it + /// could contain something; clean records are copied through byte for byte. + /// `None` when nothing changed. + /// + /// [`might_match`]: Scrubber::might_match + pub fn scrub_envelope(&self, doc: &[u8]) -> Option> { + if !self.cfg.enabled { + return None; + } + let (Some(open), true) = (doc.iter().position(|&c| c == b'['), doc.ends_with(b"]}")) else { + return self.scrub_json_bytes(doc); + }; + let body_start = open + 1; + let body_end = doc.len() - 2; + if body_start > body_end { + return self.scrub_json_bytes(doc); + } + let mut changes: Vec<(usize, usize, Vec)> = Vec::new(); + let (mut depth, mut in_str, mut esc) = (0usize, false, false); + let mut rec_start = body_start; + let mut i = body_start; + while i <= body_end { + let at_end = i == body_end; + let c = if at_end { b',' } else { doc[i] }; + if !at_end && in_str { + if esc { + esc = false; + } else if c == b'\\' { + esc = true; + } else if c == b'"' { + in_str = false; + } + } else if !at_end && c == b'"' { + in_str = true; + } else if !at_end && (c == b'{' || c == b'[') { + depth += 1; + } else if !at_end && (c == b'}' || c == b']') { + depth = depth.saturating_sub(1); + } else if c == b',' && depth == 0 { + let rec = &doc[rec_start..i]; + if !rec.is_empty() && self.might_match(rec) { + if let Some(clean) = self.scrub_json_bytes(rec) { + changes.push((rec_start, i, clean)); + } + } + rec_start = i + 1; + } + i += 1; + } + if changes.is_empty() { + return None; + } + let mut out = Vec::with_capacity(doc.len()); + let mut at = 0; + for (s, e, clean) in changes { + out.extend_from_slice(&doc[at..s]); + out.extend_from_slice(&clean); + at = e; + } + out.extend_from_slice(&doc[at..]); + Some(out) + } + + /// A conservative byte-level pre-check: `false` guarantees the scrubber + /// would not change `raw` (a JSON text), so callers can skip parsing it. + /// `true` just means "look closer". Always `true` when user-defined deny + /// literals or sensitive keys are configured (they are arbitrary). + pub fn might_match(&self, raw: &[u8]) -> bool { + if !self.cfg.deny.is_empty() || !self.extra_keys.is_empty() { + return true; + } + const NEEDLES: &[&[u8]] = &[ + b"pass", + b"pwd", + b"secret", + b"token", + b"key", + b"auth", + b"session", + b"cookie", + b"credential", + b"sig", + b"csrf", + b"xsrf", + b"bearer", + b"basic", + b"jwt", + b"pin", + b"otp", + b"sk-", + b"sk_", + b"rk_", + b"gh", + b"glpat", + b"xox", + b"akia", + b"asia", + b"aiza", + b"sg.", + b"npm_", + b"shpat", + b"hf_", + b"whsec_", + b"eyj", + b"-----begin", + b"users", + b"home", + b"://", + ]; + let mut digits = 0usize; + let mut gap = 0usize; + for (i, &c) in raw.iter().enumerate() { + if c == b'@' || c == b'=' { + return true; + } + if c.is_ascii_digit() { + digits += 1; + gap = 0; + if digits >= 13 { + return true; + } + } else { + gap += 1; + // A card number may be separated by single spaces/dashes. + if gap > 1 || !(c == b' ' || c == b'-') { + digits = 0; + } + } + let lc = c.to_ascii_lowercase(); + if lc.is_ascii_alphabetic() || matches!(lc, b'-' | b':') { + let rest = &raw[i..]; + for n in NEEDLES { + if n[0] == lc + && rest.len() >= n.len() + && rest[..n.len()].eq_ignore_ascii_case(n) + { + return true; + } + } + } + } + false + } + + /// A scrubbed copy of a report's metadata: summary, description, labels and + /// attributes. The reporter's own `email` identity, the dedup `signatures` + /// and the attachments are deliberately left alone. + pub fn scrub_meta(&self, meta: &crate::reporting::ReportMeta) -> crate::reporting::ReportMeta { + let mut out = meta.clone(); + if !self.cfg.enabled { + return out; + } + for s in [&mut out.summary, &mut out.description] + .into_iter() + .flatten() + { + if let Cow::Owned(n) = self.scrub_str(s) { + *s = n; + } + } + for l in &mut out.labels { + if let Cow::Owned(n) = self.scrub_str(l) { + *l = n; + } + } + let mut attrs = Value::Object(std::mem::take(&mut out.attrs)); + self.scrub_value(&mut attrs); + if let Value::Object(m) = attrs { + out.attrs = m; + } + out + } + + // ---- helpers ---------------------------------------------------------- + + fn allowed(&self, span: &str) -> bool { + self.cfg + .allow + .iter() + .any(|a| a == span || (a.starts_with('@') && span.ends_with(a.as_str()))) + } + + fn structured_key_is_sensitive(&self, key: &str) -> bool { + is_sensitive_key(key) + || (!self.extra_keys.is_empty() + && with_normalized(key, |n| { + self.extra_keys.iter().any(|k| contains(n, k.as_bytes())) + })) + } + + fn text_key_class(&self, key: &str) -> KeyClass { + with_normalized(key, |n| { + if TEXT_STRONG.iter().any(|k| n.ends_with(k.as_bytes())) + || self.extra_keys.iter().any(|k| n.ends_with(k.as_bytes())) + { + KeyClass::Strong + } else if EXACT.iter().any(|k| n == k.as_bytes()) + || TEXT_AMBIGUOUS_SUFFIX + .iter() + .any(|k| n.ends_with(k.as_bytes())) + { + KeyClass::Ambiguous + } else { + KeyClass::None + } + }) + } + + // ---- detectors -------------------------------------------------------- + + fn emails(&self, s: &str, out: &mut Vec) { + let b = s.as_bytes(); + let mut from = 0; + while let Some(rel) = s[from..].find('@') { + let at = from + rel; + from = at + 1; + let mut st = at; + while st > 0 && is_email_local(b[st - 1]) { + st -= 1; + } + let mut en = at + 1; + while en < b.len() && is_email_domain(b[en]) { + en += 1; + } + if st == at || en <= at + 1 { + continue; + } + // The domain ends at its alphabetic top-level label, so trailing + // `-token` junk glued on by the domain charset is not swallowed. + let dom = &s[at + 1..en]; + let Some(dot) = dom.rfind('.') else { continue }; + if dot == 0 { + continue; + } + let tld_len = dom[dot + 1..] + .bytes() + .take_while(u8::is_ascii_alphabetic) + .count(); + if tld_len < 2 { + continue; + } + let en = at + 1 + dot + 1 + tld_len; + if self.allowed(&s[st..en]) { + continue; + } + out.push(Span { + start: st, + end: en, + repl: EMAIL, + }); + } + } + + /// One pass over word starts (a position whose predecessor is not a word + /// character), testing each known token shape there. Runs of token + /// characters are scanned once and cached, so adversarial input such as + /// `sk-sk-sk-...` costs O(n), not O(n^2). + fn tokens(&self, s: &str, out: &mut Vec) { + let b = s.as_bytes(); + let mut run = Run::default(); + let mut i = 0; + while i < b.len() { + if !b[i].is_ascii_alphanumeric() + || (i > 0 && is_word(b[i - 1])) + || !TOKEN_FIRST.contains(&b[i]) + { + i += 1; + continue; + } + let end = prefixed_token(b, i, &mut run) + .or_else(|| jwt(b, i, &mut run)) + .or_else(|| credential_after_scheme(b, i)); + match end { + Some((ts, te)) if !self.allowed(&s[ts..te]) => { + out.push(Span { + start: ts, + end: te, + repl: TOKEN, + }); + i = te.max(i + 1); + } + _ => i += 1, + } + } + } + + /// `-----BEGIN ... PRIVATE KEY-----` blocks, through the matching END line + /// (or the end of the string when it was truncated). + fn pem_blocks(&self, s: &str, out: &mut Vec) { + const BEGIN: &str = "-----BEGIN "; + let mut from = 0; + while let Some(rel) = s[from..].find(BEGIN) { + let st = from + rel; + let label_start = st + BEGIN.len(); + let Some(lrel) = s[label_start..].find("-----") else { + break; + }; + let label = &s[label_start..label_start + lrel]; + let body = label_start + lrel + 5; + from = body; + if !label.ends_with("PRIVATE KEY") { + continue; + } + let end = match s[body..].find("-----END ") { + Some(e) => { + let after = body + e + 9; + s[after..].find("-----").map_or(s.len(), |x| after + x + 5) + } + None => s.len(), + }; + out.push(Span { + start: st, + end, + repl: TOKEN, + }); + from = end; + } + } + + /// The password in `scheme://user:password@host` (the user name may be + /// empty: `redis://:pw@host`), regardless of what the host looks like. + fn url_userinfo(&self, s: &str, out: &mut Vec) { + let b = s.as_bytes(); + let mut from = 0; + while let Some(rel) = s[from..].find("://") { + let p = from + rel + 3; + from = p; + let mut j = p; + let mut colon = None; + while j < b.len() + && !matches!( + b[j], + b'@' | b'/' | b'?' | b'#' | b' ' | b'\t' | b'\r' | b'\n' | b'"' | b'\'' + ) + { + if b[j] == b':' && colon.is_none() { + colon = Some(j); + } + j += 1; + } + if let (Some(c), true) = (colon, j < b.len() && b[j] == b'@') { + if c + 1 < j && !self.allowed(&s[c + 1..j]) { + out.push(Span { + start: c + 1, + end: j, + repl: REDACTED, + }); + } + } + } + } + + fn keyed(&self, s: &str, out: &mut Vec) { + let b = s.as_bytes(); + // Separators before `covered` sit inside an already-claimed value. + let mut covered = 0usize; + // The unquoted value run containing the last scanned start: every start + // inside it ends at the same place, so it is never rescanned. + let mut vrun = (0usize, 0usize); + for i in 0..b.len() { + let sep = b[i]; + if (sep != b'=' && sep != b':') || i < covered { + continue; + } + // `::` paths and `scheme://` are not key/value separators. + if sep == b':' + && ((i + 1 < b.len() && matches!(b[i + 1], b':' | b'/')) + || (i > 0 && b[i - 1] == b':')) + { + continue; + } + // Key: [ident] [closing (escaped) quote] [ws] before the separator. + let mut j = i; + while j > 0 && matches!(b[j - 1], b' ' | b'\t') { + j -= 1; + } + let mut quoted_key = false; + if j > 0 && matches!(b[j - 1], b'"' | b'\'') { + j -= 1; + quoted_key = true; + if j > 0 && b[j - 1] == b'\\' { + j -= 1; + } + } + let key_end = j; + while j > 0 && (b[j - 1].is_ascii_alphanumeric() || matches!(b[j - 1], b'_' | b'-')) { + j -= 1; + } + if j == key_end { + continue; + } + let class = self.text_key_class(&s[j..key_end]); + if class == KeyClass::None { + continue; + } + // Value. + let mut k = i + 1; + while k < b.len() && matches!(b[k], b' ' | b'\t') { + k += 1; + } + let mut escaped = false; + while k + 1 < b.len() && b[k] == b'\\' && matches!(b[k + 1], b'"' | b'\'') { + escaped = true; + k += 1; + } + let quote = if k < b.len() && matches!(b[k], b'"' | b'\'') { + k += 1; + Some(b[k - 1]) + } else { + None + }; + let vs = k; + let mut ve = match quote { + Some(q) => { + let e = b[vs..] + .iter() + .position(|&c| c == q) + .map_or(b.len(), |p| vs + p); + if escaped && e > vs && b[e - 1] == b'\\' { + e - 1 + } else { + e + } + } + None if vs < b.len() && matches!(b[vs], b'{' | b'[') => bracketed_end(b, vs), + None => { + if vrun.0 <= vs && vs < vrun.1 { + vrun.1 + } else { + let mut e = vs; + while e < b.len() + && !matches!( + b[e], + b' ' | b'\t' + | b'\r' + | b'\n' + | b'&' + | b',' + | b';' + | b')' + | b'}' + | b'"' + | b'\'' + | b'<' + | b'>' + ) + { + e += 1; + } + vrun = (vs, e); + e + } + } + }; + if ve == vs { + continue; + } + let value = &s[vs..ve]; + if MARKERS.contains(&value) { + continue; + } + // `Authorization: Bearer abc` - take the credential, not the scheme. + if quote.is_none() + && ["bearer", "basic", "digest", "token"] + .iter() + .any(|w| value.eq_ignore_ascii_case(w)) + && ve < b.len() + && b[ve] == b' ' + { + let mut e = ve; + while e < b.len() && b[e] == b' ' { + e += 1; + } + while e < b.len() + && !matches!(b[e], b' ' | b'\t' | b'\r' | b'\n' | b',' | b';' | b'"') + { + e += 1; + } + ve = e; + } + if class == KeyClass::Ambiguous { + // Ordinary words ("auth: failed", "session: started") are not + // secrets: need `=` or a quoted key and a secret-looking value. + let v = &b[vs..ve]; + // Bounded look: a digit among the first 64 bytes (never O(len) per attempt). + let secretish = v.len() >= 8 && v[..v.len().min(64)].iter().any(u8::is_ascii_digit); + if !(sep == b'=' || quoted_key) || !secretish { + continue; + } + } + if self.allowed(&s[vs..ve]) { + continue; + } + covered = ve; + out.push(Span { + start: vs, + end: ve, + repl: REDACTED, + }); + } + } + + fn cards(&self, s: &str, out: &mut Vec) { + let b = s.as_bytes(); + let mut i = 0; + while i < b.len() { + if !b[i].is_ascii_digit() || (i > 0 && b[i - 1].is_ascii_alphanumeric()) { + i += 1; + continue; + } + // Digit groups separated by single ' ' or '-'. + let mut starts = [0usize; 5]; + let mut lens = [0usize; 5]; + let mut g = 0; + let mut p = i; + loop { + let mut e = p; + while e < b.len() && b[e].is_ascii_digit() { + e += 1; + } + starts[g] = p; + lens[g] = e - p; + g += 1; + p = e; + if g == 5 { + break; + } + if p + 1 < b.len() && matches!(b[p], b' ' | b'-') && b[p + 1].is_ascii_digit() { + p += 1; + } else { + break; + } + } + let first_end = starts[0] + lens[0]; + let mut hit = None; + for k in (1..=g).rev() { + if !card_layout(&lens[..k]) { + continue; + } + let end = starts[k - 1] + lens[k - 1]; + if end < b.len() && b[end].is_ascii_alphanumeric() { + continue; + } + let mut digits = [0u8; 19]; + let mut n = 0; + for gi in 0..k { + for &d in &b[starts[gi]..starts[gi] + lens[gi]] { + digits[n] = d - b'0'; + n += 1; + } + } + if plausible_issuer(&digits[..n]) + && luhn(&digits[..n]) + && !in_uuid_shape(b, i, end) + && !self.allowed(&s[i..end]) + { + hit = Some(end); + break; + } + } + match hit { + Some(end) => { + out.push(Span { + start: i, + end, + repl: CARD, + }); + i = end; + } + None => i = first_end.max(i + 1), + } + } + } + + /// Home-directory user names. Only real filesystem shapes: `/Users//` + /// (macOS, capital U), `/home//` (Linux, lowercase) and + /// `X:\Users\\`, each starting a token and followed by a deeper + /// path, so URL routes (`/users/123/orders`, `/home/feed`, `/users/{id}`) + /// are left alone. + fn paths(&self, s: &str, out: &mut Vec) { + let b = s.as_bytes(); + for i in 0..b.len() { + if b[i] != b'/' && b[i] != b'\\' { + continue; + } + let rest = &b[i + 1..]; + // `X:\Users\` / `X:/Users/`: a standalone drive letter before. + let drive = i >= 2 + && b[i - 1] == b':' + && b[i - 2].is_ascii_alphabetic() + && (i == 2 || !is_word(b[i - 3])); + let (root, windows) = + if drive && rest.len() >= 5 && rest[..5].eq_ignore_ascii_case(b"users") { + (5, true) + } else if b[i] == b'/' { + // Starts a token: not glued to a word or path segment. + if i > 0 + && (b[i - 1].is_ascii_alphanumeric() + || matches!(b[i - 1], b'/' | b'.' | b'_' | b'-' | b'~' | b'%')) + && !(b[i - 1] == b'/' && b[..i].ends_with(b"file://")) + { + continue; + } + if rest.starts_with(b"Users") { + (5, false) + } else if rest.starts_with(b"home") { + (4, false) + } else { + continue; + } + } else { + continue; + }; + if rest.len() <= root || !matches!(rest[root], b'/' | b'\\') { + continue; + } + let ns = i + 1 + root + 1; + let mut ne = ns; + while ne < b.len() + && !matches!( + b[ne], + b'/' | b'\\' | b' ' | b'\t' | b'\r' | b'\n' | b'"' | b'\'' | b')' | b':' | b',' + ) + { + ne += 1; + } + let name = &s[ns..ne]; + let deeper = ne < b.len() && matches!(b[ne], b'/' | b'\\'); + let template = name.starts_with(['{', '<', '*', '$', '[']) || name.starts_with(':'); + let numeric = name.bytes().all(|c| c.is_ascii_digit()); + if name.is_empty() + || name == USER + || template + || numeric + // `/home/feed` is a route, `/home/bob/x` a path; a bare + // `/Users/bob` or `X:\Users\bob` still names the user. + || (!deeper && !windows && root == 4) + || self.allowed(name) + { + continue; + } + out.push(Span { + start: ns, + end: ne, + repl: USER, + }); + } + } +} + +/// End of a `{...}` / `[...]` value starting at `vs` (matching close, or the +/// end of the string when unbalanced). +fn bracketed_end(b: &[u8], vs: usize) -> usize { + let mut depth = 0usize; + for (off, &c) in b[vs..].iter().enumerate() { + match c { + b'{' | b'[' => depth += 1, + b'}' | b']' => { + depth -= 1; + if depth == 0 { + return vs + off + 1; + } + } + _ => {} + } + } + b.len() +} + +/// A cached run of token characters, so repeated starts inside one run share a +/// single scan. +#[derive(Default)] +struct Run { + start: usize, + end: usize, + last_digit: Option, +} + +impl Run { + /// End of the token-character run beginning at `t`. + fn end_from(&mut self, b: &[u8], t: usize) -> usize { + if !(self.start <= t && t < self.end) { + let mut e = t; + let mut last_digit = None; + while e < b.len() && is_token_char(b[e]) { + if b[e].is_ascii_digit() { + last_digit = Some(e); + } + e += 1; + } + *self = Run { + start: t, + end: e, + last_digit, + }; + } + self.end + } + + /// Whether the run, from `t` on, contains a digit (valid after `end_from(t)`). + fn digit_from(&self, t: usize) -> bool { + self.last_digit.is_some_and(|d| d >= t) + } +} + +/// `(start, end)` of a prefixed token starting at `i`. +fn prefixed_token(b: &[u8], i: usize, run: &mut Run) -> Option<(usize, usize)> { + let rest = &b[i..]; + for &(prefix, min_tail, need_digit) in TOKEN_PREFIXES { + // Compare the first byte inline; most words stop here. + if rest[0] != prefix.as_bytes()[0] || !rest.starts_with(prefix.as_bytes()) { + continue; + } + let t = i + prefix.len(); + let e = run.end_from(b, t); + if e - t >= min_tail && (!need_digit || run.digit_from(t)) { + return Some((i, e)); + } + } + None +} + +/// `eyJ..` at `i`. +fn jwt(b: &[u8], i: usize, run: &mut Run) -> Option<(usize, usize)> { + if !b[i..].starts_with(b"eyJ") { + return None; + } + let mut en = i; + for seg in 0..3 { + let e = run.end_from(b, en); + if e == en { + return None; + } + en = e; + if seg < 2 { + if en >= b.len() || b[en] != b'.' { + return None; + } + en += 1; + } + } + Some((i, en)) +} + +/// `Bearer[:] ` / `Basic ` starting at word start `i`; +/// covers just the credential. +fn credential_after_scheme(b: &[u8], i: usize) -> Option<(usize, usize)> { + let rest = &b[i..]; + let (word, strict) = if rest.len() > 6 && rest[..6].eq_ignore_ascii_case(b"bearer") { + (6, false) + } else if rest.len() > 5 && rest[..5].eq_ignore_ascii_case(b"basic") { + (5, true) + } else { + return None; + }; + let mut j = i + word; + if j < b.len() && b[j] == b':' { + j += 1; + } + let sp = j; + while j < b.len() && b[j] == b' ' { + j += 1; + } + if j == sp { + return None; + } + let ts = j; + // Base64/URL-safe credential chars: token chars plus `. ~ + / =`. + let mut mixed = false; + while j < b.len() && (is_token_char(b[j]) || matches!(b[j], b'.' | b'~' | b'+' | b'/' | b'=')) { + mixed |= !b[j].is_ascii_alphabetic(); + j += 1; + } + let min = if strict { 8 } else { 6 }; + (j - ts >= min && (!strict || mixed)).then_some((ts, j)) +} + +fn is_word(c: u8) -> bool { + c.is_ascii_alphanumeric() || c == b'_' +} + +fn is_token_char(c: u8) -> bool { + c.is_ascii_alphanumeric() || c == b'_' || c == b'-' +} + +fn is_email_local(c: u8) -> bool { + c.is_ascii_alphanumeric() || matches!(c, b'.' | b'_' | b'%' | b'+' | b'-') +} + +fn is_email_domain(c: u8) -> bool { + c.is_ascii_alphanumeric() || matches!(c, b'.' | b'-') +} + +/// Real card layouts: contiguous 13-19 digits, or the printed groupings +/// 4-4-4-4, 4-4-4-4-3, 4-6-4 and 4-6-5. +fn card_layout(lens: &[usize]) -> bool { + match lens { + [n] => (13..=19).contains(n), + [4, 4, 4, 4] | [4, 4, 4, 4, 3] | [4, 6, 4] | [4, 6, 5] => true, + _ => false, + } +} + +/// Whether `[start,end)` sits inside a canonical 8-4-4-4-12 hex UUID. +fn in_uuid_shape(b: &[u8], start: usize, end: usize) -> bool { + let is_hexish = |c: u8| c.is_ascii_hexdigit() || c == b'-'; + let mut ts = start; + while ts > 0 && is_hexish(b[ts - 1]) { + ts -= 1; + } + let mut te = end; + while te < b.len() && is_hexish(b[te]) { + te += 1; + } + // A UUID may be a window inside a longer hex/dash chain; test every + // 36-byte window that covers the candidate. + let lo = ts.max(end.saturating_sub(36)); + let hi = start.min(te.saturating_sub(36)); + (lo..=hi).any(|w| w + 36 <= te && is_uuid(&b[w..w + 36])) +} + +fn is_uuid(w: &[u8]) -> bool { + w.iter().enumerate().all(|(i, &c)| { + if matches!(i, 8 | 13 | 18 | 23) { + c == b'-' + } else { + c.is_ascii_hexdigit() + } + }) +} + +/// Card numbers start with 3 (Amex/Diners/JCB), 4 (Visa), 5 (Mastercard), +/// 6 (Discover/UnionPay) or a 2221-2720 Mastercard BIN. Notably NOT `1`, which +/// is what epoch-millisecond/microsecond timestamps start with. +fn plausible_issuer(d: &[u8]) -> bool { + match d[0] { + 3..=6 => true, + 2 => { + let bin = d[..4].iter().fold(0u32, |a, &x| a * 10 + x as u32); + (2221..=2720).contains(&bin) + } + _ => false, + } +} + +fn luhn(d: &[u8]) -> bool { + let mut sum = 0u32; + for (i, &x) in d.iter().rev().enumerate() { + let mut v = x as u32; + if i % 2 == 1 { + v *= 2; + if v > 9 { + v -= 9; + } + } + sum += v; + } + sum % 10 < 1 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn s(x: &str) -> String { + Scrubber::default().scrub_str(x).into_owned() + } + + #[test] + fn clean_strings_are_borrowed() { + let sc = Scrubber::default(); + for t in [ + "connection refused", + "took 1712345678901 ms", + "token_count: 5000", + "see /opt/home/x and /usr/lib", + "a@b", + "version 1.2.3", + "auth: failed", + "session: started", + "GET /users/123/orders", + "GET /home/feed", + "/users/{id}/x", + "my_app::auth::login::verify", + "cancellation_token::CancellationToken", + ] { + assert!(matches!(sc.scrub_str(t), Cow::Borrowed(_)), "{t}"); + } + } + + #[test] + fn emails() { + assert_eq!(s("mail bob.smith+x@Example.co.uk now"), "mail [email] now"); + assert_eq!(s("(a@b.io)"), "([email])"); + } + + #[test] + fn tokens() { + assert_eq!(s("k=sk_live_abcdefgh12345678"), "k=[token]"); + assert_eq!(s("ghp_abcdefghijklmnopqrstuvwxyz0123456789"), "[token]"); + assert_eq!(s("hf_abcdefghijklmnopqrstuvwxyzABCDEF"), "[token]"); + assert_eq!(s("whsec_abcdefghijklmnopqrstuvwx"), "[token]"); + assert_eq!(s("id AKIAIOSFODNN7EXAMPLE x"), "id [token] x"); + let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk"; + assert_eq!(s(&format!("t {jwt}!")), "t [token]!"); + assert_eq!(s("Bearer abcdef0123456789"), "Bearer [token]"); + assert_eq!(s("Bearer: abc123"), "Bearer: [token]"); + assert_eq!(s("Bearer abc123"), "Bearer [token]"); + assert_eq!( + s("sk-learn-something-long-enough"), + "sk-learn-something-long-enough" + ); + } + + #[test] + fn pem_private_keys_and_url_userinfo() { + let pem = "x -----BEGIN RSA PRIVATE KEY-----\nMIIEow\nIBAAKCAQEA\n-----END RSA PRIVATE KEY----- y"; + assert_eq!(s(pem), "x [token] y"); + let cert = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----"; + assert_eq!(s(cert), cert); + assert_eq!( + s("redis://:s3cr3t@cache:6379/0"), + "redis://:[redacted]@cache:6379/0" + ); + // A dotted host makes `pw@host.tld` look like an email too; either + // marker is fine, the password must be gone. + let out = s("postgres://app:pw@db.internal.example.com/x"); + assert!(!out.contains("pw"), "{out}"); + assert_eq!(s("https://example.com/a:b@c"), "https://example.com/a:b@c"); + } + + #[test] + fn keyed() { + assert_eq!(s("password=hunter2&x=1"), "password=[redacted]&x=1"); + assert_eq!( + s(r#"{"api_key": "abc123"}"#), + r#"{"api_key": "[redacted]"}"# + ); + assert_eq!( + s("Authorization: Bearer abc.def"), + "Authorization: [redacted]" + ); + assert_eq!(s("access_token=xyz"), "access_token=[redacted]"); + assert_eq!(s("password=[redacted]"), "password=[redacted]"); + assert_eq!(s("password=[a, b, c] tail"), "password=[redacted] tail"); + assert_eq!(s(r#"password={"a":"b"} tail"#), "password=[redacted] tail"); + assert_eq!(s("AccountKey=abc123def456=="), "AccountKey=[redacted]"); + // Escaped quotes in raw JSON text. + assert_eq!( + s(r#"{\"password\": \"hunter2\", \"n\": 1}"#), + r#"{\"password\": \"[redacted]\", \"n\": 1}"# + ); + // Ambiguous names need `=`/quoted key and a secret-looking value. + assert_eq!(s("session=abcd1234efgh"), "session=[redacted]"); + assert_eq!(s("key=short"), "key=short"); + assert_eq!(s("auth: failed for user"), "auth: failed for user"); + } + + #[test] + fn double_colon_paths_survive() { + for t in [ + "my_app::auth::login::verify", + "cancellation_token::CancellationToken::new", + "app::secret::Vault::open", + "std::sync::password::x", + "::parse", + ] { + assert_eq!(s(t), t); + } + } + + #[test] + fn cards_need_real_layout_luhn_and_issuer() { + assert_eq!(s("card 4111 1111 1111 1111."), "card [card]."); + assert_eq!(s("4111-1111-1111-1111"), "[card]"); + assert_eq!(s("378282246310005"), "[card]"); + assert_eq!(s("3782 822463 10005"), "[card]"); + assert_eq!(s("4111111111111112"), "4111111111111112"); // bad Luhn + assert_eq!(s("1712345678901"), "1712345678901"); // epoch ms + assert_eq!(s("4111 1111 11 1111 11"), "4111 1111 11 1111 11"); // not a layout + // UUID / Mach-O code_id shapes never match. + for u in [ + "41111111-1111-1111-1111-111111111111", + "code_id 4A111111-1111-1111-1111-111111111111", + "55555555-5555-5555-5555-555555555555", + ] { + assert_eq!(s(u), u, "{u}"); + } + } + + #[test] + fn paths() { + assert_eq!(s("/Users/alice/proj/x.rs"), "/Users/[user]/proj/x.rs"); + assert_eq!( + s("at /home/bob/.cargo/a.rs:3"), + "at /home/[user]/.cargo/a.rs:3" + ); + assert_eq!(s(r"C:\Users\Carol\app\m.rs"), r"C:\Users\[user]\app\m.rs"); + assert_eq!(s("file:///Users/dan/x"), "file:///Users/[user]/x"); + assert_eq!(s("/opt/home/dan/x"), "/opt/home/dan/x"); + assert_eq!(s("/api/Users/dan/x"), "/api/Users/dan/x"); + // Routes are not paths. + for r in [ + "/users/123/orders", + "/home/feed", + "/users/{id}", + "/home/{id}/x", + "/Users/42/x", + ] { + assert_eq!(s(r), r, "{r}"); + } + } + + #[test] + fn overlapping_spans_merge_without_leaking_the_tail() { + let out = s("a@b.com-ghp_abcdefghijklmnopqrstuvwxyz0123456789 end"); + assert!(!out.contains("ghp_") && !out.contains("abcdefgh"), "{out}"); + let out = s("redis://u:ghp_abcdefghijklmnopqrstuvwxyz0123456789@h.example.com/x"); + assert!(!out.contains("abcdefghijkl"), "{out}"); + } + + #[test] + fn allow_and_deny_and_extra_keys() { + let sc = Scrubber::new( + ScrubConfig::default() + .allow("@example.com") + .allow("ops") + .deny("acme-internal") + .sensitive_key("x-customer"), + ); + assert_eq!( + sc.scrub_str("a@example.com b@other.org").as_ref(), + "a@example.com [email]" + ); + assert_eq!(sc.scrub_str("/home/ops/x").as_ref(), "/home/ops/x"); + assert_eq!( + sc.scrub_str("host acme-internal.lan").as_ref(), + "host [redacted].lan" + ); + assert_eq!( + sc.scrub_str("X-Customer: 42").as_ref(), + "X-Customer: [redacted]" + ); + } + + #[test] + fn off_switch_and_detector_toggles() { + let off = Scrubber::new(ScrubConfig::disabled()); + assert_eq!(off.scrub_str("a@b.com").as_ref(), "a@b.com"); + let no_email = Scrubber::new(ScrubConfig::default().emails(false)); + assert_eq!(no_email.scrub_str("a@b.com").as_ref(), "a@b.com"); + } + + #[test] + fn json_values_and_keys() { + let sc = Scrubber::default(); + let mut v = serde_json::json!({ + "message": "user a@b.com failed", + "headers": {"Authorization": "Bearer zzz", "Cookie": "s=1", "token_count": 5}, + "list": ["/Users/eve/x", 7], + "password": null, + "api_key": ["a", {"b": "c", "n": 3}, true], + "session_ttl": 30, + "signatures": ["a1b2c3"] + }); + assert!(sc.scrub_value(&mut v)); + assert_eq!(v["message"], "user [email] failed"); + assert_eq!(v["headers"]["Authorization"], "[redacted]"); + assert_eq!(v["headers"]["Cookie"], "[redacted]"); + assert_eq!( + v["headers"]["token_count"], 5, + "numbers are never rewritten" + ); + assert_eq!(v["list"][0], "/Users/[user]/x"); + assert_eq!(v["api_key"][0], "[redacted]"); + assert_eq!(v["api_key"][1]["b"], "[redacted]"); + assert_eq!(v["api_key"][1]["n"], 3); + assert_eq!(v["api_key"][2], true); + assert_eq!(v["session_ttl"], 30); + assert_eq!(v["signatures"][0], "a1b2c3"); + assert!(sc.scrub_json_bytes(br#"{"a":"fine"}"#).is_none()); + } + + #[test] + fn crash_json_structure_survives() { + let sc = Scrubber::default(); + let crash = serde_json::json!({ + "exception": {"name": "cancellation_token::CancellationToken", "message": "boom"}, + "signatures": ["0123456789abcdef0123456789abcdef01234567"], + "threads": [{"frames": [ + {"trace": "my_app::auth::login::verify (src/auth.rs:42)", "module": "my_app"}, + {"trace": "tokio::runtime::task::harness::Harness::poll::h1406d87bf3ffb336"} + ]}], + "modules": [{ + "filename": "/usr/lib/libsystem_kernel.dylib", + "code_id": "A1B2C3D4-1234-4567-89AB-0123456789AB", + "debug_id": "41111111-1111-1111-1111-111111111111" + }] + }); + let mut v = crash.clone(); + assert!(!sc.scrub_value(&mut v)); + assert_eq!(v, crash); + assert!(sc + .scrub_envelope(br#"{"version":2,"events":[{"t":"app::auth::x"}]}"#) + .is_none()); + } + + #[test] + fn envelope_scrubs_only_matching_records() { + let sc = Scrubber::default(); + let doc = br#"{"version":2,"events":[{"m":"fine, with a comma"},{"m":"mail a@b.com [x]"},{"m":"ok"}]}"#; + let out = sc.scrub_envelope(doc).unwrap(); + let v: Value = serde_json::from_slice(&out).unwrap(); + assert_eq!(v["events"][0]["m"], "fine, with a comma"); + assert_eq!(v["events"][1]["m"], "mail [email] [x]"); + assert_eq!(v["events"][2]["m"], "ok"); + let empty = br#"{"version":2,"events":[]}"#; + assert!(sc.scrub_envelope(empty).is_none()); + } + + #[test] + fn unicode_is_boundary_safe() { + assert_eq!( + s("hΓ©llo ΓΌnΓ―@dΓΆmain.com wΓΆrld"), + "hΓ©llo ΓΌnΓ―@dΓΆmain.com wΓΆrld" + ); + assert_eq!( + s("ζ—₯本 password=сСкрСт ζ—₯本"), + "ζ—₯本 password=[redacted] ζ—₯本" + ); + assert_eq!(s("/Users/Ρ‘Π»ΠΊΠ°/x"), "/Users/[user]/x"); + } + + /// Deterministic pseudo-random (xorshift) soup of every trigger fragment. + #[test] + fn fuzz_never_panics() { + const FRAGS: &[&str] = &[ + "@", + "a@b.co", + "sk-", + "sk_live_", + "ghp_", + "eyJ", + ".", + "-", + "_", + "password=", + "auth:", + "key=", + ":", + "::", + "=", + "\"", + "\\\"", + "\\", + "[", + "]", + "{", + "}", + "/Users/", + "/home/", + "C:\\Users\\", + "4111 1111 1111 1111", + "4111", + "1", + "9", + " ", + "\n", + "Γ©", + "ζ—₯", + "Bearer ", + "Basic ", + "-----BEGIN PRIVATE KEY-----", + "-----END ", + "://", + "x", + "token", + "AKIA", + ]; + let sc = Scrubber::default(); + let mut x: u64 = 0x2545F4914F6CDD1D; + for _ in 0..3000 { + let mut input = String::new(); + for _ in 0..(x % 40) { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + input.push_str(FRAGS[(x % FRAGS.len() as u64) as usize]); + } + // Output is valid UTF-8 by construction; the point is: no panic + // on any slice boundary. (Not idempotent in general: replacing a + // match can expose a new word boundary for the next pass.) + let _ = sc.scrub_str(&input); + } + } + + /// Adversarial 1 MB inputs finish in a bounded time (they used to be + /// quadratic). The bound is generous so debug builds and loaded CI pass. + #[test] + fn pathological_inputs_are_not_quadratic() { + let sc = Scrubber::default(); + let mb = 1 << 20; + let limit = std::time::Duration::from_secs(if cfg!(debug_assertions) { 20 } else { 2 }); + for unit in [ + "sk-", + "eyJ-", + "eyJ.", + "auth:", + "password=", + "key=", + "a@", + "@", + "4111 ", + "1-", + "/Users/", + "/home/a/", + "::", + "Bearer ", + "basic ", + "://", + "-----BEGIN PRIVATE KEY-----", + "ghp_", + "AKIA", + "[", + "\\\"password\\\":", + ] { + let input = unit.repeat(mb / unit.len()); + let t = std::time::Instant::now(); + let out = sc.scrub_str(&input); + let took = t.elapsed(); + assert!( + took < limit, + "{unit:?} took {took:?} (out {} bytes)", + out.len() + ); + } + // One huge token run and one huge value. + for input in [ + format!("sk-{}", "a1".repeat(mb / 2)), + format!("password={}", "x".repeat(mb)), + ] { + let t = std::time::Instant::now(); + let _ = sc.scrub_str(&input); + assert!(t.elapsed() < limit); + } + } + + /// Run with `--ignored --nocapture` for the documented hot-path cost. + #[test] + #[ignore] + fn scrub_throughput() { + let sc = Scrubber::default(); + let clean = + "GET /api/v1/orders failed: connection reset by peer after 3 retries (attempt=3)"; + let n = 200_000; + let t = std::time::Instant::now(); + let mut keep = 0; + for _ in 0..n { + keep += sc.scrub_str(std::hint::black_box(clean)).len(); + } + println!("clean {}B: {:?}/str ({keep})", clean.len(), t.elapsed() / n); + let dirty = "user bob@example.com token=abc123 at /home/bob/app/src/main.rs:10"; + let t = std::time::Instant::now(); + for _ in 0..n { + keep += sc.scrub_str(std::hint::black_box(dirty)).len(); + } + println!("dirty {}B: {:?}/str ({keep})", dirty.len(), t.elapsed() / n); + // A realistic 100-record log channel. + let rec = r#"{"timestamp":1712345678901,"level":1,"source":2,"message":"GET /api/v1/orders failed: connection reset by peer after 3 retries"}"#; + let doc = format!(r#"{{"version":2,"events":[{}]}}"#, vec![rec; 100].join(",")); + let t = std::time::Instant::now(); + for _ in 0..2000 { + keep += sc + .scrub_envelope(std::hint::black_box(doc.as_bytes())) + .map_or(0, |v| v.len()); + } + println!( + "envelope 100 recs ({}B): {:?}/doc ({keep})", + doc.len(), + t.elapsed() / 2000 + ); + } +} diff --git a/crates/bugsee-core/tests/scrub_ratelimit_e2e.rs b/crates/bugsee-core/tests/scrub_ratelimit_e2e.rs new file mode 100644 index 0000000..815bf7b --- /dev/null +++ b/crates/bugsee-core/tests/scrub_ratelimit_e2e.rs @@ -0,0 +1,381 @@ +// +// scrub_ratelimit_e2e.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! End-to-end for the default PII scrubber and report rate limiting: seeded +//! secrets in every report section must not reach the transport, and an error +//! storm is bounded while crashes still get through. + +use std::io::{Cursor, Read}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use bugsee_core::model::entry::{Breadcrumb, CaptureEntry, EventEntry, LogEntry, TraceEntry}; +use bugsee_core::model::enums::{BreadcrumbLevel, IssueType, LogLevel, LogSource}; +use bugsee_core::ratelimit::{Rate, RateLimitConfig}; +use bugsee_core::reporting::manual_upload_meta; +use bugsee_core::runtime::DropReason; +use bugsee_core::scrub::ScrubConfig; +use bugsee_core::util::epoch_ms; +use bugsee_core::{MockTransport, Recorder, RecorderConfig, ReportMeta}; +use serde_json::{json, Map, Value}; +use zip::ZipArchive; + +struct TempDir(PathBuf); +impl TempDir { + fn new() -> Self { + let mut p = std::env::temp_dir(); + p.push(format!("bugsee-scrub-{}", bugsee_core::util::random_hex(8))); + std::fs::create_dir_all(&p).unwrap(); + TempDir(p) + } +} +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +// Split so the literal never appears whole in source (push protection / secret scanners). +const STRIPE_LIKE: &str = concat!("sk_", "live_", "4eC39HqLyjWDarjtT1zdp7dc"); + +const SECRETS: &[&str] = &[ + "alice.smith@corp-example.com", + STRIPE_LIKE, + "ghp_abcdefghijklmnopqrstuvwxyz0123456789", + "AKIAIOSFODNN7EXAMPLE", + "4111 1111 1111 1111", + "hunter2-super-pw", + "/Users/jdoe/", + "/home/jdoe/", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk", +]; + +/// One string packed with every kind of secret. +fn dirty() -> String { + format!( + "mail alice.smith@corp-example.com key {STRIPE_LIKE} \ + gh ghp_abcdefghijklmnopqrstuvwxyz0123456789 aws AKIAIOSFODNN7EXAMPLE \ + card 4111 1111 1111 1111 password=hunter2-super-pw at /Users/jdoe/proj/a.rs \ + and /home/jdoe/b.rs jwt eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk" + ) +} + +fn unzip_all(zip: &[u8]) -> Vec<(String, String)> { + let mut z = ZipArchive::new(Cursor::new(zip.to_vec())).unwrap(); + (0..z.len()) + .map(|i| { + let mut f = z.by_index(i).unwrap(); + let mut buf = Vec::new(); + f.read_to_end(&mut buf).unwrap(); + ( + f.name().to_string(), + String::from_utf8_lossy(&buf).into_owned(), + ) + }) + .collect() +} + +fn config(dir: &TempDir) -> RecorderConfig { + RecorderConfig::new(&dir.0, "TOKEN") +} + +fn capture_all_kinds(r: &Recorder) { + let now = epoch_ms(); + r.capture(CaptureEntry::Log(LogEntry { + timestamp: now, + level: LogLevel::Info, + source: LogSource::StdOut, + tag: Some("t".into()), + message: Some(dirty()), + custom: Map::new(), + })); + let mut params = Map::new(); + params.insert("note".into(), Value::from(dirty())); + params.insert("api_key".into(), Value::from("hunter2-super-pw")); + r.capture(CaptureEntry::UserEvent(EventEntry { + timestamp: now, + name: Some("evt".into()), + params, + custom: Map::new(), + })); + let mut data = Map::new(); + data.insert("path".into(), Value::from(dirty())); + r.capture(CaptureEntry::Breadcrumb(Breadcrumb { + timestamp: now, + crumb_type: Some("nav".into()), + category: Some("c".into()), + level: BreadcrumbLevel::Info, + message: Some(dirty()), + data: Some(data), + })); + r.capture(CaptureEntry::UserTrace(TraceEntry { + timestamp: now, + display_id: None, + name: Some("t".into()), + value: json!(dirty()), + custom: Map::new(), + })); +} + +fn dirty_meta(kind: IssueType) -> ReportMeta { + let mut m = manual_upload_meta(); + m.issue_type = kind; + m.summary = Some(dirty()); + m.description = Some(dirty()); + m.labels = vec!["alice.smith@corp-example.com".into()]; + m.signatures = vec!["a".repeat(40)]; + m.attrs.insert("note".into(), Value::from(dirty())); + m.attrs + .insert("password".into(), Value::from("hunter2-super-pw")); + m +} + +#[test] +fn seeded_secrets_in_every_report_section_do_not_survive_by_default() { + let dir = TempDir::new(); + let transport = Arc::new(MockTransport::default()); + let r = Recorder::launch(config(&dir), transport.clone()).unwrap(); + r.with_scope(|s| s.set_attribute("scope_attr", Value::from(dirty()))); + capture_all_kinds(&r); + let crash = json!({"message": dirty(), "frames": [{"file": "/Users/jdoe/src/main.rs"}]}); + r.report( + dirty_meta(IssueType::Crash), + Some(serde_json::to_vec(&crash).unwrap()), + ); + r.report(dirty_meta(IssueType::Bug), None); + assert!(r.flush(Duration::from_secs(10))); + + let bundles = transport.uploaded_bundles.lock().unwrap(); + assert_eq!(bundles.len(), 2); + for bundle in bundles.iter() { + let files = unzip_all(bundle); + let names: Vec<&str> = files.iter().map(|f| f.0.as_str()).collect(); + for want in [ + ".log.json", + ".events.user.json", + ".breadcrumbs.json", + ".traces.user.json", + ] { + assert!( + names.iter().any(|n| n.ends_with(want)), + "{want} in {names:?}" + ); + } + for (name, body) in &files { + if name == ".apptoken" { + continue; + } + for secret in SECRETS { + assert!( + !body.contains(secret), + "{secret:?} survived in {name}: {body}" + ); + } + } + // Markers are present and the document still parses. + let log = files.iter().find(|f| f.0.ends_with(".log.json")).unwrap(); + assert!(log.1.contains("[email]") && log.1.contains("[token]") && log.1.contains("[card]")); + serde_json::from_str::(&log.1).unwrap(); + } + assert!( + unzip_all(&bundles[0]).iter().any(|f| f.0 == "crash.json"), + "crash.json scrubbed, not dropped" + ); +} + +#[test] +fn scrubbing_can_be_switched_off() { + let dir = TempDir::new(); + let transport = Arc::new(MockTransport::default()); + let mut cfg = config(&dir); + cfg.scrub = ScrubConfig::disabled(); + let r = Recorder::launch(cfg, transport.clone()).unwrap(); + r.report(dirty_meta(IssueType::Bug), None); + assert!(r.flush(Duration::from_secs(10))); + let body = String::from_utf8_lossy(&transport.created_requests.lock().unwrap()[0]).into_owned(); + assert!(body.contains("alice.smith@corp-example.com")); +} + +#[test] +fn reporter_email_identity_is_kept() { + let dir = TempDir::new(); + let transport = Arc::new(MockTransport::default()); + let r = Recorder::launch(config(&dir), transport.clone()).unwrap(); + let mut m = manual_upload_meta(); + m.email = Some("reporter@example.com".into()); + r.report(m, None); + assert!(r.flush(Duration::from_secs(10))); + let req: Value = + serde_json::from_slice(&transport.created_requests.lock().unwrap()[0]).unwrap(); + assert_eq!(req["email"], "reporter@example.com"); +} + +#[test] +fn error_storm_is_bounded_crashes_survive_and_drops_are_summarised() { + let dir = TempDir::new(); + let transport = Arc::new(MockTransport::default()); + let dropped = Arc::new(AtomicUsize::new(0)); + let d2 = dropped.clone(); + let mut cfg = config(&dir); + cfg.rate_limit = RateLimitConfig::default() + .per_signature(Rate::new(3, Duration::from_millis(300))) + .global(Rate::new(1000, Duration::from_secs(1))); + cfg.on_report_dropped = Some(Box::new(move |reason| { + if reason == DropReason::RateLimited { + d2.fetch_add(1, Ordering::SeqCst); + } + })); + let r = Recorder::launch(cfg, transport.clone()).unwrap(); + + let error = |sig: &str| { + let mut m = manual_upload_meta(); + m.issue_type = IssueType::Error; + m.signatures = vec![sig.into()]; + m + }; + for _ in 0..200 { + r.report(error("hot"), None); + } + // A different signature and a crash are unaffected by the hot loop. + r.report(error("other"), None); + let mut crash = manual_upload_meta(); + crash.issue_type = IssueType::Crash; + crash.signatures = vec!["hot".into()]; + r.report(crash, Some(b"{}".to_vec())); + assert!(r.flush(Duration::from_secs(20))); + + let created = transport.created_requests.lock().unwrap().len(); + // 3 hot errors + 1 other + 1 crash (+ at most a refill or two while the + // storm was being processed). + assert!((5..=7).contains(&created), "created {created}"); + let n_dropped = dropped.load(Ordering::SeqCst); + assert!((190..=197).contains(&n_dropped), "dropped {n_dropped}"); + { + let reqs = transport.created_requests.lock().unwrap(); + assert!(reqs + .iter() + .any(|b| { serde_json::from_slice::(b).unwrap()["type"] == "crash" })); + } + + // After the bucket refills a hot error is admitted again. + std::thread::sleep(Duration::from_millis(400)); + r.report(error("hot"), None); + assert!(r.flush(Duration::from_secs(10))); + + // The suppressed count rode on the first report admitted after the storm + // (and was then reset, so it is not repeated). + let bundles = transport.uploaded_bundles.lock().unwrap(); + assert_eq!(bundles.len(), created + 1); + let counts: Vec = bundles + .iter() + .filter_map(|b| { + let files = unzip_all(b); + let m: Value = + serde_json::from_str(&files.iter().find(|f| f.0 == "manifest.json")?.1).ok()?; + m["attrs"]["bugsee.reports_suppressed"].as_u64() + }) + .collect(); + assert_eq!(counts.len(), 1, "{counts:?}"); + assert!(counts[0] >= 190, "{counts:?}"); +} + +#[test] +fn rate_limiting_can_be_switched_off() { + let dir = TempDir::new(); + let transport = Arc::new(MockTransport::default()); + let mut cfg = config(&dir); + cfg.rate_limit = RateLimitConfig::disabled(); + let r = Recorder::launch(cfg, transport.clone()).unwrap(); + for _ in 0..30 { + let mut m = manual_upload_meta(); + m.issue_type = IssueType::Error; + m.signatures = vec!["x".into()]; + r.report(m, None); + } + assert!(r.flush(Duration::from_secs(30))); + assert_eq!(transport.created_requests.lock().unwrap().len(), 30); +} + +#[test] +fn in_memory_mode_reports_are_scrubbed_too() { + let dir = TempDir::new(); + let transport = Arc::new(MockTransport::default()); + let mut cfg = RecorderConfig::new(dir.0.join("must-stay-absent"), "T"); + cfg.memory_only = Some("test".to_string()); + let r = Recorder::launch(cfg, transport.clone()).unwrap(); + let crash = json!({"message": dirty(), "frames": [{"file": "/Users/jdoe/src/main.rs"}]}); + r.report( + dirty_meta(IssueType::Crash), + Some(serde_json::to_vec(&crash).unwrap()), + ); + r.report(dirty_meta(IssueType::Error), None); + assert!(r.flush(Duration::from_secs(10))); + let bundles = transport.uploaded_bundles.lock().unwrap(); + assert_eq!(bundles.len(), 2); + for b in bundles.iter() { + for (name, body) in unzip_all(b) { + if name == ".apptoken" { + continue; + } + for secret in SECRETS { + assert!(!body.contains(secret), "{secret:?} survived in {name}"); + } + } + } + for req in transport.created_requests.lock().unwrap().iter() { + let body = String::from_utf8_lossy(req); + for secret in SECRETS { + assert!(!body.contains(secret), "{secret:?} in request.json"); + } + } +} + +#[test] +fn structural_report_fields_survive_scrubbing_unchanged() { + let dir = TempDir::new(); + let scrubbed = Arc::new(MockTransport::default()); + let plain = Arc::new(MockTransport::default()); + let crash = json!({ + "exception": {"name": "cancellation_token::CancellationToken", "message": "boom"}, + "signatures": ["0123456789abcdef0123456789abcdef01234567"], + "threads": [{"frames": [{"trace": "my_app::auth::login::verify (src/auth.rs:42)"}]}], + "modules": [{"filename": "/usr/lib/libsystem_kernel.dylib", + "code_id": "A1B2C3D4-1234-4567-89AB-0123456789AB", + "debug_id": "41111111-1111-1111-1111-111111111111"}] + }); + for (transport, scrub) in [(&scrubbed, true), (&plain, false)] { + let mut cfg = config(&dir); + if !scrub { + cfg.scrub = ScrubConfig::disabled(); + } + let r = Recorder::launch(cfg, transport.clone()).unwrap(); + let mut m = manual_upload_meta(); + m.issue_type = IssueType::Crash; + m.signatures = vec!["0123456789abcdef0123456789abcdef01234567".into()]; + m.summary = Some("my_app::auth::login::verify panicked".into()); + r.report(m, Some(serde_json::to_vec(&crash).unwrap())); + assert!(r.flush(Duration::from_secs(10))); + drop(r); + } + let get = |t: &MockTransport, name: &str| -> Value { + let b = t.uploaded_bundles.lock().unwrap(); + let f = unzip_all(&b[0]); + serde_json::from_str(&f.iter().find(|f| f.0 == name).unwrap().1).unwrap() + }; + assert_eq!(get(&scrubbed, "crash.json"), get(&plain, "crash.json")); + let strip = |mut v: Value| { + for k in ["created_on", "environment"] { + v.as_object_mut().unwrap().remove(k); + } + v + }; + let req = |t: &MockTransport| -> Value { + strip(serde_json::from_slice(&t.created_requests.lock().unwrap()[0]).unwrap()) + }; + assert_eq!(req(&scrubbed), req(&plain)); +} diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index d8ab57a..bdb369d 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -226,7 +226,7 @@ fn legacy_directory_leftovers_are_delivered_once_and_cleaned_up() { &Default::default(), ); let window = TimeWindow { start: 0, end: 1 }; - let stranded = assemble_docs(Vec::new(), window, &meta, &env, None, &[], "T", 1).unwrap(); + let stranded = assemble_docs(Vec::new(), window, &meta, &env, None, &[], "T", 1, None).unwrap(); bugsee_core::queue::enqueue(&legacy, &stranded).unwrap(); // ...and a session of that build that died (marker left behind, owner gone). diff --git a/crates/bugsee-reqwest/src/lib.rs b/crates/bugsee-reqwest/src/lib.rs index 1a0c58d..50cdb58 100644 --- a/crates/bugsee-reqwest/src/lib.rs +++ b/crates/bugsee-reqwest/src/lib.rs @@ -175,44 +175,14 @@ fn error_entry(id: &str, method: &str, url: &str, error: String) -> NetworkEntry /// Whether a key (query parameter, form field, or JSON object key) names a value /// that must be redacted. Used by URL, form-body, and recursive JSON redaction. /// -/// Matching normalizes away case and separators so `accessToken`, `access_token`, -/// and `access-token` all collapse to the same stem, then applies a substring -/// heuristic (like [`is_sensitive_header`]) rather than exact equality β€” exact -/// matching silently leaked common shapes like `accessToken`/`authorization`. +/// Delegates to `bugsee_core::scrub::is_sensitive_key`, the single credential-key +/// matcher shared with the general PII scrubber: normalised (case and separators +/// ignored) substring match over a credential stem list, plus exact matches for +/// the short ambiguous names `key`/`sig`/`pin`/`otp`. fn is_sensitive_param(name: &str) -> bool { - // Lowercase and drop non-alphanumerics: accessToken/access_token/access-token - // β†’ "accesstoken"; x-api-key β†’ "xapikey". - let n: String = name - .chars() - .filter(|c| c.is_ascii_alphanumeric()) - .map(|c| c.to_ascii_lowercase()) - .collect(); - - // Short, ambiguous tokens matched exactly to avoid over-redacting words that - // merely contain them (e.g. "keyword", "monkey", "design"). - if matches!(n.as_str(), "key" | "sig" | "pin" | "otp") { - return true; - } - // Credential-bearing stems; substring so compounds are covered - // (clientSecret, refreshToken, x-session-id, csrfToken, …). - const STEMS: &[&str] = &[ - "password", - "passwd", - "passphrase", - "pwd", - "secret", - "token", - "apikey", - "auth", - "session", - "cookie", - "credential", - "signature", - "csrf", - "xsrf", - "bearer", - ]; - STEMS.iter().any(|stem| n.contains(stem)) + // One matcher for the whole SDK: the PII scrubber uses the same function, so + // a key redacted here is redacted everywhere (and vice versa). + bugsee::core::scrub::is_sensitive_key(name) } /// Header names whose values are redacted. Delegates to the SAME normalized diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index a693e7f..fff3e5f 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -139,6 +139,8 @@ impl Bugsee { config.on_report_dropped = options.on_report_dropped; config.before_breadcrumb = options.before_breadcrumb; config.sample_rate = options.sample_rate; + config.scrub = options.scrub.clone(); + config.rate_limit = options.rate_limit.clone(); config.app_identity = bugsee_core::model::environment::AppIdentity { package_id: options.app_package_id.clone(), version: options.app_version.clone(), @@ -454,6 +456,11 @@ impl Bugsee { crate::perf::Transaction::start(name.into(), operation.into()) } + /// Reports the rate limiter has suppressed since launch (`0` before launch). + pub fn suppressed_reports() -> u64 { + Self::with_recorder(|r| r.suppressed_reports()).unwrap_or(0) + } + /// Block until pending work drains, or `timeout` elapses. pub fn flush(timeout: Duration) -> bool { // Off-lock: flush blocks; must not hold the mutex across the wait. diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index e4de340..426ddbb 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -39,6 +39,8 @@ pub struct LaunchOptions { pub(crate) before_send: Option, pub(crate) before_breadcrumb: Option, pub(crate) sample_rate: f64, + pub(crate) scrub: bugsee_core::scrub::ScrubConfig, + pub(crate) rate_limit: bugsee_core::ratelimit::RateLimitConfig, } impl LaunchOptions { @@ -63,9 +65,48 @@ impl LaunchOptions { before_send: None, before_breadcrumb: None, sample_rate: 1.0, + scrub: Default::default(), + rate_limit: Default::default(), } } + /// Configure the PII / secret scrubber that runs over every report (capture + /// channels, crash data, summary/description/attributes, environment) just + /// before it is queued, after `before_send`. **On by default**: emails, + /// well-known token shapes, `password=`-style values, card-like numbers and + /// home-directory user names are replaced with markers such as `[email]`. + /// Use the config's `allow` / `deny` / `sensitive_keys` to tune it. + /// Attachments and native minidumps are opaque and not scrubbed. + pub fn scrub(mut self, config: bugsee_core::scrub::ScrubConfig) -> Self { + self.scrub = config; + self + } + + /// Turn the PII scrubber off entirely. Reports then carry exactly what was + /// captured; make sure your own `before_send` redacts what must not leave. + pub fn disable_scrubbing(mut self) -> Self { + self.scrub = bugsee_core::scrub::ScrubConfig::disabled(); + self + } + + /// Configure report rate limiting (token buckets applied before the durable + /// queue). **On by default**: non-fatal errors are limited per signature + /// (burst 5, +1/10 s) and globally (burst 60, +1/s); crashes are never + /// limited by those and draw from a small crash floor (burst 20); manual + /// bug reports are exempt. Suppressed reports surface via + /// `on_report_dropped(DropReason::RateLimited)` and are counted in the + /// `bugsee.reports_suppressed` attribute of the next report sent. + pub fn rate_limit(mut self, config: bugsee_core::ratelimit::RateLimitConfig) -> Self { + self.rate_limit = config; + self + } + + /// Turn report rate limiting off entirely. + pub fn disable_rate_limiting(mut self) -> Self { + self.rate_limit = bugsee_core::ratelimit::RateLimitConfig::disabled(); + self + } + /// Fraction of non-fatal (`error`) reports to keep, in `[0.0, 1.0]`. Crashes /// are never sampled out. Default `1.0` (keep everything). pub fn sample_rate(mut self, rate: f64) -> Self { diff --git a/crates/bugsee/tests/facade_rate_limit.rs b/crates/bugsee/tests/facade_rate_limit.rs new file mode 100644 index 0000000..971c5e4 --- /dev/null +++ b/crates/bugsee/tests/facade_rate_limit.rs @@ -0,0 +1,53 @@ +// +// facade_rate_limit.rs +// bugsee +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! A handled-error storm is rate limited by default through the facade; `disable_rate_limiting` lifts it. + +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use bugsee::core::MockTransport; +use bugsee::{Bugsee, LaunchOptions, LogLevel}; + +struct TempDir { + path: PathBuf, +} +impl TempDir { + fn new() -> Self { + let mut path = std::env::temp_dir(); + path.push(format!("bugsee-fac-{}", bugsee::core::util::random_hex(8))); + std::fs::create_dir_all(&path).unwrap(); + TempDir { path } + } +} +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } +} + +fn storm(options: LaunchOptions, n: usize) -> usize { + let dir = TempDir::new(); + let mock = Arc::new(MockTransport::default()); + let guard = + Bugsee::launch_with(options.data_dir(&dir.path).with_transport(mock.clone())).unwrap(); + for _ in 0..n { + Bugsee::capture_message(LogLevel::Error, "the same failure, again"); + } + assert!(Bugsee::flush(Duration::from_secs(30))); + drop(guard); + let n = mock.uploaded_bundles.lock().unwrap().len(); + n +} + +#[test] +fn storm_is_bounded_by_default_and_unbounded_when_disabled() { + let limited = storm(LaunchOptions::new("T"), 40); + assert!((1..=7).contains(&limited), "delivered {limited}"); + let unlimited = storm(LaunchOptions::new("T").disable_rate_limiting(), 40); + assert_eq!(unlimited, 40); +} diff --git a/crates/bugsee/tests/facade_scrub.rs b/crates/bugsee/tests/facade_scrub.rs new file mode 100644 index 0000000..1b8ef3c --- /dev/null +++ b/crates/bugsee/tests/facade_scrub.rs @@ -0,0 +1,80 @@ +// +// facade_scrub.rs +// bugsee +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! The default scrubber is on through the public facade, and `disable_scrubbing` turns it off. + +use std::io::{Cursor, Read}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use bugsee::core::MockTransport; +use bugsee::{Bugsee, LaunchOptions, LogLevel}; +use zip::ZipArchive; + +struct TempDir { + path: PathBuf, +} +impl TempDir { + fn new() -> Self { + let mut path = std::env::temp_dir(); + path.push(format!("bugsee-fac-{}", bugsee::core::util::random_hex(8))); + std::fs::create_dir_all(&path).unwrap(); + TempDir { path } + } +} +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } +} + +fn all_text(zip: &[u8]) -> String { + let mut z = ZipArchive::new(Cursor::new(zip.to_vec())).unwrap(); + let mut out = String::new(); + for i in 0..z.len() { + let mut f = z.by_index(i).unwrap(); + if f.name() == ".apptoken" { + continue; + } + let mut b = Vec::new(); + f.read_to_end(&mut b).unwrap(); + out.push_str(&String::from_utf8_lossy(&b)); + } + out +} + +#[test] +fn default_scrubs_and_opt_out_does_not() { + for (disable, expect_leak) in [(false, false), (true, true)] { + let dir = TempDir::new(); + let mock = Arc::new(MockTransport::default()); + let mut options = LaunchOptions::new("T") + .data_dir(&dir.path) + .with_transport(mock.clone()); + if disable { + options = options.disable_scrubbing(); + } + let guard = Bugsee::launch_with(options).unwrap(); + Bugsee::capture_message( + LogLevel::Error, + "login failed for carol@corp-example.com password=hunter2 in /home/carol/app", + ); + assert!(Bugsee::flush(Duration::from_secs(5))); + drop(guard); + + let bundles = mock.uploaded_bundles.lock().unwrap(); + assert_eq!(bundles.len(), 1); + let text = all_text(&bundles[0]); + for secret in ["carol@corp-example.com", "hunter2", "/home/carol/"] { + assert_eq!( + text.contains(secret), + expect_leak, + "{secret} (disable={disable})" + ); + } + } +} From 16937c8190078388a7a0dcf9da1096d7c251a1c8 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:41:39 +0500 Subject: [PATCH 09/52] fix(core): arbitrate cross-process claims and session liveness with OS file locks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the pid/start-time claim scheme found it unsound where #14 matters most: containers with separate pid namespaces sharing a volume (both apps pid 1) cannot compare pids or start times, so a peer's live claim was judged stale and its live session reported as a crash. Pid-in-file schemes also needed a lease, a takeover guard with a timer, and identity checks, each with its own race. Replace them with kernel locks: fcntl(F_SETLK) on unix (per process, released on death, not inherited by fork) and LockFileEx on Windows, on dedicated *.lock files (queue/.lock, sessions/.recover.lock, and sessions/.lock held for a session's life). Nothing can go stale, so there is no lease, takeover or start-time logic. An in-process registry (pid-keyed, spin-locked with the holder's pid so a fork child cannot inherit a held lock) excludes threads and keeps a process from ever opening one lock file twice, which POSIX would punish by dropping the lock. - .alive stays a bare pid and lock files are new names, so older builds sharing a directory keep working (no exactly-once between old and new; documented). Markers with no .lock beside them still use the pid probe. - Liveness probe: zombies count as dead (Linux /proc state, macOS proc_pidinfo); Windows uses WaitForSingleObject. - Leftover lock files of finished items and dead sessions are swept when free. - Tests: a live peer whose marker names a colliding pid (parent's, init's) is not recovered until it really dies, then exactly once; concurrent claimers; probing never drops a held lock; zombie probe. Closes #14 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 14 +- crates/bugsee-core/src/claim.rs | 514 +++++++++++++---------- crates/bugsee-core/src/process.rs | 246 ++++------- crates/bugsee-core/src/queue.rs | 11 +- crates/bugsee-core/src/recovery.rs | 79 ++-- crates/bugsee-core/src/session.rs | 29 +- crates/bugsee-core/tests/multiprocess.rs | 46 ++ 7 files changed, 507 insertions(+), 432 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 90bc74b..a2d1009 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -197,9 +197,19 @@ Crash recovery needs the data directory to **outlive the process**: on a contain **Release note, default location moved.** Earlier builds defaulted to `/bugsee/`; the default is now the platform location (3) or the private temp fallback (4). When no directory was requested and the old location exists, is a real directory and is owned by the current user, the launch recovers its crashed sessions and delivers its queue once, best effort (delivery is asynchronous and `flush` does not wait for it). It is not claimed to be lossless: a legacy session whose process is still alive is skipped, and a legacy directory that fails the ownership check is left alone. Hosts that set `data_dir` were never using it. -**Unsafe.** The facade makes two `geteuid()` calls (unix) to name and vet the private temp directory. Core's process probes (`process.rs`) are `kill(pid, 0)` and `proc_pidinfo` (macOS), plus `OpenProcess`/`GetExitCodeProcess`/`GetProcessTimes` on Windows. +**Unsafe.** The facade makes two `geteuid()` calls (unix) to name and vet the private temp directory. Core adds `fcntl(F_SETLK)` (unix) / `LockFileEx` (Windows) for claims, a `kill(pid, 0)` probe (plus `proc_pidinfo` on macOS, `OpenProcess`/`WaitForSingleObject` on Windows) for legacy markers only, and one `UnsafeCell` registry guarded by a pid-stamped spin lock (below). -**Several processes on one directory (#14).** Every pending item is owned by exactly one process at a time: a crashed session by `sessions/.claim`, a queued report by `queue/.claim`. A claim is a file created with `O_EXCL`/`CREATE_NEW` holding `pid start-time nonce`; it is released on drop and, if the holder dies, **taken over** once stale. Stale means: the owner pid is gone, or alive with a different start time (pid reuse; a container's restarted app is pid 1 again), or the file has no owner and is older than 5 s, or it is older than a 30 minute lease (wedged holder, or a platform with no start time). Takeover is serialised by a short-lived `.takeover` guard file (re-checked under the guard), so two takers cannot both remove the other's fresh claim; a guard older than 10 s is abandoned. No OS file lock is used: it would be tied to the open file description, which `fork` duplicates. After winning a session claim the marker is re-checked, because a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s, since a peer's in-flight `enqueue` looks like an orphan (reaping it lost the report). The `.alive` marker also records the start time, so a recycled pid no longer keeps a dead session unrecovered. Delivery is at-least-once: a holder that dies after uploading but before removing the item makes its successor upload it again. Tests: `bugsee-core/tests/multiprocess.rs` (real child processes; racing launchers, SIGKILLed holder) and the unit tests in `claim.rs`, `process.rs`, `queue.rs`. +**Several processes on one directory (#14).** Every pending item is owned by exactly one process at a time, arbitrated by an **OS byte-range lock** on a `*.lock` file: a queued report by `queue/.lock`, a session being recovered by `sessions/.recover.lock`, and a *live session* by `sessions/.lock`, held for the session's whole life. The kernel releases the lock when the process dies however it dies, so nothing can go stale: there is no owner pid, start time, lease or takeover protocol to get wrong. Choices that follow from this: + +- **Pid namespaces.** Containers sharing a volume each see their app as pid 1; a pid (or pid plus start time) written in a file means nothing to the other. A lock does not depend on it, so a peer's live session is never mistaken for a crash and a dead one is never kept alive by pid collision. The `.alive` marker's pid is consulted only for a marker with no `.lock` beside it, i.e. one written by an older SDK build. +- **Why `fcntl`, not `flock`/OFD locks.** `fcntl` locks are per process and are not inherited across `fork`, so a prefork child never holds its parent's claim; `flock`/OFD locks follow the open file description and would. (MSRV 1.88 predates `File::lock` anyway, hence libc/FFI.) +- **Costs of per-process locks, handled in `claim.rs`.** They do not exclude threads, so a registry of held paths does (keyed by pid and spin-locked with the holder's pid stamped in, so a `fork` child that inherits it mid-update breaks the lock instead of deadlocking). POSIX drops a process's locks on a file when *any* descriptor for it closes, so a lock file is never opened twice by one process: the registry is checked before each open, and a claim closes its descriptor before leaving the registry. +- **Ordering at launch.** `Session::begin` creates the marker (`O_EXCL`), takes `.lock`, then writes the pid. Until the pid is written the marker is empty and fresh, which peers skip. +- **Stale scans.** After winning a session claim the marker is re-checked, since a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s: a peer's in-flight `enqueue` looks like an orphan, and reaping it lost the report. Lock files of finished items and dead sessions are swept when free. +- **Compatibility.** The `.alive` marker is still a bare pid and the lock files are new names, so older builds sharing the directory keep working. They do not take part in the locking, so a mixed-version directory (a rolling upgrade) gives no exactly-once guarantee between old and new processes, and a new process treats an old build's session by its pid as before. Run one SDK version per directory for the guarantee. +- **Limits.** Advisory locks on network filesystems are best effort (NFS needs lockd; some mounts do not support them, in which case a claim fails closed: the item is skipped, never double-handled). The only timestamps relied on are the 30 s grace for an empty marker and the 60 s orphan grace, which assume clocks within seconds. Delivery is at-least-once: a holder that dies after uploading but before removing the item (or after enqueueing a recovered report but before discarding the session) makes its successor do it again. Platforms with neither unix nor Windows locks exclude threads only. + +Tests: `bugsee-core/tests/multiprocess.rs` (real child processes: racing launchers, a SIGKILLed holder, a live peer whose marker names a colliding pid) and unit tests in `claim.rs`, `process.rs`, `queue.rs`. Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, a memory-mode case in `bugsee/tests/fork_safety.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised, since mounting one needs root; failures are injected by blocking path components and removing the directory mid-run. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index df40239..ea80538 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -4,184 +4,322 @@ // // Copyright Β© 2026 Bugsee. All rights reserved. // -//! Cross-process exclusive claims on on-disk work items. +//! Cross-process exclusive claims on on-disk work items, and session liveness. //! //! Several processes can share one data directory β€” a prefork server's master -//! and its workers above all β€” and the work items in it (a crashed session to -//! recover, a queued report to upload) must each be handled by exactly one of -//! them, or the backend receives duplicates. A claim is a small file created -//! with `O_EXCL` (`CREATE_NEW` on Windows) that records its owner as -//! `" "`. +//! and its workers, containers sharing a volume β€” and the work items in it (a +//! crashed session to recover, a queued report to upload) must each be handled +//! by exactly one of them, or the backend receives duplicates. //! -//! A claim must never block the work forever if its owner dies, so a stale claim -//! is taken over. It is stale when +//! A claim is an **OS byte-range lock** on a small `*.lock` file: //! -//! * its owner process is gone, or the pid now belongs to a *different* process -//! (the recorded start time no longer matches: pid reuse, or a container's -//! restarted app being pid 1 again), -//! * it has no readable owner and is older than [`UNOWNED_GRACE`] (its creator -//! died between creating the file and writing it), or -//! * it is older than [`MAX_LEASE`], a backstop for a holder that is alive but -//! wedged, or on a platform that cannot tell processes apart. +//! * unix: `fcntl(F_SETLK)`, a *process-scoped* lock. The kernel releases it the +//! instant the process dies, however it dies, and a `fork` child does not +//! inherit it. +//! * Windows: `LockFileEx` on the handle, released when the handle closes or the +//! process exits. //! -//! Taking over is serialised by a short-lived guard file, so two processes that -//! both find the same claim stale cannot each remove the other's fresh claim and -//! both end up holding it. (An OS file lock would release on death automatically, -//! but it is per-open-file-description, not per-process, which makes it unreliable -//! across `fork`.) +//! Because the kernel is the arbiter there is nothing to go stale: no owner pid +//! to probe, no start time, no lease to expire, no takeover protocol. That is +//! also why it is correct across **pid namespaces** (two containers sharing a +//! volume, both with a pid 1) and across pid reuse, where recording a pid in a +//! file is not. The lock file itself may be left behind by a dead holder; it is +//! just an unlocked file, and anyone may lock it. +//! +//! Two costs of process-scoped locks, both handled here: +//! +//! * They do not exclude *threads* of one process, so an in-process registry of +//! held paths does that. +//! * POSIX drops a process's locks on a file when it closes **any** descriptor +//! for that file. So a lock file is never opened twice by one process: the +//! registry is consulted before every open, and a claim closes its descriptor +//! *before* leaving the registry. +//! +//! Advisory locks on network filesystems are best effort (NFS needs lockd). The +//! SDK relies on no file timestamps for claims; the only ones it reads are the +//! grace windows for half-written files in `recovery` and `queue`, which assume +//! clocks that agree to within seconds. //! //! None of this runs on the crash path; it allocates freely. -use std::io::Write; +use std::cell::UnsafeCell; +use std::fs::{File, OpenOptions}; use std::path::{Path, PathBuf}; -use std::time::{Duration, SystemTime}; - -use crate::process; - -/// A claim that has no readable owner is given this long before it is treated as -/// stale: its creator writes the owner a moment after creating the file. -const UNOWNED_GRACE: Duration = Duration::from_secs(5); - -/// No claim is honoured for longer than this, whatever its owner's state. Work -/// items are short (one upload, one recovery), so a claim this old is wedged. -const MAX_LEASE: Duration = Duration::from_secs(30 * 60); - -/// The takeover guard is held for a few syscalls; one older than this was left -/// by a process that died inside the critical section. -const GUARD_TTL: Duration = Duration::from_secs(10); +use std::sync::atomic::{AtomicU32, Ordering}; /// Holds a claim; dropping it releases the claim. #[derive(Debug)] pub struct Claim { path: PathBuf, - /// Exactly what this claim wrote, so release never removes a successor's claim. - content: String, + file: Option, } impl Drop for Claim { fn drop(&mut self) { - if std::fs::read_to_string(&self.path).is_ok_and(|c| c == self.content) { - let _ = std::fs::remove_file(&self.path); - } + // Close (releasing the OS lock) strictly before leaving the registry, so + // a thread that acquires next never has its lock dropped by our close. + drop(self.file.take()); + registry().remove(&self.path); } } -/// Try to claim `path`. `None` if another live process (or another thread of this -/// one) holds it, or if it cannot be claimed. +/// Try to claim `path` (created if absent). `None` if another process, or another +/// thread of this one, holds it, or if it cannot be claimed. pub fn try_claim(path: &Path) -> Option { - let content = owner_record(); - match create(path, &content) { - Create::Done => Some(Claim { - path: path.to_path_buf(), - content, - }), - Create::Exists => take_over(path, content), - // Directory missing / unwritable: nothing can be claimed, and the - // caller's own I/O will fail more usefully than we can here. - Create::Failed => None, + match acquire(path, true) { + Acquire::Got(claim) => Some(claim), + Acquire::Held | Acquire::Unavailable => None, } } -enum Create { - Done, - Exists, - Failed, +/// Whether the lock file at `path` is held by a live process. +pub(crate) enum Probe { + Held, + Free, } -/// Atomically create `path` holding `content` (one `write`, so the owner is -/// visible as soon as the file is). -fn create(path: &Path, content: &str) -> Create { - match std::fs::OpenOptions::new() +/// Is `path` currently claimed? A missing file is free. A file that cannot be +/// inspected is reported held: the conservative answer never disturbs a peer. +pub(crate) fn probe(path: &Path) -> Probe { + if !path.exists() { + return Probe::Free; + } + match acquire(path, false) { + Acquire::Got(claim) => { + drop(claim); + Probe::Free + } + // Vanished since the existence check: nothing holds it. + Acquire::Unavailable if !path.exists() => Probe::Free, + Acquire::Held | Acquire::Unavailable => Probe::Held, + } +} + +/// Remove the lock file at `path` if nobody holds it (a dead holder's leftover). +pub(crate) fn remove_if_free(path: &Path) { + if let Some(claim) = try_claim(path) { + let _ = std::fs::remove_file(path); + drop(claim); + } +} + +enum Acquire { + Got(Claim), + Held, + Unavailable, +} + +fn acquire(path: &Path, create: bool) -> Acquire { + if !registry().insert(path) { + return Acquire::Held; // another thread of this process + } + let file = match OpenOptions::new() + .read(true) .write(true) - .create_new(true) + .create(create) + .truncate(false) .open(path) { - Ok(mut file) => { - let _ = file.write_all(content.as_bytes()); - Create::Done + Ok(f) => f, + Err(_) => { + registry().remove(path); + return Acquire::Unavailable; + } + }; + match os::try_lock(&file) { + Ok(true) => Acquire::Got(Claim { + path: path.to_path_buf(), + file: Some(file), + }), + Ok(false) => { + drop(file); + registry().remove(path); + Acquire::Held + } + Err(_) => { + drop(file); + registry().remove(path); + Acquire::Unavailable } - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Create::Exists, - Err(_) => Create::Failed, } } -/// Replace a stale claim with ours. Only one process at a time may do this: the -/// guard file excludes a second taker between our staleness check and our create, -/// which is exactly the window in which it could otherwise remove the claim we -/// just made. -fn take_over(path: &Path, content: String) -> Option { - let observed = std::fs::read_to_string(path).ok()?; - if !is_stale(path, &observed) { - return None; - } - let guard_path = sibling(path, "takeover"); - match create(&guard_path, "") { - Create::Done => {} - Create::Exists => { - // Held by a peer mid-takeover (it finishes in microseconds, and the - // work is retried on the next drain), or left by one that died. - if age(&guard_path).is_some_and(|a| a > GUARD_TTL) { - let _ = std::fs::remove_file(&guard_path); +/// The paths this process currently holds, keyed by pid so that a `fork` child, +/// which inherits a copy of this memory, ignores the parent's entries. +/// +/// A hand-rolled spin lock rather than a `Mutex`: if another thread held a +/// `Mutex` at the instant of `fork`, the child's copy would stay locked forever +/// and its first claim would deadlock. Here the lock word records the holder's +/// pid, so a child that finds it held by a different pid knows the holder does +/// not exist in this process and breaks it. +struct Registry { + owner: AtomicU32, + held: UnsafeCell>, +} + +// SAFETY: `held` is only touched while `owner` is held (see `with`). +unsafe impl Sync for Registry {} + +static REGISTRY: Registry = Registry { + owner: AtomicU32::new(0), + held: UnsafeCell::new(Vec::new()), +}; + +fn registry() -> &'static Registry { + ®ISTRY +} + +impl Registry { + fn with(&self, f: impl FnOnce(&mut Vec<(u32, PathBuf)>) -> R) -> R { + let pid = std::process::id(); + loop { + match self + .owner + .compare_exchange(0, pid, Ordering::Acquire, Ordering::Relaxed) + { + Ok(_) => break, + Err(holder) if holder != pid => { + // Held by a pid that is not us: a parent's thread at the time + // of `fork`. It does not exist here. Take the lock, and + // discard (leak, never drop) the vector it may have left + // half-modified. + if self + .owner + .compare_exchange(holder, pid, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + // SAFETY: we now hold the lock; the old value is leaked + // because its allocator state may be mid-update. + unsafe { std::ptr::write(self.held.get(), Vec::new()) }; + break; + } + } + Err(_) => std::thread::yield_now(), } - return None; } - Create::Failed => return None, + // SAFETY: exclusive while `owner == pid` (held above). + let out = f(unsafe { &mut *self.held.get() }); + self.owner.store(0, Ordering::Release); + out } - let _guard = Guard(guard_path); - // Re-check under the guard: a peer may have completed its own takeover since - // we looked, and what is there now is a live claim, not the stale one. - if std::fs::read_to_string(path).ok().as_deref() != Some(observed.as_str()) { - return None; + + /// Record `path` as held by this process; `false` if it already is. + fn insert(&self, path: &Path) -> bool { + let pid = std::process::id(); + self.with(|held| { + if held.iter().any(|(p, q)| *p == pid && q == path) { + return false; + } + // Entries of a different pid are a forked-from parent's; drop them. + held.retain(|(p, _)| *p == pid); + held.push((pid, path.to_path_buf())); + true + }) } - let _ = std::fs::remove_file(path); - match create(path, &content) { - Create::Done => Some(Claim { - path: path.to_path_buf(), - content, - }), - // A fresh creator slipped in between our remove and create: it holds it. - Create::Exists | Create::Failed => None, + + fn remove(&self, path: &Path) { + let pid = std::process::id(); + self.with(|held| held.retain(|(p, q)| !(*p == pid && q == path))); } } -struct Guard(PathBuf); -impl Drop for Guard { - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.0); +#[cfg(unix)] +mod os { + use std::fs::File; + use std::io; + use std::os::unix::io::AsRawFd; + + /// Take a non-blocking exclusive lock on byte 0. `Ok(false)` if a peer holds it. + pub fn try_lock(file: &File) -> io::Result { + // SAFETY: an all-zero `flock` is a valid plain-data value. + let mut fl: libc::flock = unsafe { std::mem::zeroed() }; + fl.l_type = libc::F_WRLCK as _; + fl.l_whence = libc::SEEK_SET as _; + fl.l_start = 0; + fl.l_len = 1; + loop { + // SAFETY: `fd` is a valid open descriptor and `fl` a valid `flock`. + let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETLK, &fl) }; + if ret == 0 { + return Ok(true); + } + let err = io::Error::last_os_error(); + match err.raw_os_error() { + Some(libc::EAGAIN) | Some(libc::EACCES) => return Ok(false), + Some(libc::EINTR) => continue, + _ => return Err(err), + } + } } } -fn sibling(path: &Path, ext: &str) -> PathBuf { - let mut s = path.as_os_str().to_os_string(); - s.push("."); - s.push(ext); - PathBuf::from(s) -} +#[cfg(windows)] +mod os { + use std::ffi::c_void; + use std::fs::File; + use std::io; + use std::os::windows::io::AsRawHandle; -/// `" "` for this process; the nonce makes every claim's -/// content unique, so "is this still my claim" is a content comparison. -fn owner_record() -> String { - let (pid, start) = - process::parse_token(&process::self_token()).unwrap_or((std::process::id(), None)); - let start = start.map_or_else(|| "-".to_string(), |s| s.to_string()); - format!("{pid} {start} {}", crate::util::random_hex(8)) -} + const LOCKFILE_FAIL_IMMEDIATELY: u32 = 1; + const LOCKFILE_EXCLUSIVE_LOCK: u32 = 2; + const ERROR_LOCK_VIOLATION: i32 = 33; -fn age(path: &Path) -> Option { - let modified = std::fs::metadata(path).and_then(|m| m.modified()).ok()?; - SystemTime::now().duration_since(modified).ok() -} + #[repr(C)] + struct Overlapped { + internal: usize, + internal_high: usize, + offset: u32, + offset_high: u32, + event: *mut c_void, + } -fn is_stale(path: &Path, content: &str) -> bool { - let age = age(path); - if age.is_some_and(|a| a > MAX_LEASE) { - return true; + extern "system" { + fn LockFileEx( + file: *mut c_void, + flags: u32, + reserved: u32, + len_low: u32, + len_high: u32, + overlapped: *mut Overlapped, + ) -> i32; + } + + /// Take a non-blocking exclusive lock on byte 0. `Ok(false)` if a peer holds it. + pub fn try_lock(file: &File) -> io::Result { + let mut ov = Overlapped { + internal: 0, + internal_high: 0, + offset: 0, + offset_high: 0, + event: std::ptr::null_mut(), + }; + // SAFETY: the handle is valid for the call and `ov` outlives it. + let ok = unsafe { + LockFileEx( + file.as_raw_handle(), + LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, + 0, + 1, + 0, + &mut ov, + ) + }; + if ok != 0 { + return Ok(true); + } + let err = io::Error::last_os_error(); + match err.raw_os_error() { + Some(ERROR_LOCK_VIOLATION) => Ok(false), + _ => Err(err), + } } - match process::parse_token(content) { - // A live owner with the recorded identity (including another thread of - // this very process) holds it. - Some((pid, start)) => !process::owner_is_alive(pid, start), - None => age.is_some_and(|a| a > UNOWNED_GRACE), +} + +/// No file locks: claims exclude threads of this process only. +#[cfg(not(any(unix, windows)))] +mod os { + pub fn try_lock(_file: &std::fs::File) -> std::io::Result { + Ok(true) } } @@ -195,122 +333,58 @@ mod tests { p } - fn backdate(path: &Path, by: Duration) { - let f = std::fs::OpenOptions::new().write(true).open(path).unwrap(); - f.set_modified(SystemTime::now() - by).unwrap(); - } - - /// This process's own live owner record, as a peer would have written it. - fn live_owner() -> String { - format!("{} nonce", process::self_token()) - } - #[test] fn a_claim_is_exclusive_and_released_on_drop() { let dir = tmp(); - let path = dir.join("x.claim"); + let path = dir.join("x.lock"); let first = try_claim(&path).expect("first claim"); assert!(try_claim(&path).is_none(), "already held"); + assert!(matches!(probe(&path), Probe::Held)); drop(first); + assert!(matches!(probe(&path), Probe::Free)); assert!(try_claim(&path).is_some(), "free again after release"); } #[test] - fn a_claim_whose_owner_is_dead_is_taken_over() { - let dir = tmp(); - let path = dir.join("x.claim"); - // A pid that cannot be alive: far above any real pid range. - std::fs::write(&path, "4294967290").unwrap(); - assert!(try_claim(&path).is_some(), "stale claim must be reclaimed"); - assert!(!sibling(&path, "takeover").exists(), "guard released"); - } - - #[test] - fn a_claim_held_by_a_live_process_is_respected() { - let dir = tmp(); - let path = dir.join("x.claim"); - std::fs::write(&path, live_owner()).unwrap(); - assert!(try_claim(&path).is_none()); - // Old-format owner (pid only), pid 1: alive on unix. - #[cfg(unix)] - { - std::fs::write(&path, "1").unwrap(); - assert!(try_claim(&path).is_none()); - } - } - - #[test] - fn a_recycled_pid_does_not_keep_a_dead_owners_claim_alive() { - let Some(start) = process::start_time(std::process::id()) else { - return; // platform cannot tell processes apart - }; - let dir = tmp(); - let path = dir.join("x.claim"); - // Our own (live) pid, but with a start time that is not ours: the claim - // was written by an earlier process that had this pid. - std::fs::write( - &path, - format!("{} {} stale", std::process::id(), start.wrapping_add(1)), - ) - .unwrap(); - assert!(try_claim(&path).is_some(), "recycled pid must not block"); - } - - #[test] - fn a_freshly_created_unowned_claim_is_not_stolen_but_an_old_one_is() { - let dir = tmp(); - let path = dir.join("x.claim"); - std::fs::write(&path, "").unwrap(); // creator has not written its owner yet - assert!(try_claim(&path).is_none()); - backdate(&path, UNOWNED_GRACE + Duration::from_secs(5)); - assert!(try_claim(&path).is_some(), "creator died before writing"); - } - - #[test] - fn a_wedged_holder_cannot_hold_a_claim_past_the_lease() { + fn a_leftover_lock_file_from_a_dead_holder_is_just_free() { let dir = tmp(); - let path = dir.join("x.claim"); - std::fs::write(&path, live_owner()).unwrap(); - assert!(try_claim(&path).is_none()); - backdate(&path, MAX_LEASE + Duration::from_secs(60)); + let path = dir.join("x.lock"); + // Contents are irrelevant: a dead holder's pid, even one that collides + // with a live process (here, our own), must not block anything. + std::fs::write(&path, std::process::id().to_string()).unwrap(); + assert!(matches!(probe(&path), Probe::Free)); assert!(try_claim(&path).is_some()); } #[test] - fn releasing_a_claim_never_removes_its_successors() { + fn a_missing_lock_file_probes_free_and_is_not_created() { let dir = tmp(); - let path = dir.join("x.claim"); - let first = try_claim(&path).unwrap(); - // First is judged wedged and replaced behind its back. - std::fs::write(&path, live_owner()).unwrap(); - drop(first); - assert!(path.exists(), "the successor's claim must survive"); + let path = dir.join("nope.lock"); + assert!(matches!(probe(&path), Probe::Free)); + assert!(!path.exists()); } #[test] - fn an_abandoned_takeover_guard_does_not_block_forever() { + fn dropping_one_claim_never_releases_another() { + // POSIX drops a process's locks on a file when ANY descriptor for it + // closes; the registry must keep a second open from ever happening. let dir = tmp(); - let path = dir.join("x.claim"); - std::fs::write(&path, "4294967290").unwrap(); - let guard = sibling(&path, "takeover"); - std::fs::write(&guard, "").unwrap(); - // A taker is (apparently) mid-takeover: back off. + let path = dir.join("x.lock"); + let held = try_claim(&path).unwrap(); assert!(try_claim(&path).is_none()); - // It died there long ago: clear the guard, then succeed. - backdate(&guard, GUARD_TTL + Duration::from_secs(5)); + assert!(matches!(probe(&path), Probe::Held)); assert!( try_claim(&path).is_none(), - "clearing the guard is its own step" + "probing must not have closed our lock" ); - assert!(try_claim(&path).is_some()); + drop(held); } #[test] - fn racing_takers_of_one_stale_claim_yield_exactly_one_holder() { - for _ in 0..200 { + fn racing_claimers_of_one_path_yield_exactly_one_holder() { + for _ in 0..100 { let dir = tmp(); - let path = dir.join("x.claim"); - std::fs::write(&path, "4294967290").unwrap(); + let path = dir.join("x.lock"); let barrier = std::sync::Arc::new(std::sync::Barrier::new(8)); let handles: Vec<_> = (0..8) .map(|_| { @@ -318,7 +392,7 @@ mod tests { std::thread::spawn(move || { barrier.wait(); let claim = try_claim(&path); - // Keep the claim held until every thread has tried. + // Hold until every thread has tried. barrier.wait(); claim.is_some() }) @@ -329,7 +403,7 @@ mod tests { .map(|h| h.join().unwrap()) .filter(|held| *held) .count(); - assert_eq!(holders, 1, "exactly one taker may win"); + assert_eq!(holders, 1, "exactly one claimer may win"); let _ = std::fs::remove_dir_all(&dir); } } diff --git a/crates/bugsee-core/src/process.rs b/crates/bugsee-core/src/process.rs index c37f583..c857e46 100644 --- a/crates/bugsee-core/src/process.rs +++ b/crates/bugsee-core/src/process.rs @@ -4,24 +4,19 @@ // // Copyright Β© 2026 Bugsee. All rights reserved. // -//! Process identity and liveness, for deciding whether the owner recorded in a -//! claim or a liveness marker is still the process that wrote it. +//! Process liveness probe, used only for **legacy** session markers. //! -//! A bare pid is not an identity: pids are recycled, and in a container the -//! restarted app is routinely pid 1 again. An *owner token* therefore pairs the -//! pid with the process's **start time** (an opaque, stable per-process number), -//! so a recycled pid is told apart from the original owner. Where the platform -//! offers no start time the token carries the pid alone and liveness degrades to -//! "a process with that pid exists". +//! Current sessions and claims are arbitrated by OS file locks (see `claim`), +//! which need no pid and are correct across pid namespaces. A session marker +//! written by an older SDK build has no lock beside it, so for those alone the +//! recorded pid is probed here. The probe is inherently single-namespace. -use std::sync::OnceLock; - -/// Whether a process with `pid` currently exists. +/// Whether a process with `pid` currently exists and has not terminated. A +/// zombie (exited, not yet reaped by its parent) counts as dead. /// /// Unix: `kill(pid, 0)`; `0` (permitted) or `EPERM` (exists, not permitted) both -/// mean alive, `ESRCH` means gone. Windows: the process can be opened and has not -/// exited. A pid-reuse false positive is closed by [`owner_is_alive`]'s start -/// time comparison, not here. +/// mean alive, `ESRCH` means gone. Windows: the process can be opened and its +/// handle is not yet signalled. #[cfg(unix)] pub(crate) fn is_alive(pid: u32) -> bool { if pid == 0 { @@ -30,167 +25,93 @@ pub(crate) fn is_alive(pid: u32) -> bool { // SAFETY: `kill` with signal 0 performs only an error/permission check and // has no side effects; it is safe to call with any pid value. let ret = unsafe { libc::kill(pid as libc::pid_t, 0) }; - ret == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) -} - -#[cfg(windows)] -pub(crate) fn is_alive(pid: u32) -> bool { - win::with_process(pid, |handle| { - let mut code = 0u32; - // SAFETY: `handle` is a valid open process handle for the closure's duration. - let ok = unsafe { win::GetExitCodeProcess(handle, &mut code) }; - ok == 0 || code == win::STILL_ACTIVE - }) - // Access denied means the process exists but belongs to someone else. - .unwrap_or_else(|denied| denied) -} - -#[cfg(not(any(unix, windows)))] -pub(crate) fn is_alive(_pid: u32) -> bool { - false + (ret == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)) + && !is_zombie(pid) } -/// An opaque per-process start time for `pid`, or `None` if it cannot be read -/// (platform without support, process gone, or not permitted). Only ever compared -/// for equality against a value read the same way. -#[cfg(target_os = "linux")] -pub(crate) fn start_time(pid: u32) -> Option { - let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; - // `pid (comm) state ...`: comm may contain spaces and parens, so split after - // the LAST ')'. `starttime` is field 22; `state` (field 3) is index 0 here. - let rest = &stat[stat.rfind(')')? + 1..]; - rest.split_whitespace().nth(19)?.parse().ok() +#[cfg(any(target_os = "linux", target_os = "android"))] +fn is_zombie(pid: u32) -> bool { + let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + // `pid (comm) state ...`: comm may contain spaces and parens, so look after + // the LAST ')'. + stat.rfind(')') + .and_then(|i| stat[i + 1..].split_whitespace().next()) + .is_some_and(|state| state == "Z" || state == "X") } #[cfg(target_os = "macos")] -pub(crate) fn start_time(pid: u32) -> Option { - // SAFETY: an all-zero `proc_bsdinfo` is a valid plain-data value, and +fn is_zombie(pid: u32) -> bool { + // The "short" BSD info, unlike the full one, is still available for a zombie. + // SAFETY: an all-zero `proc_bsdshortinfo` is valid plain data, and // `proc_pidinfo` writes at most `size_of` bytes into it. - let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; - let size = std::mem::size_of::() as libc::c_int; + let mut info: libc::proc_bsdshortinfo = unsafe { std::mem::zeroed() }; + let size = std::mem::size_of::() as libc::c_int; let n = unsafe { libc::proc_pidinfo( pid as libc::c_int, - libc::PROC_PIDTBSDINFO, + libc::PROC_PIDT_SHORTBSDINFO, 0, - (&mut info as *mut libc::proc_bsdinfo).cast(), + (&mut info as *mut libc::proc_bsdshortinfo).cast(), size, ) }; - (n == size).then(|| info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec) -} - -#[cfg(windows)] -pub(crate) fn start_time(pid: u32) -> Option { - win::with_process(pid, |handle| { - let mut ft = [win::FileTime::default(); 4]; - // SAFETY: `handle` is valid; the four out-pointers point into `ft`. - let [creation, exit, kernel, user] = &mut ft; - let ok = unsafe { win::GetProcessTimes(handle, creation, exit, kernel, user) }; - (ok != 0).then(|| (u64::from(ft[0].high) << 32) | u64::from(ft[0].low)) - }) - .ok() - .flatten() + const SZOMB: u32 = 5; + // Observed on macOS: for an exited-but-unreaped child `kill(pid, 0)` still + // succeeds while `proc_pidinfo` returns 0 (no info), whereas any live process, + // including other users', reports its info. So "no info" means zombie here. + n == 0 || (n == size && info.pbsi_status == SZOMB) } -#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] -pub(crate) fn start_time(_pid: u32) -> Option { - None +#[cfg(all( + unix, + not(any(target_os = "linux", target_os = "android", target_os = "macos")) +))] +fn is_zombie(_pid: u32) -> bool { + false } -/// This process's owner token, `""` or `" "`. Cached, but keyed -/// by pid: a `fork` child inherits the cache and must not reuse the parent's token. -pub(crate) fn self_token() -> String { - static CACHE: OnceLock<(u32, String)> = OnceLock::new(); - let pid = std::process::id(); - let (cached_pid, token) = CACHE.get_or_init(|| (pid, make_token(pid))); - if *cached_pid == pid { - token.clone() - } else { - make_token(pid) +#[cfg(windows)] +pub(crate) fn is_alive(pid: u32) -> bool { + if pid == 0 { + return false; } -} - -fn make_token(pid: u32) -> String { - match start_time(pid) { - Some(start) => format!("{pid} {start}"), - None => pid.to_string(), + // SAFETY: plain Win32 calls; the handle is closed on every path. + unsafe { + let handle = win::OpenProcess(win::SYNCHRONIZE, 0, pid); + if handle.is_null() { + // Access denied means the process exists but belongs to someone else. + return std::io::Error::last_os_error().raw_os_error() + == Some(win::ERROR_ACCESS_DENIED); + } + // A process handle is signalled once the process has terminated. + let alive = win::WaitForSingleObject(handle, 0) == win::WAIT_TIMEOUT; + win::CloseHandle(handle); + alive } } -/// Parse an owner token (`""` or `" "`; trailing fields, such as -/// a claim's nonce, are ignored). `None` if no pid can be read. -pub(crate) fn parse_token(text: &str) -> Option<(u32, Option)> { - let mut it = text.split_whitespace(); - let pid = it.next()?.parse().ok()?; - let start = it.next().and_then(|s| s.parse().ok()); - Some((pid, start)) -} - -/// Whether the process an owner token names is still alive *and is the same -/// process that wrote the token*. With a recorded start time, a live process -/// whose start time differs is a recycled pid, i.e. the owner is gone. -pub(crate) fn owner_is_alive(pid: u32, recorded_start: Option) -> bool { - if !is_alive(pid) { - return false; - } - match (recorded_start, start_time(pid)) { - (Some(recorded), Some(now)) => recorded == now, - // Cannot compare (old-format token, or start time unreadable): the pid - // exists, so give the owner the benefit of the doubt. - _ => true, - } +#[cfg(not(any(unix, windows)))] +pub(crate) fn is_alive(_pid: u32) -> bool { + false } #[cfg(windows)] mod win { //! The few kernel32 calls needed, declared directly: kernel32 is always - //! linked, so this avoids a dependency for four functions. + //! linked, so this avoids a dependency for three functions. use std::ffi::c_void; pub type Handle = *mut c_void; - pub const STILL_ACTIVE: u32 = 259; - const PROCESS_QUERY_LIMITED_INFORMATION: u32 = 0x1000; - const ERROR_ACCESS_DENIED: i32 = 5; - - #[repr(C)] - #[derive(Clone, Copy, Default)] - pub struct FileTime { - pub low: u32, - pub high: u32, - } + pub const SYNCHRONIZE: u32 = 0x0010_0000; + pub const WAIT_TIMEOUT: u32 = 258; + pub const ERROR_ACCESS_DENIED: i32 = 5; extern "system" { - fn OpenProcess(access: u32, inherit: i32, pid: u32) -> Handle; - fn CloseHandle(handle: Handle) -> i32; - pub fn GetExitCodeProcess(handle: Handle, code: *mut u32) -> i32; - pub fn GetProcessTimes( - handle: Handle, - creation: *mut FileTime, - exit: *mut FileTime, - kernel: *mut FileTime, - user: *mut FileTime, - ) -> i32; - } - - /// Run `f` with a query handle to `pid`. `Err(true)` if the process exists but - /// access was denied, `Err(false)` if it cannot be opened (gone). - pub fn with_process(pid: u32, f: impl FnOnce(Handle) -> T) -> Result { - if pid == 0 { - return Err(false); - } - // SAFETY: plain Win32 calls; the handle is closed on every path. - unsafe { - let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); - if handle.is_null() { - let denied = - std::io::Error::last_os_error().raw_os_error() == Some(ERROR_ACCESS_DENIED); - return Err(denied); - } - let out = f(handle); - CloseHandle(handle); - Ok(out) - } + pub fn OpenProcess(access: u32, inherit: i32, pid: u32) -> Handle; + pub fn CloseHandle(handle: Handle) -> i32; + pub fn WaitForSingleObject(handle: Handle, millis: u32) -> u32; } } @@ -199,34 +120,23 @@ mod tests { use super::*; #[test] - fn own_process_is_alive_with_a_stable_start_time() { + fn own_process_is_alive_and_an_impossible_pid_is_not() { assert!(is_alive(std::process::id())); - let a = start_time(std::process::id()); - assert_eq!(a, start_time(std::process::id())); - #[cfg(any(target_os = "linux", target_os = "macos", windows))] - assert!(a.is_some(), "start time must be readable on this platform"); + assert!(!is_alive(0)); + assert!(!is_alive(4_294_967_290)); } + #[cfg(unix)] #[test] - fn a_pid_with_the_wrong_start_time_is_a_recycled_pid() { - let me = std::process::id(); - if let Some(start) = start_time(me) { - assert!(owner_is_alive(me, Some(start))); - assert!(!owner_is_alive(me, Some(start.wrapping_add(1)))); + fn a_zombie_is_dead() { + let mut child = std::process::Command::new("true").spawn().unwrap(); + let pid = child.id(); + // Wait for it to exit without reaping it (a non-consuming probe). + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while is_alive(pid) && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(10)); } - assert!(owner_is_alive(me, None), "no recorded start: pid alone"); - assert!(!owner_is_alive(4_294_967_290, Some(1))); - } - - #[test] - fn tokens_round_trip() { - assert_eq!(parse_token("12 345"), Some((12, Some(345)))); - assert_eq!(parse_token("12"), Some((12, None))); - assert_eq!(parse_token("12 345 deadbeef"), Some((12, Some(345)))); - assert_eq!(parse_token("junk"), None); - assert_eq!(parse_token(""), None); - let (pid, start) = parse_token(&self_token()).unwrap(); - assert_eq!(pid, std::process::id()); - assert_eq!(start, start_time(pid)); + assert!(!is_alive(pid), "exited but unreaped child counts as dead"); + let _ = child.wait(); } } diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index 11e5254..804b95d 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -35,15 +35,15 @@ impl QueuedReport { fn endpoint_path(&self) -> PathBuf { sibling(&self.bundle, "endpoint") } - fn claim_path(&self) -> PathBuf { - sibling(&self.bundle, "claim") + fn lock_path(&self) -> PathBuf { + sibling(&self.bundle, "lock") } } /// Claim a queued report for delivery, so that of several processes draining one /// shared queue exactly one uploads it. Released when the guard drops. pub fn try_claim(report: &QueuedReport) -> Option { - crate::claim::try_claim(&report.claim_path()) + crate::claim::try_claim(&report.lock_path()) } fn sibling(bundle: &Path, ext: &str) -> PathBuf { @@ -151,6 +151,9 @@ pub fn remove(report: &QueuedReport) { let _ = std::fs::remove_file(report.req_path()); let _ = std::fs::remove_file(report.meta_path()); let _ = std::fs::remove_file(report.endpoint_path()); + // The delivering claimant's own lock file (removing a held lock file is fine: + // the item is gone, so no one will look for it). + let _ = std::fs::remove_file(report.lock_path()); } /// The bundle file's base name (for reconstructing an [`AssembledReport`]). @@ -249,6 +252,8 @@ pub fn gc_orphans(data_dir: &Path) { .strip_suffix(".req") .or_else(|| name.strip_suffix(".meta")) .or_else(|| name.strip_suffix(".endpoint")) + .or_else(|| name.strip_suffix(".lock")) + // Leftovers of an earlier claim scheme. .or_else(|| name.strip_suffix(".claim")) .or_else(|| name.strip_suffix(".claim.takeover")) .or_else(|| name.strip_suffix(".tmp")); diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 0262252..51c6a36 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -80,34 +80,18 @@ pub fn find_pending(data_dir: &Path, current_generation: u64) -> Vec= current_generation { continue; } - // Decide from the marker's recorded owner pid whether this is a real - // crash or a still-running shared-dir peer. - let owner = std::fs::read_to_string(&marker).unwrap_or_default(); - let owner = owner.trim(); - if owner.is_empty() { - // Present but not yet populated: `Session::begin` creates the marker - // (O_EXCL) and writes the pid a moment later, so a peer launching - // concurrently can observe this empty window. Skip only while the - // marker is FRESH β€” a peer writes its pid within microseconds. An empty - // marker older than the grace window is a session that died in the - // createβ†’write gap (or whose pid write silently failed on a full - // disk); recover it so its crash is surfaced and its generation - // reclaimed instead of being re-scanned and leaked on every future - // launch (F28). No pid β‡’ no liveness check; fall through to recover. - if marker_is_fresh(&marker) { - continue; - } - } else if let Some((pid, start)) = crate::process::parse_token(owner) { - // Skip a marker still owned by a live *other* process. Our own pid is - // never skipped (a live process cannot share our pid, so a marker - // bearing it is a dead prior incarnation β€” recover it). The marker - // may carry the owner's start time, which tells a recycled pid (a - // dead owner whose pid something else now uses) from the owner. - if pid != self_pid && crate::process::owner_is_alive(pid, start) { + // A session whose `.lock` exists is arbitrated by that OS lock: held + // means a live process (in any pid namespace), free means it died. The + // marker's pid is not consulted, since it is only meaningful inside the + // writer's own pid namespace and can be reused. + let lock = crate::session::Session::lock_path(data_dir, generation); + if lock.exists() { + if matches!(crate::claim::probe(&lock), crate::claim::Probe::Held) { continue; } + } else if !legacy_marker_is_dead(&marker, self_pid) { + continue; } - // Non-empty but unparseable content β‡’ the owner is gone; recover. let parts_dir = data_dir.join("parts").join(generation.to_string()); let minidump = exists_opt(parts_dir.join(MINIDUMP_NAME)); let crash_info = exists_opt(parts_dir.join(CRASH_INFO_NAME)); @@ -127,6 +111,31 @@ fn exists_opt(p: PathBuf) -> Option { p.exists().then_some(p) } +/// Whether a marker with no lock file beside it (written by an older SDK build, +/// or by a session that could not lock) is a crashed session rather than a live +/// peer, judged from its recorded pid. Only meaningful within one pid namespace. +fn legacy_marker_is_dead(marker: &Path, self_pid: u32) -> bool { + let owner = std::fs::read_to_string(marker).unwrap_or_default(); + let owner = owner.trim(); + if owner.is_empty() { + // Present but not yet populated: `Session::begin` creates the marker + // (O_EXCL) and writes the pid a moment later, so a peer launching + // concurrently can observe this empty window. Skip only while the + // marker is FRESH. An empty marker older than the grace window is a + // session that died in the create-to-write gap (or whose pid write failed + // on a full disk); recover it so its crash is surfaced and its + // generation reclaimed instead of being re-scanned on every launch. + return !marker_is_fresh(marker); + } + match owner.parse::() { + // Our own pid is never skipped (a live process cannot share our pid, so a + // marker bearing it is a dead prior incarnation). + Ok(pid) => pid == self_pid || !crate::process::is_alive(pid), + // Non-empty but unparseable content: the owner is gone. + Err(_) => true, + } +} + /// Grace window for an empty (pid-not-yet-written) liveness marker: a launching /// peer writes its pid within microseconds of the O_EXCL create, so a marker /// still empty after this is a dead session, not a live peer mid-launch. @@ -332,7 +341,7 @@ pub fn claim_pending(data_dir: &Path, pending: &PendingSession) -> Option() { - if generation < current_generation && !alive(generation) { - drop(crate::claim::try_claim(&entry.path())); - } + // Lock files of a generation with no marker: the session lock of a + // crashed process (its marker already recovered) and the recovery + // claim. Removed only if free, so a live holder's is left alone. + let lock_gen = name + .strip_suffix(".recover.lock") + .or_else(|| name.strip_suffix(".lock")) + .and_then(|g| g.parse::().ok()); + if let Some(generation) = lock_gen { + if generation < current_generation && !alive(generation) { + crate::claim::remove_if_free(&entry.path()); } } } diff --git a/crates/bugsee-core/src/session.rs b/crates/bugsee-core/src/session.rs index 5a81692..8d75c8c 100644 --- a/crates/bugsee-core/src/session.rs +++ b/crates/bugsee-core/src/session.rs @@ -27,6 +27,12 @@ const SESSIONS_DIR: &str = "sessions"; pub struct Session { data_dir: PathBuf, generation: u64, + /// The OS lock on `.lock`, held for the session's life. Peers sharing the + /// data dir (including from another pid namespace) probe it to tell a live + /// session from a crashed one; the kernel drops it when this process dies. + /// `None` if the lock file could not be taken (the bare pid in the marker is + /// then all a peer has). + _lock: Option, } impl Session { @@ -38,7 +44,7 @@ impl Session { std::fs::create_dir_all(&markers)?; let mut candidate = floor_generation(data_dir) + 1; - let generation = loop { + let (generation, lock) = loop { let marker = markers.join(format!("{candidate}.alive")); match std::fs::OpenOptions::new() .write(true) @@ -46,10 +52,13 @@ impl Session { .open(&marker) { Ok(mut f) => { - // " ": the start time lets recovery tell this - // process from a later one that reuses the pid. - let _ = write!(f, "{}", crate::process::self_token()); - break candidate; + // Lock BEFORE writing the pid: the marker is empty (and fresh, + // so peers skip it) until then, which closes the window in + // which it is neither locked nor attributed. + let lock = crate::claim::try_claim(&Session::lock_path(data_dir, candidate)); + // The marker stays a bare pid, readable by older builds. + let _ = write!(f, "{}", std::process::id()); + break (candidate, lock); } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { candidate += 1; @@ -64,6 +73,7 @@ impl Session { Ok(Session { data_dir: data_dir.to_path_buf(), generation, + _lock: lock, }) } @@ -79,9 +89,18 @@ impl Session { .join(format!("{generation}.alive")) } + /// The liveness lock-file path for `generation`. Its presence marks a session + /// whose liveness is arbitrated by the lock rather than the marker's pid. + pub(crate) fn lock_path(data_dir: &Path, generation: u64) -> PathBuf { + data_dir + .join(SESSIONS_DIR) + .join(format!("{generation}.lock")) + } + /// Mark a clean shutdown: remove this session's liveness marker. pub fn end(&self) { let _ = std::fs::remove_file(Session::marker_path(&self.data_dir, self.generation)); + let _ = std::fs::remove_file(Session::lock_path(&self.data_dir, self.generation)); } } diff --git a/crates/bugsee-core/tests/multiprocess.rs b/crates/bugsee-core/tests/multiprocess.rs index 348f50c..9b2ae1e 100644 --- a/crates/bugsee-core/tests/multiprocess.rs +++ b/crates/bugsee-core/tests/multiprocess.rs @@ -94,6 +94,13 @@ fn child_entry() { std::fs::rename(dir.join("READY.tmp"), dir.join("READY")).unwrap(); std::thread::sleep(Duration::from_secs(120)); } + "session" => { + // A live session of its own, as a peer's launch would create. + let session = bugsee_core::session::Session::begin(&dir).unwrap(); + std::fs::write(dir.join("READY.tmp"), session.generation().to_string()).unwrap(); + std::fs::rename(dir.join("READY.tmp"), dir.join("READY")).unwrap(); + std::thread::sleep(Duration::from_secs(120)); + } other => panic!("unknown role {other}"), } } @@ -243,3 +250,42 @@ fn a_holder_killed_mid_work_does_not_block_the_others() { } assert_eq!(delivered(&log).len(), 7, "log: {:?}", delivered(&log)); } + +/// Containers with separate pid namespaces share a volume, and each app is pid 1, +/// so a peer's recorded pid can name a live process HERE that is not the peer. +/// The marker's pid must therefore play no part in deciding the peer is dead. +#[test] +fn a_live_peers_session_is_not_mistaken_for_a_crash_whatever_pid_its_marker_names() { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + std::fs::create_dir_all(&dir).unwrap(); + let log = tmp.0.join("delivered.log"); + + let mut peer = spawn("session", &dir, &log); + let ready = dir.join("READY"); + let deadline = Instant::now() + Duration::from_secs(20); + while !ready.exists() { + assert!(Instant::now() < deadline, "peer never started"); + std::thread::sleep(Duration::from_millis(10)); + } + let generation: u64 = std::fs::read_to_string(&ready).unwrap().parse().unwrap(); + + // Make the marker name a pid that is alive in THIS namespace and is not the + // peer: the parent's own, then init's. Neither may make the peer look dead, + // nor make it look alive once it is gone. + let marker = dir.join("sessions").join(format!("{generation}.alive")); + for foreign in [std::process::id().to_string(), "1".to_string()] { + std::fs::write(&marker, &foreign).unwrap(); + run_workers(&dir, &log, 3); + let _ = std::fs::remove_file(dir.join("GO")); + assert!(delivered(&log).is_empty(), "live peer reported as a crash"); + assert!(marker.exists(), "live peer's marker was reclaimed"); + } + + // Once the peer really dies, its session is recovered exactly once, even + // though the marker still names a live pid. + peer.kill().unwrap(); + let _ = peer.wait(); + run_workers(&dir, &log, 3); + assert_eq!(delivered(&log), vec!["recovered-crash".to_string()]); +} From c462eef01676faaa8b986f645f56bd8b97279e82 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:42:20 +0500 Subject: [PATCH 10/52] fix: harden the termination hook and shutdown after review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Detect a handler installed after ours (tokio, signal-hook) by asking the kernel what is in force; flush only, never terminate the host. - A stop no longer waits on the uploader to end the session cleanly (an HTTP call cannot be interrupted); delivery is documented at-least-once. - Fork-child re-install ordering, SIG_IGN left alone, watcher-alive and pending-signal tracking, host sa_mask/SA_ONSTACK/SA_NODEFER/SA_RESETHAND carried over, fork-aware install lock, pipe2, clean rollback on failure. - flush_at_exit is unix only and skipped in fork children. - Flush aborts between deliveries on stop (memory and every queue dir). - Tests: late/SA_SIGINFO/ignored handlers, SIGINT, double signal, fork child, hanging transport; no fixed sleeps; temp dirs cleaned. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 7 +- crates/bugsee-core/src/runtime.rs | 16 +- crates/bugsee/src/api.rs | 49 +++- crates/bugsee/src/options.rs | 5 +- crates/bugsee/src/termination.rs | 276 ++++++++++++--------- crates/bugsee/tests/shutdown.rs | 393 ++++++++++++++++++++++++------ 6 files changed, 548 insertions(+), 198 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index c411abf..600d79a 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -201,17 +201,18 @@ Crash recovery needs the data directory to **outlive the process**: on a contain Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, a memory-mode case in `bugsee/tests/fork_safety.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised, since mounting one needs root; failures are injected by blocking path components and removing the directory mid-run. -**Shutdown and flush contract (issue #8).** `flush(timeout)` persists everything captured before the call and tries to upload every queued report; it returns `true` only if the outbound queue is empty, leaves the SDK running, and is idempotent. `stop()` (and the launch guard's `Drop`) persists, then ends the session by removing the liveness marker; it never uploads, so an undelivered report stays in the durable queue and goes out next launch (disk mode; in memory mode `flush` is the only guarantee). `shutdown(timeout)` is `flush` then `stop`. A `stop` arriving during a flush aborts the flush instead of waiting out its deadline, so the 2 s acknowledgement bound holds and the session still ends cleanly. The marker is intentionally left when a thread does not acknowledge in time, so a wedged shutdown is recovered, not lost. None of it is async-signal-safe; in a `fork` child it acts on the child's own revived recorder. +**Shutdown and flush contract (issue #8).** `flush(timeout)` persists everything captured before the call and tries to upload every queued report; it returns `true` only if the outbound queue is empty, leaves the SDK running, and is idempotent. `stop()` (and the launch guard's `Drop`) persists, then ends the session by removing the liveness marker; it never uploads, so an undelivered report stays in the durable queue and goes out next launch (disk mode; in memory mode `flush` is the only guarantee). `shutdown(timeout)` is `flush` then `stop`. A `stop` arriving during a flush aborts the flush between deliveries instead of waiting out its deadline. The uploader's acknowledgement does not gate ending the session (it writes no session parts and may sit in an HTTP call for tens of seconds), so a stop during a slow upload still ends cleanly and the abandoned report is re-sent next launch: delivery is **at-least-once**, and a graceful stop plus restart reports nothing twice only in the absence of a kill mid-upload. Worst case a `shutdown(t)` takes `t` plus about 4 s. The marker is intentionally left when a thread does not acknowledge in time, so a wedged shutdown is recovered, not lost. None of it is async-signal-safe; in a `fork` child it acts on the child's own revived recorder. | Exit path | Marker | Reports | Next launch | |---|---|---|---| | return from `main` / `stop` / `shutdown` | removed | persisted, queue intact | no abnormal-exit; queue delivered once | | `process::exit` / `exit()` | left (default) | as already written | one abnormal-exit report; queue delivered once | -| same, with `LaunchOptions::flush_at_exit` | removed by an `atexit` handler | persisted | as the first row | +| same, with `LaunchOptions::flush_at_exit` (unix only; ignored on Windows, call `shutdown`) | removed by an `atexit` handler (not run in `fork` children) | persisted | as the first row | | `SIGTERM`/`SIGINT` + `Bugsee::install_termination_flush` | removed if the signal would have terminated the process | flushed, bounded | as the first row; host handler still called | +| Windows Ctrl+C/Break with the hook and no host handler | left (flush only) | flushed, bounded | one abnormal-exit report; queue delivered | | `SIGTERM`/`SIGINT` without the hook, `SIGKILL`, OOM, power loss | left | records appended as drained survive | one abnormal-exit report | -*Decision: `exit` without `stop` stays an abnormal exit.* From inside the process an `exit` is indistinguishable from a runtime tearing down after a fault the host swallowed, so the default errs towards reporting; hosts that exit on purpose opt in with `flush_at_exit` (or call `shutdown`). *Decision: the signal hook is opt-in and chaining.* The handler only writes the signal number to a non-blocking self-pipe and then calls the previously installed handler; a watcher thread does the flush. With a default previous disposition it runs `shutdown`, restores `SIG_DFL` and re-raises, so the exit status is unchanged; a second signal during the shutdown terminates at once. With a host handler or `SIG_IGN` it flushes only and the host decides. A `fork` child inherits the handler but not the watcher: it chains/terminates as if the hook were absent until `install_termination_flush` is called again there. Windows: a console control handler (Ctrl+C/Break flush only, since an earlier handler may swallow them; close/logoff/shutdown flush and stop) that returns `FALSE` to pass the event on; service-control stop is not a console event and needs a `shutdown` call from the host's stop handler. **Not compiled or run on Windows** (a macOS cross-check stops at the C dependencies of `zstd-sys`/`ring`); the Windows half was reviewed by reading only. +*Decision: `exit` without `stop` stays an abnormal exit.* From inside the process an `exit` is indistinguishable from a runtime tearing down after a fault the host swallowed, so the default errs towards reporting; hosts that exit on purpose opt in with `flush_at_exit` (or call `shutdown`). *Decision: the signal hook is opt-in and chaining.* The handler only writes the signal number to a non-blocking self-pipe and then calls the previously installed handler; a watcher thread does the flush. With a default previous disposition it runs `shutdown`, restores `SIG_DFL` and re-raises, so the exit status is unchanged; a second signal during the shutdown terminates at once. With a host handler it flushes only and the host decides; a host handler installed AFTER ours (tokio, `signal-hook`) is detected by asking the kernel what is in force, and is treated the same, so we never kill a host mid-graceful-shutdown. A signal ignored at install is left alone. A second signal while the first is served, or a dead watcher, terminates at once. The chained call keeps the host's `sa_mask`, `SA_ONSTACK`, `SA_NODEFER`, `SA_SIGINFO`, and `SA_RESETHAND` (emulated). Do not call `exit` from a host handler with `flush_at_exit` on: the exit-time `shutdown` would run in signal context. A `fork` child inherits the handler but not the watcher: it chains/terminates as if the hook were absent until `install_termination_flush` is called again there. Windows: a console control handler (Ctrl+C/Break flush only, since an earlier handler may swallow them; close/logoff/shutdown flush and stop) that returns `FALSE` to pass the event on; service-control stop is not a console event and needs a `shutdown` call from the host's stop handler. **Not compiled or run on Windows** (a macOS cross-check stops at the C dependencies of `zstd-sys`/`ring`); the Windows half was reviewed by reading only. Tests: `bugsee/tests/shutdown.rs` (a real child process per exit path, then a relaunch asserting exactly-once delivery and the abnormal-exit count). diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 3f04ad0..98aa984 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -648,7 +648,7 @@ impl Drop for Recorder { let _ = handle.join(); } } - // Only mark a genuinely clean shutdown. If either thread merely DETACHED + // Only mark a genuinely clean shutdown. If the capture worker merely DETACHED // (didn't ack in time β€” e.g. a slow flush), it may still be writing this // session's data, so removing the marker now would make next-launch // recovery miss the generation entirely if the process then dies mid-tail. @@ -666,7 +666,13 @@ impl Drop for Recorder { // A panic that was merely *contained* never reaches here while // unwinding, and its snapshot is deleted by `report_caught`, so this // cannot turn a handled panic into a phantom crash. - if worker_acked && uploader_acked && !std::thread::panicking() { + // The uploader is deliberately NOT part of this condition: it never + // writes session parts, and it can be inside an HTTP call that no signal + // can interrupt (connect/read timeouts are tens of seconds). Waiting on + // it would turn every stop during a slow upload into a false + // abnormal-exit report. The cost is at-least-once delivery: a report + // whose upload was cut short is re-sent next launch. + if worker_acked && !std::thread::panicking() { if let Some(session) = &self.session { session.end(); } @@ -1621,7 +1627,7 @@ fn drain_memory( // Each entry is tried once per call, whatever its outcome. let mut tried: Vec = Vec::new(); loop { - if deadline.is_some_and(|d| Instant::now() >= d) { + if deadline.is_some_and(|d| Instant::now() >= d) || shared.stopping.load(Ordering::SeqCst) { break; } let now = epoch_ms(); @@ -1717,7 +1723,7 @@ fn drain_until_empty( } let now = epoch_ms(); drain_memory(shared, backoff_base, true, false, Some(start + deadline)); - for &data_dir in dirs { + 'dirs: for &data_dir in dirs { for report in queue::list_pending(data_dir) { let Some(_claim) = queue::try_claim(&report) else { continue; // a peer is delivering it; the loop below waits it out @@ -1737,7 +1743,7 @@ fn drain_until_empty( false, ); if start.elapsed() >= deadline || shared.stopping.load(Ordering::SeqCst) { - break; + break 'dirs; } } } diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 5b8bfa9..0054159 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -11,7 +11,7 @@ //! and after stop. use std::io; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use std::time::Duration; use bugsee_core::errors::{self, Cause}; @@ -271,15 +271,20 @@ impl Bugsee { /// | `SIGTERM`/`SIGINT` without the hook, `SIGKILL`, OOM kill, power loss | Records are appended as the worker drains them, so completed records and queued reports survive; the last few in flight may not | One abnormal-exit report, carrying the recovered window | /// | Fatal signal, aborting panic, unwinding panic out of `main` | See the crash capture docs | One crash report | /// - /// Nothing is reported twice: a stopped session's marker is gone, and a - /// recovered one is consumed when its report is queued. In memory mode + /// A stopped session's marker is gone and a recovered one is consumed when + /// its report is queued, so a graceful stop followed by a restart reports + /// nothing twice. Delivery itself is **at-least-once**: a process killed + /// while a report's upload was in flight may send it again next launch. In + /// memory mode /// (see [`StorageMode`]) nothing survives the process, so `flush` before /// exiting is the only delivery guarantee. /// /// Bounds: `stop` waits at most 2 s each for the capture worker and the - /// uploader to acknowledge. If one does not (a host callback wedged), the - /// marker is deliberately left, so the session is recovered rather than - /// lost. It is not async-signal-safe and must not be called from a signal + /// uploader to acknowledge. If the worker does not (a host callback + /// wedged), the marker is deliberately left, so the session is recovered + /// rather than lost. The uploader does not gate the marker: an upload in + /// flight (an HTTP call can outlast the 2 s) is abandoned and its report + /// re-sent next launch. It is not async-signal-safe and must not be called from a signal /// handler (use [`Bugsee::install_termination_flush`]). In a `fork` child /// it acts on that child's own recorder. pub fn stop() { @@ -304,8 +309,8 @@ impl Bugsee { /// the session cleanly whatever the flush returned. Returns the flush /// result: `true` if every report was delivered. `false` is not a loss β€” /// undelivered reports stay in the durable queue for the next launch (in - /// disk mode) β€” and also covers "not launched". Idempotent. Total time is - /// bounded by `timeout` plus the bounds documented on `stop`. + /// disk mode) β€” and also covers "not launched". Idempotent. Worst-case time is + /// `timeout` plus about 4 s (the two 2 s acknowledgement bounds on `stop`). pub fn shutdown(timeout: Duration) -> bool { let flushed = Self::flush(timeout); Self::stop(); @@ -340,9 +345,18 @@ impl Bugsee { /// `timeout`. A service-control stop is not a console event and is not /// covered; call [`Bugsee::shutdown`] from your stop handler. /// + /// + /// If the signal is ignored (`SIG_IGN`) when this is called, it is left + /// alone. If the host installs its own handler **after** this call, the SDK + /// notices (it asks the kernel what is in force) and only flushes, never + /// terminating the process; a host handler that itself calls `exit` while + /// [`LaunchOptions::flush_at_exit`] is on would run `shutdown` in signal + /// context and can deadlock, so end the process from a normal thread. + /// /// Idempotent; a second call only updates `timeout`. Works before `launch` /// (it flushes nothing until there is something to flush). Choose `timeout` - /// below your orchestrator's grace period (Kubernetes defaults to 30 s). + /// below your orchestrator's grace period (Kubernetes defaults to 30 s) by + /// at least the ~4 s stop bounds. pub fn install_termination_flush(timeout: Duration) -> io::Result<()> { crate::termination::install(timeout) } @@ -565,24 +579,36 @@ impl Bugsee { /// Exit-time flush timeout in milliseconds, `0` when not requested. static EXIT_FLUSH_MS: AtomicU64 = AtomicU64::new(0); +/// The process that registered the `atexit` handler. A `fork` child inherits +/// the registration, and running `shutdown` there would revive the SDK in a +/// process that may be mid-`fork`-and-`exec`; the child must do its own stop. +static EXIT_FLUSH_PID: AtomicU32 = AtomicU32::new(0); +#[cfg(unix)] extern "C" { fn atexit(callback: extern "C" fn()) -> std::os::raw::c_int; } +#[cfg(unix)] extern "C" fn flush_at_exit() { let ms = EXIT_FLUSH_MS.load(Ordering::SeqCst); - if ms > 0 { + if ms > 0 && EXIT_FLUSH_PID.load(Ordering::SeqCst) == std::process::id() { let _ = std::panic::catch_unwind(|| Bugsee::shutdown(Duration::from_millis(ms))); } } /// Arm (or, with `None`, disarm) the `atexit` handler. libc cannot unregister /// it, so it is registered once per process and consults the timeout. +/// +/// Unix only: on Windows `std::process::exit` calls `ExitProcess` directly and +/// never runs CRT `atexit` handlers, and in a `cdylib` such a handler would run +/// at DLL detach under the loader lock. Windows hosts call `shutdown`. +#[cfg(unix)] fn register_exit_flush(timeout: Option) { static REGISTERED: AtomicBool = AtomicBool::new(false); let ms = timeout.map_or(0, |t| t.as_millis().clamp(1, u64::MAX as u128) as u64); EXIT_FLUSH_MS.store(ms, Ordering::SeqCst); + EXIT_FLUSH_PID.store(std::process::id(), Ordering::SeqCst); if ms > 0 && !REGISTERED.swap(true, Ordering::SeqCst) { // SAFETY: registering a plain `extern "C"` function. unsafe { @@ -591,6 +617,9 @@ fn register_exit_flush(timeout: Option) { } } +#[cfg(not(unix))] +fn register_exit_flush(_timeout: Option) {} + /// If this process was `fork()`ed since the SDK last looked, rebuild what the /// fork did not carry over. Cheap on the hot path: one atomic load. /// diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index 03b057b..975b739 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -137,7 +137,10 @@ impl LaunchOptions { /// /// It registers a libc `atexit` handler, so it does not run on `abort`, a /// fatal signal or `_exit` β€” those stay abnormal terminations, as they - /// should. Caveat: a panic that unwinds out of `main` while the launch + /// should β€” nor in a `fork` child (which must stop for itself). **Unix + /// only**: on Windows `process::exit` never runs CRT `atexit` handlers, so + /// the option is ignored there; call [`crate::Bugsee::shutdown`] before + /// exiting. Caveat: a panic that unwinds out of `main` while the launch /// guard has been leaked (`mem::forget`, a `static`) would be treated as a /// deliberate exit; keep the guard as a local in `main`. /// diff --git a/crates/bugsee/src/termination.rs b/crates/bugsee/src/termination.rs index 476aa7a..c3bc176 100644 --- a/crates/bugsee/src/termination.rs +++ b/crates/bugsee/src/termination.rs @@ -16,23 +16,31 @@ //! work on an ordinary thread, where taking locks, allocating and joining //! threads are all fine: //! -//! - the previous disposition was the default (terminate): `shutdown` (flush, -//! then a clean stop, because the exit is deliberate), restore the default -//! disposition and re-raise the signal, so the process ends exactly as it -//! would have β€” same exit status β€” only later; -//! - the previous disposition was a host handler, or "ignore": flush only. The -//! host decides whether and how to exit; it ends the session cleanly by -//! dropping the launch guard (or calling `stop`) on its way out. +//! - the previous disposition was the default (terminate) and ours is still in +//! force: `shutdown` (flush, then a clean stop, because the exit is +//! deliberate), restore the default disposition and re-raise the signal, so +//! the process ends exactly as it would have β€” same exit status β€” only later; +//! - anything else β€” the previous disposition was a host handler, or the host +//! installed its own handler AFTER ours (tokio, `signal-hook`), which is +//! detected by asking the kernel what is in force before acting: flush only, +//! never terminate. The host decides whether and how to exit; it ends the +//! session cleanly by dropping the launch guard (or calling `stop`); +//! - a signal that was ignored at install time (`SIG_IGN`: a background job, +//! `nohup`) is left alone and never armed. //! -//! A second signal while the default-disposition shutdown is still running -//! terminates immediately: an operator pressing Ctrl+C twice is not made to -//! wait out the flush timeout. +//! A second terminating signal while the first is still being served, or any +//! signal when the watcher is gone (or in a `fork` child), terminates at once: +//! an operator pressing Ctrl+C twice is not made to wait out the flush timeout. //! //! # Windows //! //! A console control handler. Windows runs it on a thread of its own, so it //! flushes in place, then returns `FALSE` so the next handler (the host's, or -//! the default) still runs. +//! the default) still runs. Close, logoff and shutdown also stop the SDK, as +//! the process ends regardless. Ctrl+C and Ctrl+Break only flush: with no host +//! handler the default one ends the process without a clean stop, so the next +//! launch reports an abnormal exit (an earlier handler may instead swallow the +//! event and keep running, which a stop here would wrongly cut short). use std::io; use std::time::Duration; @@ -44,54 +52,63 @@ pub(crate) fn install(timeout: Duration) -> io::Result<()> { #[cfg(unix)] mod imp { - use std::cell::UnsafeCell; use std::io; - use std::mem::MaybeUninit; - use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, Ordering}; - use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, AtomicUsize, Ordering}; use std::time::Duration; + use bugsee_core::fork::ForkMutex; use libc::{c_int, c_void}; use crate::Bugsee; const SIGNALS: [c_int; 2] = [libc::SIGTERM, libc::SIGINT]; - /// The dispositions in force before ours, indexed like [`SIGNALS`]. - /// - /// Written only by `install`, before the `sigaction` that arms the handler, - /// and read by the handler afterwards β€” the same discipline the native crash - /// handler uses for its table. - struct Previous(UnsafeCell<[MaybeUninit; 2]>); - // SAFETY: see above; there is no concurrent write once the handler is armed. - unsafe impl Sync for Previous {} - static PREVIOUS: Previous = Previous(UnsafeCell::new([MaybeUninit::uninit(); 2])); + /// What was in force before ours for one signal. Atomics, because the + /// handler itself may reset the handler (`SA_RESETHAND` emulation). The + /// host's signal mask is not kept here: it is installed as ours. + struct Slot { + handler: AtomicUsize, + flags: AtomicI32, + } + + #[allow(clippy::declare_interior_mutable_const)] + const SLOT: Slot = Slot { + handler: AtomicUsize::new(libc::SIG_DFL), + flags: AtomicI32::new(0), + }; + static SLOTS: [Slot; 2] = [SLOT; 2]; /// Write end of the self-pipe; `-1` until installed. static PIPE_W: AtomicI32 = AtomicI32::new(-1); /// The process that owns the watcher thread. A `fork` child inherits the - /// handler but not the thread. + /// handler but not the thread. Stored AFTER `PIPE_W` (release) and loaded + /// before it (acquire), so an owner always sees its own pipe. static OWNER_PID: AtomicU32 = AtomicU32::new(0); + /// Whether the watcher thread is running to act on a pipe byte. + static WATCHER_ALIVE: AtomicBool = AtomicBool::new(false); + /// Terminating signals seen since the watcher last handled one. A second + /// one means the operator is impatient: die at once. + static PENDING: AtomicU32 = AtomicU32::new(0); static TIMEOUT_MS: AtomicU64 = AtomicU64::new(0); - /// Set by the watcher once a terminating shutdown has begun. - static SHUTTING_DOWN: AtomicBool = AtomicBool::new(false); - /// Serialises `install` (never taken by the handler). - static INSTALL: Mutex = Mutex::new(false); + /// Whether the handler is armed. Serialises `install`; the lock is never + /// taken by the handler, and being a fork-aware lock a child never inherits + /// it held by a vanished thread. + static INSTALL: ForkMutex = ForkMutex::new(|| false); fn index(sig: c_int) -> Option { SIGNALS.iter().position(|s| *s == sig) } - /// The disposition that was in force before ours for `SIGNALS[idx]`. - /// - /// # Safety - /// The table must have been filled in (the handler only runs after that). - unsafe fn previous(idx: usize) -> libc::sigaction { - unsafe { (*PREVIOUS.0.get())[idx].assume_init() } - } - - fn is_default(prev: &libc::sigaction) -> bool { - prev.sa_sigaction == libc::SIG_DFL + /// Whether our handler is still the one in force for `sig`. If not, someone + /// installed over us (the host after launch, e.g. tokio or `signal-hook`) and + /// owns the signal: we then only flush and never terminate the process. + fn current_is_ours(sig: c_int) -> bool { + // SAFETY: querying with a null new-action; async-signal-safe. + unsafe { + let mut cur: libc::sigaction = core::mem::zeroed(); + libc::sigaction(sig, core::ptr::null(), &mut cur) == 0 + && cur.sa_sigaction == handler as *const () as usize + } } /// Restore the default disposition and send ourselves `sig`. Every call is @@ -123,25 +140,26 @@ mod imp { /// call into the previous handler. extern "C" fn handler(sig: c_int, info: *mut libc::siginfo_t, ctx: *mut c_void) { // SAFETY: errno access and the calls below are async-signal-safe; the - // table is filled in before this handler is armed. + // slot is filled in before this handler is armed. unsafe { let saved_errno = *errno_location(); let Some(idx) = index(sig) else { return }; - let prev = previous(idx); - let prev_default = is_default(&prev); - let owner = libc::getpid() as u32 == OWNER_PID.load(Ordering::Relaxed); - - if !owner { - // A fork child: no watcher thread here. Behave as if we were not - // installed. - if prev_default { - die(sig); - } - } else { - if prev_default && SHUTTING_DOWN.load(Ordering::SeqCst) { + let slot = &SLOTS[idx]; + let prev = slot.handler.load(Ordering::SeqCst); + let prev_default = prev == libc::SIG_DFL; + let owner = OWNER_PID.load(Ordering::Acquire) == libc::getpid() as u32; + let live = owner && WATCHER_ALIVE.load(Ordering::SeqCst); + + if prev_default && current_is_ours(sig) { + // Nobody to flush (a fork child, or the watcher is gone), or a + // second signal while the first is still being served: behave + // as if we were not installed. + if !live || PENDING.fetch_add(1, Ordering::SeqCst) >= 1 { die(sig); } - let fd = PIPE_W.load(Ordering::Relaxed); + } + if live { + let fd = PIPE_W.load(Ordering::Acquire); if fd >= 0 { let byte = sig as u8; // A full pipe means wake-ups are already pending: ignore. @@ -149,13 +167,19 @@ mod imp { } } - if !prev_default && prev.sa_sigaction != libc::SIG_IGN { - if prev.sa_flags & libc::SA_SIGINFO != 0 { + if !prev_default && prev != libc::SIG_IGN { + let flags = slot.flags.load(Ordering::SeqCst); + if flags & libc::SA_RESETHAND != 0 { + // The host's one-shot handler has now fired: from here on + // the previous disposition is the default again. + slot.handler.store(libc::SIG_DFL, Ordering::SeqCst); + } + if flags & libc::SA_SIGINFO != 0 { let f: extern "C" fn(c_int, *mut libc::siginfo_t, *mut c_void) = - core::mem::transmute(prev.sa_sigaction); + core::mem::transmute(prev); f(sig, info, ctx); } else { - let f: extern "C" fn(c_int) = core::mem::transmute(prev.sa_sigaction); + let f: extern "C" fn(c_int) = core::mem::transmute(prev); f(sig); } } @@ -163,7 +187,16 @@ mod imp { } } + /// Clears [`WATCHER_ALIVE`] however the watcher thread ends. + struct AliveGuard; + impl Drop for AliveGuard { + fn drop(&mut self) { + WATCHER_ALIVE.store(false, Ordering::SeqCst); + } + } + fn watcher(read_fd: c_int) { + let _alive = AliveGuard; let mut byte = 0u8; loop { // SAFETY: reading one byte into a local. @@ -179,19 +212,22 @@ mod imp { } let sig = byte as c_int; let Some(idx) = index(sig) else { continue }; - // SAFETY: the table is filled in before the pipe can carry anything. - let prev = unsafe { previous(idx) }; let timeout = Duration::from_millis(TIMEOUT_MS.load(Ordering::Relaxed)); - if is_default(&prev) { + let prev_default = SLOTS[idx].handler.load(Ordering::SeqCst) == libc::SIG_DFL; + if prev_default && current_is_ours(sig) { // The process is about to end because of this signal: finish // cleanly, then let it end the way it would have. - SHUTTING_DOWN.store(true, Ordering::SeqCst); let _ = std::panic::catch_unwind(|| Bugsee::shutdown(timeout)); - die(sig); - // Still here: the signal did not kill us (blocked or ignored - // elsewhere). Carry on serving later signals. - SHUTTING_DOWN.store(false, Ordering::SeqCst); + // The host may have taken the signal over meanwhile. + if current_is_ours(sig) { + die(sig); + } + // Still here: the signal did not kill us (blocked or taken over). + // Carry on serving later signals. + PENDING.store(0, Ordering::SeqCst); } else { + // Host-handled (its handler was before ours, or replaced ours): + // the host decides whether and how to exit. let _ = std::panic::catch_unwind(|| Bugsee::flush(timeout)); } } @@ -201,23 +237,34 @@ mod imp { fn start_watcher() -> io::Result<()> { let mut fds = [0 as c_int; 2]; // SAFETY: `fds` has room for the two descriptors `pipe` writes. - if unsafe { libc::pipe(fds.as_mut_ptr()) } != 0 { - return Err(io::Error::last_os_error()); + unsafe { + #[cfg(any(target_os = "linux", target_os = "android"))] + if libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC) != 0 { + return Err(io::Error::last_os_error()); + } + #[cfg(not(any(target_os = "linux", target_os = "android")))] + { + if libc::pipe(fds.as_mut_ptr()) != 0 { + return Err(io::Error::last_os_error()); + } + for fd in fds { + libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC); + } + } } let (r, w) = (fds[0], fds[1]); - // SAFETY: fcntl on descriptors this function owns. + // SAFETY: fcntl on a descriptor this function owns. The handler must + // never block on a full pipe (the read end stays blocking). unsafe { - for fd in [r, w] { - libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC); - } - // The handler must never block on a full pipe. let flags = libc::fcntl(w, libc::F_GETFL); libc::fcntl(w, libc::F_SETFL, flags | libc::O_NONBLOCK); } + WATCHER_ALIVE.store(true, Ordering::SeqCst); let spawned = std::thread::Builder::new() .name("bugsee-termination".into()) .spawn(move || watcher(r)); if let Err(e) = spawned { + WATCHER_ALIVE.store(false, Ordering::SeqCst); // SAFETY: closing the descriptors created above. unsafe { libc::close(r); @@ -227,60 +274,71 @@ mod imp { } // The old write end (a fork child's inherited copy, or none) is left // open on purpose: closing a descriptor the handler may be using races. - OWNER_PID.store(std::process::id(), Ordering::SeqCst); - PIPE_W.store(w, Ordering::SeqCst); + PIPE_W.store(w, Ordering::Release); + OWNER_PID.store(std::process::id(), Ordering::Release); Ok(()) } pub(super) fn install(timeout: Duration) -> io::Result<()> { - let mut installed = INSTALL.lock().unwrap_or_else(|e| e.into_inner()); + let mut installed = INSTALL.lock(); TIMEOUT_MS.store( timeout.as_millis().min(u64::MAX as u128) as u64, Ordering::Relaxed, ); + // First install, a retry after a failed one, or a fork child that + // inherited the handler without its watcher thread. + if OWNER_PID.load(Ordering::Acquire) != std::process::id() { + PENDING.store(0, Ordering::SeqCst); + start_watcher()?; + } if *installed { - // Already armed. In a fork child the watcher thread is gone: start one. - if OWNER_PID.load(Ordering::SeqCst) != std::process::id() { - SHUTTING_DOWN.store(false, Ordering::SeqCst); - start_watcher()?; - } return Ok(()); } - // Record what is in force now, BEFORE arming. + let mut armed: Vec<(c_int, libc::sigaction)> = Vec::new(); for (i, sig) in SIGNALS.iter().enumerate() { - // SAFETY: querying with a null new-action fills the old one. - unsafe { + // SAFETY: querying, then arming, with zeroed + filled structs. The + // slot is filled in before the `sigaction` that arms the handler. + let result = unsafe { let mut cur: libc::sigaction = core::mem::zeroed(); if libc::sigaction(*sig, core::ptr::null(), &mut cur) != 0 { - return Err(io::Error::last_os_error()); - } - (*PREVIOUS.0.get())[i] = MaybeUninit::new(cur); - } - } - start_watcher()?; - - for (i, sig) in SIGNALS.iter().enumerate() { - // SAFETY: arming our handler with a zeroed + filled struct. - unsafe { - let prev = previous(i); - let mut sa: libc::sigaction = core::mem::zeroed(); - sa.sa_sigaction = handler as *const () as usize; - // Keep the restart semantics the host chose; with no host - // handler there is nothing to preserve, and EINTR on every - // blocking call of every thread would only be a side effect of - // ours. - let host_handler = !is_default(&prev) && prev.sa_sigaction != libc::SIG_IGN; - sa.sa_flags = libc::SA_SIGINFO - | if !host_handler || prev.sa_flags & libc::SA_RESTART != 0 { - libc::SA_RESTART + Err(io::Error::last_os_error()) + } else if cur.sa_sigaction == libc::SIG_IGN { + // Ignored on purpose (a shell running us in the background, + // `nohup`): that is the host's choice, leave it. + continue; + } else { + let slot = &SLOTS[i]; + slot.handler.store(cur.sa_sigaction, Ordering::SeqCst); + slot.flags.store(cur.sa_flags, Ordering::SeqCst); + + let host_handler = cur.sa_sigaction != libc::SIG_DFL; + let mut sa: libc::sigaction = core::mem::zeroed(); + sa.sa_sigaction = handler as *const () as usize; + // The host's mask, stack and restart choices carry over to + // the chained call. With no host handler there is nothing to + // preserve, and EINTR on every blocking call of every thread + // would only be a side effect of ours. + sa.sa_flags = libc::SA_SIGINFO + | (cur.sa_flags & (libc::SA_RESTART | libc::SA_ONSTACK | libc::SA_NODEFER)) + | if host_handler { 0 } else { libc::SA_RESTART }; + sa.sa_mask = cur.sa_mask; + if libc::sigaction(*sig, &sa, core::ptr::null_mut()) != 0 { + Err(io::Error::last_os_error()) } else { - 0 - }; - libc::sigemptyset(&mut sa.sa_mask); - if libc::sigaction(*sig, &sa, core::ptr::null_mut()) != 0 { - return Err(io::Error::last_os_error()); + armed.push((*sig, cur)); + Ok(()) + } + } + }; + if let Err(e) = result { + // Leave nothing half-armed, or a retry would record our own + // handler as "the previous one" and recurse into itself. + for (sig, old) in armed { + // SAFETY: restoring what was read above. + unsafe { libc::sigaction(sig, &old, core::ptr::null_mut()) }; } + return Err(e); } } *installed = true; diff --git a/crates/bugsee/tests/shutdown.rs b/crates/bugsee/tests/shutdown.rs index 3a8c54c..1993060 100644 --- a/crates/bugsee/tests/shutdown.rs +++ b/crates/bugsee/tests/shutdown.rs @@ -6,7 +6,8 @@ // //! The shutdown contract, per exit path (see `Bugsee::stop`): what is delivered //! by the dying process, what is recovered on the next launch, and that nothing -//! is reported twice or lost. +//! is lost. Delivery is at-least-once; these cases avoid killing a process +//! mid-upload, so they can assert exactly-once. //! //! Each case runs a real child process (this same test binary, re-entered //! through `child_entry`), ends it the way the case says, then relaunches the SDK @@ -31,15 +32,24 @@ use zip::ZipArchive; const KIND_ENV: &str = "BUGSEE_SHUTDOWN_KIND"; const DIR_ENV: &str = "BUGSEE_SHUTDOWN_DIR"; const FAIL_ENV: &str = "BUGSEE_SHUTDOWN_FAIL_UPLOADS"; +const SLOW_ENV: &str = "BUGSEE_SHUTDOWN_SLOW_UPLOADS"; +const TERM_MS_ENV: &str = "BUGSEE_SHUTDOWN_TERM_MS"; const PENDING: &str = "pending-report-marker"; +const ABNORMAL: &str = "terminated abnormally"; static SERIAL: Mutex<()> = Mutex::new(()); +fn serial() -> std::sync::MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|e| e.into_inner()) +} + /// Writes every uploaded bundle to a file so the parent can see what a child -/// delivered; optionally fails every upload (an unreachable backend). +/// delivered; optionally fails every upload (an unreachable backend) or hangs +/// in it (a backend that accepts the connection and never answers). struct FileTransport { dir: PathBuf, fail: bool, + slow: bool, n: AtomicUsize, } @@ -48,7 +58,10 @@ impl Transport for FileTransport { Ok("token".into()) } fn create_issue(&self, _: &str, _: Option<&str>, _: &[u8]) -> Result { - if self.fail { + if self.slow { + std::thread::sleep(Duration::from_secs(20)); + } + if self.fail || self.slow { return Err(TransportError::Transient("backend unreachable".into())); } Ok("https://uploads.example/x".into()) @@ -86,21 +99,65 @@ fn delivered_by_child(dir: &Path) -> Vec { // --------------------------------------------------------------------------- static HOST_HANDLER_RAN: AtomicBool = AtomicBool::new(false); +/// What was in force when the "late" host handler took the signal over. +static LATE_PREVIOUS: AtomicUsize = AtomicUsize::new(0); extern "C" fn host_handler(_: libc::c_int) { HOST_HANDLER_RAN.store(true, Ordering::SeqCst); } -fn install_host_handler(sig: libc::c_int) { +extern "C" fn host_siginfo_handler( + sig: libc::c_int, + info: *mut libc::siginfo_t, + _: *mut libc::c_void, +) { + // SAFETY: the kernel passes a valid siginfo to an SA_SIGINFO handler. + if !info.is_null() && unsafe { (*info).si_signo } == sig { + HOST_HANDLER_RAN.store(true, Ordering::SeqCst); + } +} + +/// A host that took the signal over after the SDK (tokio, `signal-hook`) and +/// chains to the previous handler, which is the SDK's. +extern "C" fn late_chaining_handler( + sig: libc::c_int, + info: *mut libc::siginfo_t, + ctx: *mut libc::c_void, +) { + HOST_HANDLER_RAN.store(true, Ordering::SeqCst); + let prev = LATE_PREVIOUS.load(Ordering::SeqCst); + if prev > 1 { + // SAFETY: the SDK armed its handler with SA_SIGINFO. + let f: extern "C" fn(libc::c_int, *mut libc::siginfo_t, *mut libc::c_void) = + unsafe { std::mem::transmute(prev) }; + f(sig, info, ctx); + } +} + +fn set_action(sig: libc::c_int, handler: usize, flags: libc::c_int) -> usize { // SAFETY: a zeroed sigaction filled in with a plain handler. unsafe { let mut sa: libc::sigaction = std::mem::zeroed(); - sa.sa_sigaction = host_handler as *const () as usize; + sa.sa_sigaction = handler; + sa.sa_flags = flags; libc::sigemptyset(&mut sa.sa_mask); - assert_eq!(libc::sigaction(sig, &sa, std::ptr::null_mut()), 0); + let mut old: libc::sigaction = std::mem::zeroed(); + assert_eq!(libc::sigaction(sig, &sa, &mut old), 0); + old.sa_sigaction } } +fn wait_for_file(path: &Path, limit: Duration) -> bool { + let start = Instant::now(); + while !path.exists() { + if start.elapsed() > limit { + return false; + } + std::thread::sleep(Duration::from_millis(10)); + } + true +} + /// Not a test by itself: the body of every child. A plain run does nothing. #[test] fn child_entry() { @@ -108,51 +165,149 @@ fn child_entry() { return; }; let dir = PathBuf::from(std::env::var(DIR_ENV).unwrap()); - let fail = std::env::var(FAIL_ENV).is_ok(); let ready = || std::fs::write(dir.join("ready"), b"1").unwrap(); + let term_ms: u64 = std::env::var(TERM_MS_ENV) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(2000); - if kind == "sigterm_host_handler" { - install_host_handler(libc::SIGTERM); + match kind.as_str() { + "sigterm_host_handler" => { + set_action(libc::SIGTERM, host_handler as *const () as usize, 0); + } + "sigterm_siginfo_handler" => { + set_action( + libc::SIGTERM, + host_siginfo_handler as *const () as usize, + libc::SA_SIGINFO, + ); + } + "sigterm_ignored" => { + set_action(libc::SIGTERM, libc::SIG_IGN, 0); + } + _ => {} } let mut options = LaunchOptions::new("APP_TOKEN") .data_dir(dir.join("data")) .native_crash_capture(false) .with_transport(Arc::new(FileTransport { dir: dir.clone(), - fail, + fail: std::env::var(FAIL_ENV).is_ok(), + slow: std::env::var(SLOW_ENV).is_ok(), n: AtomicUsize::new(0), })); if kind == "exit_flush" { options = options.flush_at_exit(Duration::from_secs(2)); } let guard = Bugsee::launch_with(options).expect("launch"); - if kind.starts_with("sigterm") && kind != "sigterm_no_hook" { - Bugsee::install_termination_flush(Duration::from_secs(2)).expect("install"); - // Idempotent. - Bugsee::install_termination_flush(Duration::from_secs(2)).expect("install twice"); + let hooked = kind.starts_with("sigterm") || kind.starts_with("fork"); + if hooked && kind != "sigterm_no_hook" { + let t = Duration::from_millis(term_ms); + Bugsee::install_termination_flush(t).expect("install"); + Bugsee::install_termination_flush(t).expect("install twice (idempotent)"); + } + if kind == "sigterm_late_handler" { + let old = set_action( + libc::SIGTERM, + late_chaining_handler as *const () as usize, + libc::SA_SIGINFO, + ); + LATE_PREVIOUS.store(old, Ordering::SeqCst); } Bugsee::capture_message(LogLevel::Error, PENDING); + // A host that decides for itself: wait for its handler, then leave through + // the normal path. + let exit_via_host = |guard: bugsee::LaunchGuard| -> ! { + let start = Instant::now(); + while !HOST_HANDLER_RAN.load(Ordering::SeqCst) { + assert!(start.elapsed() < Duration::from_secs(60), "no signal"); + std::thread::sleep(Duration::from_millis(10)); + } + std::fs::write(dir.join("host-handler-ran"), b"1").unwrap(); + // Let the SDK's own handling (a flush) finish first, as a real host + // would while it winds down. + std::thread::sleep(Duration::from_millis(300)); + drop(guard); + std::process::exit(0); + }; + match kind.as_str() { // The signal must be what persists the report: no flush beforehand. - "sigterm" | "sigterm_host_handler" => { + "sigterm" => { + ready(); + std::thread::sleep(Duration::from_secs(60)); + std::process::exit(3); // the signal should have ended us + } + "sigterm_host_handler" | "sigterm_siginfo_handler" | "sigterm_late_handler" => { + ready(); + exit_via_host(guard) + } + // Ignored at install: the SDK must leave it ignored. + "sigterm_ignored" => { ready(); - if kind == "sigterm" { + std::thread::sleep(Duration::from_secs(3)); + drop(guard); + std::process::exit(0); + } + // A stop while an upload hangs in the transport: it must neither wait + // out the upload nor leave the session looking like a crash. + "slow_transport" => { + ready(); + std::thread::sleep(Duration::from_millis(500)); // uploader is in the call + let start = Instant::now(); + drop(guard); + std::fs::write(dir.join("stop-ms"), start.elapsed().as_millis().to_string()).unwrap(); + std::process::exit(0); + } + // A fork child with and without re-arming the hook. + "fork_plain" | "fork_reinstall" => { + Bugsee::flush(Duration::from_secs(1)); + // SAFETY: the grandchild only touches the SDK and `_exit`s/gets killed. + let pid = unsafe { libc::fork() }; + assert!(pid >= 0); + if pid == 0 { + if kind == "fork_reinstall" { + Bugsee::install_termination_flush(Duration::from_secs(1)).expect("re-arm"); + } + std::fs::write(dir.join("grandchild-ready"), b"1").unwrap(); std::thread::sleep(Duration::from_secs(60)); - std::process::exit(3); // the signal should have ended us + // SAFETY: never return into the harness copy. + unsafe { libc::_exit(7) }; } + assert!(wait_for_file( + &dir.join("grandchild-ready"), + Duration::from_secs(10) + )); + // SAFETY: signalling the process forked above. + unsafe { libc::kill(pid, libc::SIGTERM) }; let start = Instant::now(); - while !HOST_HANDLER_RAN.load(Ordering::SeqCst) { - assert!(start.elapsed() < Duration::from_secs(60), "no signal"); + let mut status = 0; + let result = loop { + // SAFETY: waiting for our own child. + let r = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + if r == pid { + break if libc::WIFSIGNALED(status) { + format!("signaled:{}", libc::WTERMSIG(status)) + } else { + format!("exited:{}", libc::WEXITSTATUS(status)) + }; + } + if start.elapsed() > Duration::from_secs(15) { + // SAFETY: killing our own stuck child. + unsafe { libc::kill(pid, libc::SIGKILL) }; + unsafe { libc::waitpid(pid, &mut status, 0) }; + break "hung".to_string(); + } std::thread::sleep(Duration::from_millis(10)); - } - std::fs::write(dir.join("host-handler-ran"), b"1").unwrap(); - // The host decides to exit: a normal return path. + }; + std::fs::write(dir.join("grandchild-status"), result).unwrap(); + ready(); drop(guard); std::process::exit(0); } // Reports are durably queued before the process ends the given way. - "sigterm_no_hook" | "kill" | "exit_plain" | "exit_flush" | "return" => { + "sigterm_no_hook" | "sigterm_stuck" | "kill" | "exit_plain" | "exit_flush" | "return" => { Bugsee::flush(Duration::from_secs(1)); ready(); match kind.as_str() { @@ -177,9 +332,17 @@ struct Outcome { stderr: String, dir: PathBuf, child_delivered: Vec, + /// From the first signal sent to the child's end. + after_signal: Duration, +} + +impl Drop for Outcome { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } } -fn run_child(kind: &str, fail_uploads: bool, signal: Option) -> Outcome { +fn run_child(kind: &str, env: &[(&str, &str)], signals: &[libc::c_int]) -> Outcome { let dir = std::env::temp_dir().join(format!( "bugsee-shutdown-{kind}-{}", bugsee::core::util::random_hex(6) @@ -191,8 +354,8 @@ fn run_child(kind: &str, fail_uploads: bool, signal: Option) -> Out .env(DIR_ENV, &dir) .stdout(Stdio::null()) .stderr(Stdio::piped()); - if fail_uploads { - cmd.env(FAIL_ENV, "1"); + for (k, v) in env { + cmd.env(k, v); } let mut child = cmd.spawn().expect("spawn"); @@ -203,23 +366,27 @@ fn run_child(kind: &str, fail_uploads: bool, signal: Option) -> Out "child died before it was ready" ); assert!(start.elapsed() < Duration::from_secs(30), "child not ready"); - std::thread::sleep(Duration::from_millis(20)); + std::thread::sleep(Duration::from_millis(10)); } - if let Some(sig) = signal { + let signalled = Instant::now(); + for (i, sig) in signals.iter().enumerate() { + if i > 0 { + std::thread::sleep(Duration::from_millis(300)); + } // SAFETY: signalling a child this test spawned. - unsafe { libc::kill(child.id() as libc::pid_t, sig) }; + unsafe { libc::kill(child.id() as libc::pid_t, *sig) }; } - let start = Instant::now(); let status = loop { if let Some(s) = child.try_wait().unwrap() { break s; } - if start.elapsed() > Duration::from_secs(30) { + if signalled.elapsed() > Duration::from_secs(40) { let _ = child.kill(); - panic!("child did not end after the signal"); + panic!("child did not end"); } - std::thread::sleep(Duration::from_millis(20)); + std::thread::sleep(Duration::from_millis(10)); }; + let after_signal = signalled.elapsed(); let mut stderr = String::new(); if let Some(mut e) = child.stderr.take() { let _ = e.read_to_string(&mut stderr); @@ -230,10 +397,12 @@ fn run_child(kind: &str, fail_uploads: bool, signal: Option) -> Out stderr, dir, child_delivered, + after_signal, } } /// Launch over the child's data directory and collect what recovery delivers. +/// `flush` queues behind recovery on the same worker, so no sleeping is needed. fn relaunch(dir: &Path) -> Vec { let mock = Arc::new(MockTransport::default()); let guard = Bugsee::launch_with( @@ -243,8 +412,7 @@ fn relaunch(dir: &Path) -> Vec { .with_transport(mock.clone()), ) .expect("relaunch"); - std::thread::sleep(Duration::from_millis(1500)); - Bugsee::flush(Duration::from_secs(10)); + assert!(Bugsee::flush(Duration::from_secs(10)), "queue drains"); let out = mock .uploaded_bundles .lock() @@ -263,18 +431,14 @@ fn count(reports: &[Value], needle: &str) -> usize { .count() } -const ABNORMAL: &str = "terminated abnormally"; +const FAIL: &[(&str, &str)] = &[(FAIL_ENV, "1")]; -/// Run one case and return `(pending, abnormal)` counts over everything -/// delivered by the child and by the first relaunch, after checking that a -/// second relaunch delivers nothing more. -fn run_case( - kind: &str, - fail_uploads: bool, - signal: Option, -) -> (Outcome, usize, usize) { - let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); - let out = run_child(kind, fail_uploads, signal); +/// Run one case; returns the outcome and `(pending, abnormal)` counts over +/// everything the child delivered plus the first relaunch. A second relaunch +/// must deliver nothing more. +fn run_case(kind: &str, env: &[(&str, &str)], signals: &[libc::c_int]) -> (Outcome, usize, usize) { + let _serial = serial(); + let out = run_child(kind, env, signals); let recovered = relaunch(&out.dir); let again = relaunch(&out.dir); assert!( @@ -290,7 +454,7 @@ fn run_case( #[test] fn returning_from_main_is_a_clean_stop() { - let (out, pending, abnormal) = run_case("return", true, None); + let (out, pending, abnormal) = run_case("return", FAIL, &[]); assert_eq!(out.status.code(), Some(0), "{}", out.stderr); assert_eq!(pending, 1, "the queued report is delivered exactly once"); assert_eq!(abnormal, 0, "a deliberate stop is not a crash"); @@ -300,13 +464,13 @@ fn returning_from_main_is_a_clean_stop() { fn exit_without_stop_is_an_abnormal_exit_unless_opted_in() { // Default: nothing in the process can tell `exit` from a death, so it is // reported as abnormal. The queued report is still not lost. - let (out, pending, abnormal) = run_case("exit_plain", true, None); + let (out, pending, abnormal) = run_case("exit_plain", FAIL, &[]); assert_eq!(out.status.code(), Some(0), "{}", out.stderr); assert_eq!(pending, 1); assert_eq!(abnormal, 1); // `flush_at_exit`: the same exit is deliberate. - let (out, pending, abnormal) = run_case("exit_flush", true, None); + let (out, pending, abnormal) = run_case("exit_flush", FAIL, &[]); assert_eq!(out.status.code(), Some(0), "{}", out.stderr); assert_eq!(pending, 1); assert_eq!(abnormal, 0); @@ -314,7 +478,7 @@ fn exit_without_stop_is_an_abnormal_exit_unless_opted_in() { #[test] fn sigkill_loses_neither_the_queue_nor_the_abnormal_exit() { - let (out, pending, abnormal) = run_case("kill", true, Some(libc::SIGKILL)); + let (out, pending, abnormal) = run_case("kill", FAIL, &[libc::SIGKILL]); assert_eq!(out.status.signal(), Some(libc::SIGKILL)); assert_eq!(pending, 1); assert_eq!(abnormal, 1); @@ -322,31 +486,33 @@ fn sigkill_loses_neither_the_queue_nor_the_abnormal_exit() { #[test] fn sigterm_without_the_hook_is_untouched() { - let (out, pending, abnormal) = run_case("sigterm_no_hook", true, Some(libc::SIGTERM)); + let (out, pending, abnormal) = run_case("sigterm_no_hook", FAIL, &[libc::SIGTERM]); assert_eq!(out.status.signal(), Some(libc::SIGTERM)); assert_eq!(pending, 1); assert_eq!(abnormal, 1); } #[test] -fn sigterm_with_the_hook_flushes_then_dies_of_the_signal() { +fn sigterm_and_sigint_with_the_hook_flush_then_die_of_the_signal() { // The backend is unreachable, so the report can only survive in the queue, // and only if the signal made the SDK persist it before the process ended. - let (out, pending, abnormal) = run_case("sigterm", true, Some(libc::SIGTERM)); - assert_eq!( - out.status.signal(), - Some(libc::SIGTERM), - "the process must still die of the signal, not exit: {:?} {}", - out.status, - out.stderr - ); - assert_eq!(pending, 1, "delivered or queued, exactly once"); - assert_eq!(abnormal, 0, "a handled SIGTERM is a deliberate stop"); + for sig in [libc::SIGTERM, libc::SIGINT] { + let (out, pending, abnormal) = run_case("sigterm", FAIL, &[sig]); + assert_eq!( + out.status.signal(), + Some(sig), + "the process must still die of the signal, not exit: {:?} {}", + out.status, + out.stderr + ); + assert_eq!(pending, 1, "delivered or queued, exactly once"); + assert_eq!(abnormal, 0, "a handled termination is a deliberate stop"); + } } #[test] fn sigterm_with_the_hook_delivers_when_the_backend_is_up() { - let (out, pending, abnormal) = run_case("sigterm", false, Some(libc::SIGTERM)); + let (out, pending, abnormal) = run_case("sigterm", &[], &[libc::SIGTERM]); assert_eq!(out.status.signal(), Some(libc::SIGTERM)); assert_eq!( count(&out.child_delivered, PENDING), @@ -358,17 +524,52 @@ fn sigterm_with_the_hook_delivers_when_the_backend_is_up() { } #[test] -fn the_hosts_own_sigterm_handler_still_runs_and_decides() { - let (out, pending, abnormal) = run_case("sigterm_host_handler", true, Some(libc::SIGTERM)); - assert!( - out.dir.join("host-handler-ran").exists(), - "the previous handler was swallowed: {}", +fn a_hosts_own_handler_still_runs_and_decides() { + // Plain handler, and an SA_SIGINFO one (which must get a real siginfo). + for kind in ["sigterm_host_handler", "sigterm_siginfo_handler"] { + let (out, pending, abnormal) = run_case(kind, FAIL, &[libc::SIGTERM]); + assert!( + out.dir.join("host-handler-ran").exists(), + "[{kind}] the previous handler was swallowed: {}", + out.stderr + ); + assert_eq!( + out.status.code(), + Some(0), + "[{kind}] the host chose to exit normally; the SDK must not kill it: {:?} {}", + out.status, + out.stderr + ); + assert_eq!(pending, 1, "[{kind}]"); + assert_eq!(abnormal, 0, "[{kind}]"); + } +} + +#[test] +fn a_handler_installed_after_ours_is_never_killed_by_us() { + // tokio / signal-hook style: the host takes the signal over after the SDK + // and chains to us as its previous handler. We see "default" behind us, but + // must notice we are no longer in force and not terminate the process. + let (out, pending, abnormal) = run_case("sigterm_late_handler", FAIL, &[libc::SIGTERM]); + assert!(out.dir.join("host-handler-ran").exists(), "{}", out.stderr); + assert_eq!( + out.status.code(), + Some(0), + "the host owns the signal; the SDK killed it: {:?} {}", + out.status, out.stderr ); + assert_eq!(pending, 1); + assert_eq!(abnormal, 0); +} + +#[test] +fn a_signal_ignored_at_install_stays_ignored() { + let (out, pending, abnormal) = run_case("sigterm_ignored", FAIL, &[libc::SIGTERM]); assert_eq!( out.status.code(), Some(0), - "the host chose to exit normally; the SDK must not kill it: {:?} {}", + "an ignored SIGTERM must not terminate: {:?} {}", out.status, out.stderr ); @@ -376,9 +577,61 @@ fn the_hosts_own_sigterm_handler_still_runs_and_decides() { assert_eq!(abnormal, 0); } +#[test] +fn a_second_signal_does_not_wait_out_the_flush() { + // A 20 s flush against an unreachable backend; the operator presses Ctrl+C + // twice. + let env = [(FAIL_ENV, "1"), (TERM_MS_ENV, "20000")]; + let (out, pending, _abnormal) = + run_case("sigterm_stuck", &env, &[libc::SIGTERM, libc::SIGTERM]); + assert_eq!(out.status.signal(), Some(libc::SIGTERM)); + assert!( + out.after_signal < Duration::from_secs(10), + "took {:?}", + out.after_signal + ); + assert_eq!(pending, 1, "the queue was durable before the signal"); +} + +#[test] +fn a_stop_during_a_hanging_upload_is_still_a_clean_stop() { + let env = [(SLOW_ENV, "1")]; + let (out, pending, abnormal) = run_case("slow_transport", &env, &[]); + assert_eq!(out.status.code(), Some(0), "{}", out.stderr); + let stop_ms: u64 = std::fs::read_to_string(out.dir.join("stop-ms")) + .unwrap() + .parse() + .unwrap(); + assert!( + stop_ms < 8000, + "stop waited on the hanging upload: {stop_ms} ms" + ); + assert_eq!(abnormal, 0, "an upload in flight is not a crashed session"); + assert_eq!(pending, 1, "the abandoned report is re-sent next launch"); +} + +#[test] +fn a_fork_child_neither_hangs_nor_loses_the_signal() { + // Without re-arming, the inherited handler has no watcher thread: it must + // terminate the child like the default would. With re-arming, the child's + // own watcher flushes and then terminates it. + for kind in ["fork_plain", "fork_reinstall"] { + let (out, pending, abnormal) = run_case(kind, FAIL, &[]); + assert_eq!(out.status.code(), Some(0), "[{kind}] {}", out.stderr); + let status = std::fs::read_to_string(out.dir.join("grandchild-status")).unwrap(); + assert_eq!( + status, + format!("signaled:{}", libc::SIGTERM), + "[{kind}] the forked child must die of SIGTERM" + ); + assert_eq!(pending, 1, "[{kind}]"); + assert_eq!(abnormal, 0, "[{kind}]"); + } +} + #[test] fn shutdown_is_idempotent() { - let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let _serial = serial(); let dir = std::env::temp_dir().join(format!( "bugsee-shutdown-idem-{}", bugsee::core::util::random_hex(6) From 31a293c8cfe74111ef76ff25fe73ca0ae9356184 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:44:09 +0500 Subject: [PATCH 11/52] fix: keep draining existing data dirs that cannot take new files (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - an existing directory that fails the write probe / session open is still recovered from and drained (read + delete only); claims are lenient there and a crash that cannot be re-queued on disk is delivered from memory; delivered but undeletable bundles are remembered so they are not resent every drain - in-memory TooManySimilar blacklist (cleared in fork children) - a clean stop gives the in-memory backlog a last bounded delivery attempt - legacy temp location adopted whenever it exists and is trusted, even if no new default directory was usable - rotation tick rewrites a missing liveness marker after a wipe and marks the recorder degraded - DESIGN.md: failure table, legacy/flush wording πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 7 +- crates/bugsee-core/src/claim.rs | 37 +++- crates/bugsee-core/src/queue.rs | 6 + crates/bugsee-core/src/recovery.rs | 11 +- crates/bugsee-core/src/runtime.rs | 154 +++++++++++-- crates/bugsee-core/tests/storage_failures.rs | 222 +++++++++++++++++++ crates/bugsee/src/api.rs | 7 +- crates/bugsee/src/options.rs | 86 ++++++- crates/bugsee/tests/storage_fallback.rs | 57 +++++ 9 files changed, 539 insertions(+), 48 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index f2f03f6..4d06151 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -186,8 +186,9 @@ Global base scope behind `RwLock`; `with_scope` pushes a thread-local overlay. * | Failure | Outcome | |---|---| | Default candidate unusable, a later default works | falls through; one stderr line; `StorageStatus::skipped` lists why | -| Requested directory unusable, or no default usable, or `Session::begin`/`PartStore::new` fails (also in a fork child) | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from an in-memory backlog capped at 16 (an entry counts until its delivery settles; overflow drops the newest with `DropReason::StorageUnavailable`); an unsent report dies with the process; a fork child starts with an empty backlog (the parent's is the parent's to send) | -| Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and dropped; a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out; `StorageStatus::degraded` turns true and `crash_recovery_available()` false | +| Requested directory unusable, or no default usable, or `Session::begin`/`PartStore::new` fails (also in a fork child) | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from an in-memory backlog capped at 16 (an entry counts until its delivery settles; overflow drops the newest with `DropReason::StorageUnavailable`); a `TooManySimilar` verdict is remembered in memory; a clean stop (guard drop) gives the backlog one last bounded (1.5 s) attempt, but a report still unsent when the process dies is lost; a fork child starts with an empty backlog (the parent's is the parent's to send) | +| Directory exists but cannot take NEW files (full, read-only remount) | memory mode for new reports, but the directory is still **drained and recovered from**: its queued bundles are delivered and its crashed sessions recovered (claims are best-effort there; a crash that cannot be re-queued on disk is delivered from memory). Bundles delivered but undeletable are remembered and not resent in this process; a later launch with a writable disk may resend them and the backend deduplicates. Only a missing or non-directory path is skipped | +| Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and dropped; the rotation tick rewrites a missing `sessions/.alive` (a wiped-and-recreated tree would otherwise make the next launch's orphan sweep drop this session's crash); a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out; `StorageStatus::degraded` turns true and `crash_recovery_available()` false | | Disk full / permission denied at enqueue, snapshot, marker | same as above; the `reports/` snapshot dir is reclaimed by `SnapshotHandle::drop` | | Crash handler cannot write its marker | the raw `open`/`write` return values are ignored; the process still dies with the original signal (watchdog bounds the handler) | @@ -195,7 +196,7 @@ The in-memory backlog retries like the disk queue: it caches the presigned endpo Crash recovery needs the data directory to **outlive the process**: on a container, mount a volume and point `data_dir`/`BUGSEE_DATA_DIR` at it. The temp fallback is commonly wiped on restart. `Bugsee::storage_status()` reports `mode`, `data_dir`, `source`, `skipped`, `explicit_rejected`, `degraded`, `crash_recovery_available()` and `may_be_ephemeral()` (the SDK cannot tell a volume from a scratch disk; the temp fallback is the only case it flags). No FFI accessor yet (follow-up). -**Release note, default location moved.** Earlier builds defaulted to `/bugsee/`; the default is now the platform location (3) or the private temp fallback (4). When no directory was requested and the old location exists, is a real directory and is owned by the current user, the launch recovers its crashed sessions and delivers its queue once, best effort (delivery is asynchronous and `flush` does not wait for it). It is not claimed to be lossless: a legacy session whose process is still alive is skipped, and a legacy directory that fails the ownership check is left alone. Hosts that set `data_dir` were never using it. +**Release note, default location moved.** Earlier builds defaulted to `/bugsee/`; the default is now the platform location (3) or the private temp fallback (4). When no directory was requested and the old location exists, is a real directory and is owned by the current user, the launch recovers its crashed sessions and delivers its queue once, best effort, including when no new directory could be created. Recovery runs on the worker before a `flush` is acknowledged and `flush` drains the legacy queue with the data directory's, so a flush covers it; the uploader stops polling it once its queue is empty. It is not claimed to be lossless: a legacy session whose process is still alive is skipped, and a legacy directory that fails the ownership check is left alone. Hosts that set `data_dir` were never using it. **Unsafe.** The facade makes two `geteuid()` calls (unix) to name and vet the private temp directory, alongside core's single `kill(pid, 0)`. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index 7e01144..12ab003 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -38,8 +38,35 @@ impl Drop for Claim { } /// Try to claim `path`. `None` if another live process (or another thread of this -/// one) holds it. +/// one) holds it, or if the claim file cannot be created at all. pub fn try_claim(path: &Path) -> Option { + match claim_state(path) { + ClaimState::Held(claim) => Some(claim), + ClaimState::Taken | ClaimState::Unavailable => None, + } +} + +/// Like [`try_claim`] for a directory that may be full or read-only, where the +/// claim file cannot be created but the work (recover a crash, deliver a queued +/// report) is still worth doing: `Some(None)` means "no claim could be made, +/// proceed unclaimed". The cost is that two processes sharing such a directory +/// may both do the work, and the backend deduplicates. `None` means another +/// process holds the claim. +pub fn try_claim_lenient(path: &Path) -> Option> { + match claim_state(path) { + ClaimState::Held(claim) => Some(Some(claim)), + ClaimState::Unavailable => Some(None), + ClaimState::Taken => None, + } +} + +enum ClaimState { + Held(Claim), + Taken, + Unavailable, +} + +fn claim_state(path: &Path) -> ClaimState { for _ in 0..2 { match std::fs::OpenOptions::new() .write(true) @@ -48,13 +75,13 @@ pub fn try_claim(path: &Path) -> Option { { Ok(mut file) => { let _ = write!(file, "{}", std::process::id()); - return Some(Claim { + return ClaimState::Held(Claim { path: path.to_path_buf(), }); } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { if !is_stale(path) { - return None; + return ClaimState::Taken; } // Take over a claim whose owner is gone, then retry the create. // Losing the race to another taker just means `create_new` fails @@ -63,10 +90,10 @@ pub fn try_claim(path: &Path) -> Option { } // Directory missing / unwritable: nothing can be claimed, and the // caller's own I/O will fail more usefully than we can here. - Err(_) => return None, + Err(_) => return ClaimState::Unavailable, } } - None + ClaimState::Taken } fn is_stale(path: &Path) -> bool { diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index 2ff3efe..ea6c033 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -46,6 +46,12 @@ pub fn try_claim(report: &QueuedReport) -> Option { crate::claim::try_claim(&report.claim_path()) } +/// [`try_claim`] that still lets delivery proceed when the queue directory cannot +/// accept a claim file (full or read-only); see [`crate::claim::try_claim_lenient`]. +pub fn try_claim_lenient(report: &QueuedReport) -> Option> { + crate::claim::try_claim_lenient(&report.claim_path()) +} + fn sibling(bundle: &Path, ext: &str) -> PathBuf { let mut s = bundle.as_os_str().to_os_string(); s.push("."); diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index ca21356..65f8fd0 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -349,8 +349,15 @@ pub fn bump_attempts(data_dir: &Path, pending: &PendingSession) -> u32 { /// Claim a pending session for recovery, so that of several processes sharing the /// data directory exactly one recovers it. The claim lives beside the liveness /// marker and is released when the returned guard drops. -pub fn claim_pending(data_dir: &Path, pending: &PendingSession) -> Option { - crate::claim::try_claim( +/// +/// Lenient (see [`crate::claim::try_claim_lenient`]): a full or read-only +/// directory cannot take a claim file, and that is exactly when a stranded crash +/// most needs recovering. +pub fn claim_pending( + data_dir: &Path, + pending: &PendingSession, +) -> Option> { + crate::claim::try_claim_lenient( &data_dir .join("sessions") .join(format!("{}.claim", pending.generation)), diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 232e8f8..57a9864 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -25,6 +25,7 @@ //! under back-pressure) so a burst or a stalled worker cannot grow RSS without //! limit. +use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender}; @@ -247,6 +248,12 @@ struct Shared { /// stays here until its delivery is settled, so the length is the real backlog. mem_queue: ForkLock>, next_mem_id: AtomicU64, + /// Crash signatures the server refused as too similar, remembered in memory + /// (the on-disk `.blacklist` is unavailable or unreadable with no usable disk). + mem_blacklist: ForkLock>, + /// Queued bundles that were delivered but could not be deleted (read-only or + /// full directory). Skipped from then on, or every drain would send them again. + delivered_stuck: ForkLock>, /// Set the first time a disk write fails while running in disk mode: the /// recorder is no longer fully durable (see [`Recorder::is_degraded`]). degraded: AtomicBool, @@ -260,8 +267,12 @@ impl ForkLockable for Shared { self.scope.fork_lock(); self.session.fork_lock(); self.mem_queue.fork_lock(); + self.mem_blacklist.fork_lock(); + self.delivered_stuck.fork_lock(); } fn fork_unlock(&self) { + self.delivered_stuck.fork_unlock(); + self.mem_blacklist.fork_unlock(); self.mem_queue.fork_unlock(); self.session.fork_unlock(); self.scope.fork_unlock(); @@ -352,6 +363,8 @@ impl Recorder { paused: AtomicBool::new(false), mem_queue: ForkLock::new(Vec::new()), next_mem_id: AtomicU64::new(0), + mem_blacklist: ForkLock::new(BTreeSet::new()), + delivered_stuck: ForkLock::new(BTreeSet::new()), degraded: AtomicBool::new(false), }); @@ -385,10 +398,16 @@ impl Recorder { Some((session, store)) => (Some(session), Some(store)), None => (None, None), }; + // A directory that cannot take NEW files may still hold reports and crashes + // a previous run persisted (that is what a full disk looks like), and + // recovering them only reads and deletes. So an existing directory is + // still drained and recovered from in memory mode. + let drain_dir = (session.is_some() || data_dir.is_dir()).then_some(data_dir.as_path()); let pipeline = spawn_pipeline( &shared, store.is_some().then_some(data_dir.as_path()), - session.as_ref().map_or(0, Session::generation), + drain_dir, + session.as_ref().map_or(u64::MAX, Session::generation), store, config.caps, config.rotate_interval, @@ -454,6 +473,8 @@ impl Recorder { // The parent's unsent in-memory reports are the PARENT's to send; the child // inheriting them would deliver every one a second time. self.shared.mem_queue.lock().clear(); + self.shared.mem_blacklist.lock().clear(); + self.shared.delivered_stuck.lock().clear(); // A child that cannot open its own session degrades to memory rather than // failing: the parent's disk may be gone or read-only by now. @@ -475,7 +496,8 @@ impl Recorder { let pipeline = spawn_pipeline( &self.shared, store.is_some().then_some(self.data_dir.as_path()), - session.as_ref().map_or(0, Session::generation), + store.is_some().then_some(self.data_dir.as_path()), + session.as_ref().map_or(u64::MAX, Session::generation), store, self.caps, self.rotate_interval, @@ -820,6 +842,7 @@ fn open_disk(data_dir: &Path, caps: WindowCaps) -> std::io::Result<(Session, Par fn spawn_pipeline( shared: &Arc, data_dir: Option<&Path>, + drain_dir: Option<&Path>, generation: u64, store: Option, caps: WindowCaps, @@ -832,7 +855,7 @@ fn spawn_pipeline( // Uploader thread: drains the durable queue with retry. let (upload_tx, upload_rx) = channel(); let uploader_shared = Arc::clone(shared); - let uploader_data_dir = data_dir.map(Path::to_path_buf); + let uploader_data_dir = drain_dir.map(Path::to_path_buf); let uploader_legacy = legacy_data_dir.clone(); // Set once the worker has finished recovering the legacy directory, so the // uploader does not retire it while recovery is still about to fill its queue. @@ -856,6 +879,8 @@ fn spawn_pipeline( let (tx, rx) = channel(); let worker_shared = Arc::clone(shared); let worker_data_dir = data_dir.map(Path::to_path_buf); + let worker_drain_dir = drain_dir.map(Path::to_path_buf); + let marker = data_dir.map(|d| Session::marker_path(d, generation)); let worker_upload_tx = upload_tx.clone(); let worker = std::thread::Builder::new() @@ -869,7 +894,7 @@ fn spawn_pipeline( // first (runs before this session enqueues anything). Only a fresh // launch: a fork child's parent may be mid-enqueue right now, and // its in-flight sidecars would look orphaned. - if let (true, Some(dir)) = (fresh_launch, &worker_data_dir) { + if let (true, Some(dir)) = (fresh_launch, &worker_drain_dir) { queue::gc_orphans(dir); } // A fork child recovers too. A prefork master recovers only at ITS @@ -877,7 +902,7 @@ fn spawn_pipeline( // recover a worker that died later; the next worker started from // it must, or that crash waits for the master to restart. Claims // keep siblings from recovering the same session twice. - if let Some(dir) = &worker_data_dir { + if let Some(dir) = &worker_drain_dir { run_recovery(&worker_shared, dir, generation); } // Best effort, once: sessions a previous default location left @@ -895,6 +920,7 @@ fn spawn_pipeline( worker_shared, rotate_interval, worker_data_dir, + marker, caps, sampler, worker_upload_tx, @@ -957,7 +983,7 @@ fn recover_one(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSess merge_scope(shared, &mut built.meta); let window = TimeWindow { start, end }; - let delivered = reporting::assemble_with_extras( + let assembled = reporting::assemble_with_extras( &pending.parts_dir, window, &built.meta, @@ -966,9 +992,22 @@ fn recover_one(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSess &built.extra_files, &shared.app_token, end, - ) - .and_then(|assembled| queue::enqueue(data_dir, &assembled)) - .is_ok(); + ); + // A full or read-only directory cannot take the queue entry; deliver from + // memory instead of losing the crash for this launch. The evidence stays on + // disk (nothing to discard it with), so a later launch with a healthy disk + // reports it again and the backend deduplicates. + let delivered = match assembled { + Ok(assembled) => { + if queue::enqueue(data_dir, &assembled).is_ok() { + true + } else { + hold_in_memory(shared, assembled); + false + } + } + Err(_) => false, + }; // Only destroy the crash evidence once it is safely queued. if delivered { @@ -999,6 +1038,7 @@ fn worker_loop( shared: Arc, rotate_interval: Duration, data_dir: Option, + marker: Option, caps: WindowCaps, mut sampler: Option>, upload_tx: Sender, @@ -1092,11 +1132,30 @@ fn worker_loop( // elapsed β€” via a message OR a timeout β€” so steady traffic can't starve it. if Instant::now() >= next_rotate { rotate_tick(store.as_mut(), &shared, &mut sampler); + if let Some(marker) = &marker { + ensure_marker(&shared, marker); + } next_rotate = Instant::now() + rotate_interval; } } } +/// Make sure this session's liveness marker still exists. If the data directory +/// was removed and recreated under us (`create_dir_all` heals the tree but not +/// the marker), the next launch would not see this session as live, and its +/// crash would be dropped as an orphan. Rewrite it and mark the recorder +/// degraded: the capture window it had is gone with the removed files. +fn ensure_marker(shared: &Shared, marker: &Path) { + if marker.exists() { + return; + } + shared.degraded.store(true, Ordering::Relaxed); + if let Some(parent) = marker.parent() { + let _ = std::fs::create_dir_all(parent); + } + let _ = std::fs::write(marker, std::process::id().to_string()); +} + /// One rotation tick: sample telemetry (unless paused) then roll + evict the /// window. Panic-contained so a host `TelemetrySampler` panic can't kill the /// worker. @@ -1224,6 +1283,14 @@ fn assemble_in_memory( ) } +fn mem_blacklisted(shared: &Shared, sigs: &[String]) -> bool { + if sigs.is_empty() { + return false; + } + let set = shared.mem_blacklist.lock(); + sigs.iter().any(|s| set.contains(s)) +} + /// Park a report in the bounded in-memory queue for the uploader. Over the cap /// the NEWEST report is dropped (and the host told), keeping the backlog bounded. fn hold_in_memory(shared: &Shared, report: AssembledReport) { @@ -1273,11 +1340,12 @@ fn process_report( return Ok(()); } - // Suppress reports the server blacklisted (12004 TooManySimilar). - if let Some(dir) = data_dir { - if queue::any_blacklisted(dir, &meta.signatures) { - return Ok(()); - } + // Suppress reports the server blacklisted (12004 TooManySimilar), from the + // in-memory set (the only one with no usable disk) or the on-disk one. + if mem_blacklisted(shared, &meta.signatures) + || data_dir.is_some_and(|dir| queue::any_blacklisted(dir, &meta.signatures)) + { + return Ok(()); } let window_start = window_end.saturating_sub(caps.max_window_ms); @@ -1402,6 +1470,18 @@ fn uploader_loop( let _ = ack.send(empty); } Ok(UploadMsg::Stop(ack)) => { + // A report held in memory has no next launch to be retried by, so + // give it one last bounded attempt (the stopper waits 2 s for the + // ack). Disk reports survive in `queue/` and need nothing here. + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + drain_memory( + &shared, + backoff_base, + true, + false, + Some(Instant::now() + Duration::from_millis(1500)), + ) + })); let _ = ack.send(()); break; } @@ -1418,19 +1498,30 @@ fn uploader_loop( } Err(RecvTimeoutError::Disconnected) => break, } - retire_legacy(&mut legacy, &legacy_ready); + retire_legacy(&shared, &mut legacy, &legacy_ready); } } +/// Queued reports in `dir` still awaiting delivery: everything on disk except +/// bundles already delivered whose deletion failed. +fn pending_reports(shared: &Shared, dir: &Path) -> Vec { + let mut all = queue::list_pending(dir); + let stuck = shared.delivered_stuck.lock(); + if !stuck.is_empty() { + all.retain(|r| !stuck.contains(&r.bundle)); + } + all +} + /// Stop polling the legacy directory once recovery has run (it runs once per /// launch, so nothing new can appear) and its queue is empty. -fn retire_legacy(legacy: &mut Option, ready: &AtomicBool) { +fn retire_legacy(shared: &Shared, legacy: &mut Option, ready: &AtomicBool) { if !ready.load(Ordering::SeqCst) { return; } if legacy .as_deref() - .is_some_and(|dir| queue::list_pending(dir).is_empty()) + .is_some_and(|dir| pending_reports(shared, dir).is_empty()) { *legacy = None; } @@ -1466,9 +1557,9 @@ fn drain(shared: &Shared, data_dir: Option<&Path>, backoff_base: Duration, force fn drain_disk(shared: &Shared, data_dir: &Path, backoff_base: Duration, force: bool) -> bool { let now = epoch_ms(); let mut attempted = false; - for report in queue::list_pending(data_dir) { + for report in pending_reports(shared, data_dir) { // Another process sharing the queue may be delivering this one. - let Some(_claim) = queue::try_claim(&report) else { + let Some(_claim) = queue::try_claim_lenient(&report) else { continue; }; let Ok((zip, request_json, retry, next_attempt)) = queue::load(&report) else { @@ -1537,7 +1628,13 @@ fn deliver_queued( cached_endpoint.as_deref(), &mut endpoint_out, ) { - Ok(()) => queue::remove(report), + Ok(()) => { + queue::remove(report); + if report.bundle.exists() { + // Delivered, but the directory will not let us delete it. + shared.delivered_stuck.lock().insert(report.bundle.clone()); + } + } // A transient failure OR a still-expired session are both retryable. Err(TransportError::Transient(_)) | Err(TransportError::SessionExpired) => { // Persist the presigned endpoint if the failure left one to resume @@ -1567,6 +1664,7 @@ fn deliver_queued( signatures }; queue::blacklist_add(data_dir, &sigs); + shared.mem_blacklist.lock().extend(sigs); queue::remove(report); notify_dropped(shared, DropReason::TooManySimilar); } @@ -1665,9 +1763,17 @@ fn drain_memory( } } } - Err(TransportError::TooManySimilar { .. }) => { + Err(TransportError::TooManySimilar { signatures }) => { queue.remove(pos); dropped = Some(DropReason::TooManySimilar); + // Stop the same fault from being sent again: no disk to blacklist on. + let sigs = if signatures.is_empty() { + queue::signatures_of(&report) + } else { + signatures + }; + // Lock order: the queue lock is held; the blacklist is a leaf. + shared.mem_blacklist.lock().extend(sigs); } Err(e) => { queue.remove(pos); @@ -1709,8 +1815,8 @@ fn drain_until_empty( let now = epoch_ms(); drain_memory(shared, backoff_base, true, false, Some(start + deadline)); for &data_dir in dirs { - for report in queue::list_pending(data_dir) { - let Some(_claim) = queue::try_claim(&report) else { + for report in pending_reports(shared, data_dir) { + let Some(_claim) = queue::try_claim_lenient(&report) else { continue; // a peer is delivering it; the loop below waits it out }; let Ok((zip, request_json, retry, _next)) = queue::load(&report) else { @@ -1732,7 +1838,7 @@ fn drain_until_empty( } } } - let disk_empty = dirs.iter().all(|d| queue::list_pending(d).is_empty()); + let disk_empty = dirs.iter().all(|d| pending_reports(shared, d).is_empty()); if disk_empty && shared.mem_queue.lock().is_empty() { return true; } diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index d8ab57a..175de28 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -432,3 +432,225 @@ fn a_failed_put_retries_at_the_put_and_does_not_mint_a_duplicate_issue() { "the retry must resume at the PUT, not create the issue again" ); } + +/// Lay down what a previous run left behind: a queued report nobody delivered and +/// a session that died (marker left, owner gone). +fn seed_leftovers(dir: &Path) { + use bugsee_core::capture::record; + use bugsee_core::model::report::TimeWindow; + use bugsee_core::reporting::assemble_docs; + + let env = bugsee_core::model::environment::Environment::detect_with( + "t", + &Default::default(), + &Default::default(), + ); + let window = TimeWindow { start: 0, end: 1 }; + let queued = assemble_docs( + Vec::new(), + window, + &manual_upload_meta(), + &env, + None, + &[], + "T", + 1, + ) + .unwrap(); + bugsee_core::queue::enqueue(dir, &queued).unwrap(); + + let part = dir.join("parts").join("1").join("0"); + std::fs::create_dir_all(&part).unwrap(); + let mut bytes = Vec::new(); + let payload = br#"{"timestamp":100,"level":1,"source":2,"message":"earlier crash"}"#; + record::frame(100, payload, &mut bytes); + std::fs::write(part.join("log.part"), &bytes).unwrap(); + std::fs::create_dir_all(dir.join("sessions")).unwrap(); + std::fs::write(dir.join("sessions").join("1.alive"), "0").unwrap(); + std::fs::write(dir.join("gen"), "1").unwrap(); +} + +fn wait_for_bundles(transport: &MockTransport, n: usize) -> usize { + let start = std::time::Instant::now(); + while transport.uploaded_bundles.lock().unwrap().len() < n + && start.elapsed() < Duration::from_secs(10) + { + std::thread::sleep(Duration::from_millis(20)); + } + // Settle: anything delivered again would show up now. + std::thread::sleep(Duration::from_millis(400)); + let got = transport.uploaded_bundles.lock().unwrap().len(); + got +} + +/// A directory that cannot take new files can still hold reports and crashes a +/// previous run persisted (that is what a full disk looks like). They must be +/// delivered, and not again on every drain. +#[cfg(unix)] +#[test] +fn leftovers_in_a_directory_that_cannot_take_new_files_are_still_delivered() { + use std::os::unix::fs::PermissionsExt; + for memory_only in [false, true] { + let dir = TempDir::new(); + let data = dir.path.join("data"); + std::fs::create_dir_all(&data).unwrap(); + seed_leftovers(&data); + for d in [data.join("queue"), data.join("sessions"), data.clone()] { + std::fs::set_permissions(d, std::fs::Permissions::from_mode(0o555)).unwrap(); + } + // root ignores permission bits, which would make this test vacuous. + if std::fs::write(data.join("probe"), b"x").is_ok() { + eprintln!("skipped: permission bits are not enforced for this user"); + return; + } + + let transport = Arc::new(MockTransport::default()); + let mut config = RecorderConfig::new(&data, "T"); + if memory_only { + // What the facade does once its write probe has failed. + config.memory_only = Some("write probe failed".into()); + } + let recorder = Recorder::launch(config, transport.clone()).expect("launch"); + assert_eq!(recorder.storage_mode(), StorageMode::Memory); + + assert_eq!( + wait_for_bundles(&transport, 2), + 2, + "the queued report and the recovered crash, each delivered once \ + (memory_only={memory_only})" + ); + drop(recorder); + restore_permissions(&data); + restore_permissions(&data.join("queue")); + restore_permissions(&data.join("sessions")); + } +} + +#[test] +fn a_too_similar_verdict_is_remembered_without_a_disk() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct Refuse(Arc); + impl bugsee_core::Transport for Refuse { + fn register_session( + &self, + _: &str, + _: &[u8], + ) -> Result { + Ok("token".into()) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Err(bugsee_core::TransportError::TooManySimilar { + signatures: vec!["sig-loop".into()], + }) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), bugsee_core::TransportError> { + Ok(()) + } + } + + let dir = TempDir::new(); + let issues = Arc::new(AtomicUsize::new(0)); + let recorder = Recorder::launch( + RecorderConfig::new(blocked_path(&dir.path), "T"), + Arc::new(Refuse(issues.clone())), + ) + .expect("launch"); + for _ in 0..3 { + let mut meta = manual_upload_meta(); + meta.signatures = vec!["sig-loop".into()]; + recorder.report(meta, None); + assert!(recorder.flush(Duration::from_secs(10))); + } + assert_eq!( + issues.load(Ordering::SeqCst), + 1, + "after the first refusal the same fault must not be sent again" + ); +} + +#[test] +fn a_clean_shutdown_gives_unsent_memory_reports_a_last_attempt() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + // The first attempt fails, so the report is still unsent when we stop. + struct FailOnce { + attempts: AtomicUsize, + delivered: AtomicUsize, + } + struct Handle(Arc); + impl bugsee_core::Transport for Handle { + fn register_session( + &self, + _: &str, + _: &[u8], + ) -> Result { + Ok("token".into()) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + if self.0.attempts.fetch_add(1, Ordering::SeqCst) == 0 { + Err(bugsee_core::TransportError::Transient("blip".into())) + } else { + Ok("https://uploads.example/x".into()) + } + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), bugsee_core::TransportError> { + self.0.delivered.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + + let dir = TempDir::new(); + let state = Arc::new(FailOnce { + attempts: AtomicUsize::new(0), + delivered: AtomicUsize::new(0), + }); + let recorder = Recorder::launch( + RecorderConfig::new(blocked_path(&dir.path), "T"), + Arc::new(Handle(state.clone())), + ) + .expect("launch"); + recorder.report(manual_upload_meta(), None); + // No flush: just drop, as a `LaunchGuard` does. + drop(recorder); + assert_eq!(state.delivered.load(Ordering::SeqCst), 1); +} + +#[test] +fn a_directory_wiped_and_recreated_mid_run_gets_its_liveness_marker_back() { + let dir = TempDir::new(); + let data = dir.path.join("data"); + let transport = Arc::new(MockTransport::default()); + let mut config = RecorderConfig::new(&data, "T"); + config.rotate_interval = Duration::from_millis(50); + let recorder = Recorder::launch(config, transport).expect("launch"); + + std::fs::remove_dir_all(&data).unwrap(); + // The rotation tick recreates the tree (`create_dir_all`) but the marker + // only comes back if the SDK rewrites it. + let start = std::time::Instant::now(); + let alive = || { + std::fs::read_dir(data.join("sessions")) + .map(|rd| { + rd.flatten() + .any(|e| e.file_name().to_string_lossy().ends_with(".alive")) + }) + .unwrap_or(false) + }; + while !alive() && start.elapsed() < Duration::from_secs(5) { + std::thread::sleep(Duration::from_millis(20)); + } + assert!(alive(), "a crash after the wipe must still be recoverable"); + assert!(recorder.is_degraded(), "the lost window is a degradation"); +} diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index a693e7f..985b567 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -88,6 +88,7 @@ impl Bugsee { source, skipped, explicit_rejected, + recovery_dir, legacy_dir, } = options.resolve_data_dir(); let memory_only = dir.is_none().then(|| { @@ -96,9 +97,9 @@ impl Bugsee { .or_else(|| (!skipped.is_empty()).then(|| skipped.join("; "))) .unwrap_or_else(|| "no usable data directory".to_string()) }); - // Memory mode never touches the path; an empty one avoids resolving a - // temp directory just to ignore it. - let data_dir = dir.unwrap_or_default(); + // In memory mode no new file is written there, but an existing directory + // is still drained and recovered from; with none, an empty path. + let data_dir = dir.or(recovery_dir).unwrap_or_default(); if memory_only.is_none() && !skipped.is_empty() { static WARNED: AtomicBool = AtomicBool::new(false); bugsee_core::util::warn_once( diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index e4de340..a77619c 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -292,6 +292,10 @@ pub(crate) struct DataDirResolution { pub skipped: Vec, /// Why an explicit / environment directory was rejected (it is final). pub explicit_rejected: Option, + /// A rejected candidate that nevertheless EXISTS as a directory (full or + /// read-only). It may hold reports and crashes a previous run persisted, so + /// it is still drained and recovered from; no new files are written to it. + pub recovery_dir: Option, /// The location earlier SDK versions used by default /// (`/bugsee/`), if it exists, is trustworthy and is not `dir`. /// Its leftovers are recovered once at launch. @@ -322,13 +326,15 @@ pub(crate) fn resolve_data_dir( source, skipped: Vec::new(), explicit_rejected: None, + recovery_dir: None, legacy_dir: None, }, Err(e) => DataDirResolution { + explicit_rejected: Some(format!("{}: {e}", dir.display())), + recovery_dir: dir.is_dir().then_some(dir), dir: None, source: DataDirSource::None, skipped: Vec::new(), - explicit_rejected: Some(format!("{}: {e}", dir.display())), legacy_dir: None, }, }; @@ -352,27 +358,36 @@ pub(crate) fn resolve_data_dir( let mut skipped = Vec::new(); let mut found = None; + let mut recovery_dir = None; for (source, dir, private) in candidates { match prepare_dir(&dir, private) { Ok(()) => { found = Some((source, dir)); break; } - Err(e) => skipped.push(format!("{}: {e}", dir.display())), + Err(e) => { + skipped.push(format!("{}: {e}", dir.display())); + // Only the user's own default location: the shared temp area is + // exactly where a planted directory would be refused. + if !private && recovery_dir.is_none() && dir.is_dir() { + recovery_dir = Some(dir); + } + } } } let legacy = temp.join("bugsee").join(&token); - let legacy_dir = found - .as_ref() - .filter(|(_, dir)| legacy_is_adoptable(&legacy, dir)) - .map(|_| legacy); + // Also with no usable directory: leftovers there are exactly what a launch + // that cannot write anywhere new should still deliver. + let chosen = found.as_ref().map(|(_, dir)| dir.as_path()); + let legacy_dir = legacy_is_adoptable(&legacy, chosen).then_some(legacy); match found { Some((source, dir)) => DataDirResolution { dir: Some(dir), source, skipped, explicit_rejected: None, + recovery_dir: None, legacy_dir, }, None => DataDirResolution { @@ -380,7 +395,8 @@ pub(crate) fn resolve_data_dir( source: DataDirSource::None, skipped, explicit_rejected: None, - legacy_dir: None, + recovery_dir, + legacy_dir, }, } } @@ -492,16 +508,20 @@ fn owned_by_us(_meta: &std::fs::Metadata) -> bool { /// would let someone else's tree, or our own live data directory, be processed /// (recovering our live session as a crash). Paths are compared canonicalised, /// not as strings, so another spelling of our own directory is rejected too. -fn legacy_is_adoptable(legacy: &Path, chosen: &Path) -> bool { +fn legacy_is_adoptable(legacy: &Path, chosen: Option<&Path>) -> bool { let Some(parent) = legacy.parent() else { return false; }; if !trusted_existing_dir(parent) || !trusted_existing_dir(legacy) { return false; } - match (legacy.canonicalize(), chosen.canonicalize()) { - (Ok(a), Ok(b)) => a != b, - _ => false, + let Ok(legacy) = legacy.canonicalize() else { + return false; + }; + match chosen { + // No live directory of ours to collide with. + None => true, + Some(chosen) => chosen.canonicalize().is_ok_and(|c| c != legacy), } } @@ -807,6 +827,50 @@ mod tests { let _ = std::fs::remove_dir_all(&s); } + #[cfg(unix)] + #[test] + fn a_rejected_directory_that_exists_is_kept_for_recovery() { + use std::os::unix::fs::PermissionsExt; + let s = scratch(); + let ro = s.join("ro"); + std::fs::create_dir_all(&ro).unwrap(); + std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o555)).unwrap(); + if std::fs::write(ro.join("p"), b"x").is_ok() { + let _ = std::fs::remove_dir_all(&s); + return; // permission bits not enforced (root) + } + let r = resolve_data_dir(Some(&ro), "tok", &env_of(&[]), &|| s.join("tmp")); + assert!(r.dir.is_none() && r.explicit_rejected.is_some()); + assert_eq!( + r.recovery_dir.as_deref(), + Some(ro.as_path()), + "its queue and crashed sessions must still be drained" + ); + + // A missing directory has nothing to recover. + let r = resolve_data_dir(Some(&blocked(&s, "f5")), "tok", &env_of(&[]), &|| { + s.join("tmp") + }); + assert!(r.recovery_dir.is_none()); + let _ = std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o755)); + let _ = std::fs::remove_dir_all(&s); + } + + #[cfg(unix)] + #[test] + fn the_legacy_location_is_adopted_even_when_no_new_default_is_usable() { + let s = scratch(); + let tmp = s.join("tmp"); + let legacy = tmp.join("bugsee").join("tok"); + std::fs::create_dir_all(&legacy).unwrap(); + // The private temp name is occupied by a symlink, and there is no home. + std::os::unix::fs::symlink(s.join("elsewhere"), tmp.join(private_temp_name())).unwrap(); + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); + assert!(r.dir.is_none(), "no new directory is usable"); + assert_eq!(r.legacy_dir, Some(legacy)); + let _ = std::fs::remove_dir_all(&s); + } + #[test] fn an_explicit_directory_never_asks_for_the_temp_location() { let s = scratch(); diff --git a/crates/bugsee/tests/storage_fallback.rs b/crates/bugsee/tests/storage_fallback.rs index 32128d8..35fbefc 100644 --- a/crates/bugsee/tests/storage_fallback.rs +++ b/crates/bugsee/tests/storage_fallback.rs @@ -112,5 +112,62 @@ fn launch_degrades_through_the_resolution_order() { ); drop(guard); + // 5. A requested directory that cannot take new files but already holds a + // queued report (a full or read-only volume): memory mode, yet the + // report a previous run left there is still delivered. + { + use bugsee::core::model::report::TimeWindow; + use std::os::unix::fs::PermissionsExt; + + let ro = s.join("ro-volume"); + let env = bugsee::core::model::environment::Environment::detect_with( + "t", + &Default::default(), + &Default::default(), + ); + let queued = bugsee::core::reporting::assemble_docs( + Vec::new(), + TimeWindow { start: 0, end: 1 }, + &bugsee::core::reporting::manual_upload_meta(), + &env, + None, + &[], + "TOKEN_A", + 1, + ) + .unwrap(); + bugsee::core::queue::enqueue(&ro, &queued).unwrap(); + for d in [ro.join("queue"), ro.clone()] { + std::fs::set_permissions(d, std::fs::Permissions::from_mode(0o555)).unwrap(); + } + if std::fs::write(ro.join("probe"), b"x").is_err() { + let transport = Arc::new(MockTransport::default()); + let guard = Bugsee::launch_with( + LaunchOptions::new("TOKEN_A") + .data_dir(&ro) + .with_transport(transport.clone()), + ) + .expect("launch"); + let status = Bugsee::storage_status().expect("status"); + assert_eq!(status.mode, StorageMode::Memory); + assert!(status.explicit_rejected.is_some()); + let start = std::time::Instant::now(); + while transport.uploaded_bundles.lock().unwrap().is_empty() + && start.elapsed() < Duration::from_secs(10) + { + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!( + transport.uploaded_bundles.lock().unwrap().len(), + 1, + "the stranded report must be delivered" + ); + drop(guard); + } + for d in [ro.join("queue"), ro.clone()] { + let _ = std::fs::set_permissions(d, std::fs::Permissions::from_mode(0o755)); + } + } + let _ = std::fs::remove_dir_all(&s); } From f8162d7645c1f4d57418593038617d87cdc3cf91 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:44:21 +0500 Subject: [PATCH 12/52] fix(scrub): second-review hardening of the PII scrubber MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Numeric secrets under secret keys and integer card numbers are redacted; colon-form prose ("invalid token: expected ident") and "my_app::token:" are left alone; Cookie headers take the whole line; escaped quotes inside quoted values are honoured; the "signatures" exemption moves out of the shared key matcher into crash.json handling; tighter token shapes (AWS, hf_, sk-, prose after Bearer); contextual stem exclusions (author, tokenizer, session_state, method_signature); exact key/sig only redact credential-looking strings; request.json no longer double-scrubs metadata; file names (icon@2x.png) and /Users/Shared are not PII; the pre-check handles \u escapes and separated key stems. Test secrets are assembled from halves for push protection. Refs #11 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/reporting.rs | 19 +- crates/bugsee-core/src/scrub.rs | 631 +++++++++++++++--- .../bugsee-core/tests/scrub_ratelimit_e2e.rs | 59 +- 3 files changed, 587 insertions(+), 122 deletions(-) diff --git a/crates/bugsee-core/src/reporting.rs b/crates/bugsee-core/src/reporting.rs index 42c3167..f766c96 100644 --- a/crates/bugsee-core/src/reporting.rs +++ b/crates/bugsee-core/src/reporting.rs @@ -194,7 +194,9 @@ pub fn assemble_docs( } if let Some(mut bytes) = crash_json { - if let Some(clean) = scrubber.and_then(|s| s.scrub_json_bytes(&bytes)) { + if let Some(clean) = + scrubber.and_then(|s| s.scrub_json_bytes_keeping(&bytes, &["signatures"])) + { bytes = clean; } files.push(FileDescriptor::capture("crash.json", "crash")); @@ -239,14 +241,15 @@ pub fn assemble_docs( }; let mut request_json = serde_json::to_vec(&request)?; if let Some(s) = scrubber { - // The environment (paths, host names) is scrubbed; the reporter's own - // `email` is an identity they supplied on purpose and is kept. + // `request` was built from the already-scrubbed metadata (summary, + // description, labels) and the reporter's own `email` is an identity they + // supplied on purpose, so only the environment (paths, host names) is + // scrubbed here. if let Ok(mut v) = serde_json::to_value(&request) { - let email = v.as_object_mut().and_then(|m| m.remove("email")); - if s.scrub_value(&mut v) { - if let (Some(m), Some(e)) = (v.as_object_mut(), email) { - m.insert("email".into(), e); - } + let changed = v + .get_mut("environment") + .is_some_and(|env| s.scrub_value(env)); + if changed { request_json = serde_json::to_vec(&v)?; } } diff --git a/crates/bugsee-core/src/scrub.rs b/crates/bugsee-core/src/scrub.rs index d72fde6..e5e744a 100644 --- a/crates/bugsee-core/src/scrub.rs +++ b/crates/bugsee-core/src/scrub.rs @@ -100,7 +100,8 @@ pub struct ScrubConfig { pub paths: bool, /// Matched values left alone. A match is kept when its text equals an entry, /// or when the entry starts with `@` and the (email) match ends with it, so - /// `"@example.com"` keeps every example.com address. + /// `"@example.com"` keeps every example.com address. `deny` beats `allow`: + /// a literal in `deny` is redacted even where an `allow` entry would keep it. pub allow: Vec, /// Literal substrings that are always redacted (e.g. an internal host name /// or a customer id). @@ -218,10 +219,6 @@ const STEMS: &[&str] = &[ /// Names matched only exactly (too ambiguous to match as substrings). const EXACT: &[&str] = &["key", "sig", "pin", "otp"]; -/// Keys that look like a stem but are SDK-owned structure that must survive -/// (`signatures` is the crash dedup list). -const KEY_EXCEPTIONS: &[&str] = &["signatures"]; - /// Free-text names that are unambiguous: matched as a suffix. const TEXT_STRONG: &[&str] = &[ "password", @@ -269,6 +266,24 @@ fn with_normalized(name: &str, f: impl FnOnce(&[u8]) -> R) -> R { f(&buf[..n]) } +/// Case-insensitive `starts_with` that lets `_`, `-`, space and `.` appear +/// between the letters/digits of `needle` (but not before the first). +fn fuzzy_starts_with(rest: &[u8], needle: &[u8]) -> bool { + let mut ri = 0; + for (ni, &nc) in needle.iter().enumerate() { + if ni > 0 && nc.is_ascii_alphanumeric() { + while ri < rest.len() && matches!(rest[ri], b'_' | b'-' | b' ' | b'.') { + ri += 1; + } + } + if ri >= rest.len() || !rest[ri].eq_ignore_ascii_case(&nc) { + return false; + } + ri += 1; + } + true +} + fn contains(hay: &[u8], needle: &[u8]) -> bool { needle.is_empty() || hay.windows(needle.len()).any(|w| w == needle) } @@ -276,43 +291,122 @@ fn contains(hay: &[u8], needle: &[u8]) -> bool { /// Whether a key (JSON object key, header name, query/form parameter) names a /// value that must be redacted. The single credential-key matcher shared by the /// scrubber and `bugsee-reqwest`. +/// +/// A stem does not count when the surrounding word makes it something else: +/// `author`/`authority`/`:authority` (not `auth`), `tokenizer`/`token_type`/ +/// `token_count` (not `token`), `secretary`, `session_state`/`sessionStart`/ +/// `session_timeout` (not a session id), `method_signature`. Remaining +/// trade-offs: `pwd` also matches the shell's `PWD`/`OLDPWD` variables (their +/// values are a working directory, so the path detector still scrubs the +/// username), and any other key containing a stem (`password_hint`) is redacted. pub fn is_sensitive_key(name: &str) -> bool { with_normalized(name, |n| { - !KEY_EXCEPTIONS.iter().any(|e| n == e.as_bytes()) - && (EXACT.iter().any(|e| n == e.as_bytes()) - || STEMS.iter().any(|s| contains(n, s.as_bytes()))) + EXACT.iter().any(|e| n == e.as_bytes()) || STEMS.iter().any(|s| stem_hit(n, s.as_bytes())) }) } -/// Prefixed token shapes: `(prefix, min tail length, tail needs a digit)`. The -/// tail is a run of `[A-Za-z0-9_-]`. -const TOKEN_PREFIXES: &[(&str, usize, bool)] = &[ - ("sk_live_", 8, false), - ("sk_test_", 8, false), - ("rk_live_", 8, false), - ("rk_test_", 8, false), - ("whsec_", 16, false), - ("ghp_", 20, false), - ("gho_", 20, false), - ("ghu_", 20, false), - ("ghs_", 20, false), - ("ghr_", 20, false), - ("github_pat_", 20, false), - ("glpat-", 15, false), - ("xoxb-", 10, false), - ("xoxp-", 10, false), - ("xoxa-", 10, false), - ("xoxs-", 10, false), - ("xoxr-", 10, false), - ("sk-ant-", 20, false), - ("sk-", 20, true), - ("hf_", 20, false), - ("AKIA", 16, false), - ("ASIA", 16, false), - ("AIza", 30, false), - ("SG.", 20, false), - ("npm_", 30, false), - ("shpat_", 20, false), +/// Whether `stem` occurs in `n` in a sense that is not excluded by its context. +fn stem_hit(n: &[u8], stem: &[u8]) -> bool { + let mut from = 0; + while from + stem.len() <= n.len() { + let Some(p) = n[from..].windows(stem.len()).position(|w| w == stem) else { + return false; + }; + let at = from + p; + let (before, rest) = (&n[..at], &n[at + stem.len()..]); + let excluded = match stem { + b"auth" => rest.starts_with(b"or") && !rest.starts_with(b"oriz"), + b"token" => [ + &b"iz"[..], + b"type", + b"count", + b"len", + b"limit", + b"usage", + b"used", + b"budget", + ] + .iter() + .any(|x| rest.starts_with(x)), + b"secret" => rest.starts_with(b"ary"), + b"session" => [ + &b"start"[..], + b"state", + b"status", + b"timeout", + b"duration", + b"count", + b"length", + b"type", + b"ttl", + b"age", + b"expir", + ] + .iter() + .any(|x| rest.starts_with(x)), + b"signature" => [&b"method"[..], b"function", b"fn", b"type", b"call"] + .iter() + .any(|x| before.ends_with(x)), + _ => false, + }; + if !excluded { + return true; + } + from = at + 1; + } + false +} + +/// What a prefixed token's tail must look like beyond its length, so ordinary +/// identifiers that merely start with the prefix (`sk-cluster-node-01-primary`) +/// do not match. +#[derive(Clone, Copy)] +enum Need { + None, + /// A digit and both letter cases (OpenAI-style `sk-` keys). + DigitAndMixed, + /// A digit or both letter cases. + DigitOrMixed, + /// Exactly 16 characters of `[A-Z0-9]` (AWS access key ids). + Upper16, +} + +/// Prefixed token shapes: `(prefix, min tail length, tail shape)`. The tail is +/// a run of `[A-Za-z0-9_-]`. +const TOKEN_PREFIXES: &[(&str, usize, Need)] = &[ + ("sk_live_", 8, Need::None), + ("sk_test_", 8, Need::None), + ("rk_live_", 8, Need::None), + ("rk_test_", 8, Need::None), + ("whsec_", 16, Need::None), + ("ghp_", 20, Need::None), + ("gho_", 20, Need::None), + ("ghu_", 20, Need::None), + ("ghs_", 20, Need::None), + ("ghr_", 20, Need::None), + ("github_pat_", 20, Need::None), + ("glpat-", 15, Need::None), + ("xoxb-", 10, Need::None), + ("xoxp-", 10, Need::None), + ("xoxa-", 10, Need::None), + ("xoxs-", 10, Need::None), + ("xoxr-", 10, Need::None), + ("sk-ant-", 20, Need::None), + ("sk-", 20, Need::DigitAndMixed), + ("hf_", 20, Need::DigitOrMixed), + ("AKIA", 16, Need::Upper16), + ("ASIA", 16, Need::Upper16), + ("AIza", 30, Need::None), + ("SG.", 20, Need::None), + ("npm_", 30, Need::None), + ("shpat_", 20, Need::None), +]; + +/// "Top-level domains" that are really file extensions (`logo@2x.png`). +const FILE_EXTENSIONS: &[&str] = &[ + "png", "jpg", "jpeg", "gif", "svg", "webp", "ico", "bmp", "tif", "tiff", "css", "js", "ts", + "json", "toml", "yaml", "yml", "txt", "md", "pdf", "html", "htm", "rs", "so", "dll", "dylib", + "exe", "zip", "gz", "mp3", "mp4", "mov", "woff", "woff2", "ttf", ]; /// First bytes of every shape `tokens` can match (a word-start filter). @@ -338,12 +432,31 @@ impl Default for Scrubber { } } +/// How a secret-named structured key is redacted. +#[derive(Clone, Copy)] +struct StructKey { + /// Only strings that look like a credential (see [`looks_like_secret`]). + looks_only: bool, + /// Numbers under the key are redacted too. + numeric: bool, +} + +/// A long, space-free, colon-free string: plausibly a credential rather than an +/// ordinary word or a `user:42:profile` cache key. +fn looks_like_secret(s: &str) -> bool { + s.len() >= 16 && !s.contains(char::is_whitespace) && !s.contains(':') +} + /// Which free-text key class a name belongs to. #[derive(PartialEq, Eq, Clone, Copy)] enum KeyClass { None, Strong, Ambiguous, + /// `Cookie`/`Set-Cookie`: the value runs to the end of the line. + Cookie, + /// A key the host listed in `sensitive_keys`: always redacted. + Explicit, } impl Scrubber { @@ -431,6 +544,18 @@ impl Scrubber { if !self.cfg.enabled { return false; } + // A card number stored as a JSON integer. + if let Value::Number(n) = &*v { + let hit = self.cfg.cards + && n.as_u64().is_some_and(|u| { + let d = u.to_string(); + (13..=19).contains(&d.len()) && self.has_card(&d) + }); + if hit { + *v = Value::String(CARD.to_string()); + } + return hit; + } match v { Value::String(s) => match self.scrub_str(s) { Cow::Borrowed(_) => false, @@ -443,8 +568,8 @@ impl Scrubber { Value::Object(m) => { let mut changed = false; for (k, x) in m.iter_mut() { - if self.cfg.keyed && self.structured_key_is_sensitive(k) { - changed |= self.redact_strings(x); + if let Some(sk) = self.structured_key(k).filter(|_| self.cfg.keyed) { + changed |= self.redact_strings(x, sk); } else { changed |= self.scrub_value(x); } @@ -455,34 +580,88 @@ impl Scrubber { } } - /// Replace every string leaf under `v` (a secret-keyed value) by a marker. - fn redact_strings(&self, v: &mut Value) -> bool { + /// Replace every string leaf under `v` (a secret-keyed value) by a marker; + /// numbers too when the key unambiguously holds a secret (`pin`, `otp`, + /// `password`, ...). Booleans and nulls are never touched. + fn redact_strings(&self, v: &mut Value, sk: StructKey) -> bool { match v { Value::String(s) => { - if s.is_empty() || s == REDACTED || self.allowed(s) { + if s.is_empty() + || s == REDACTED + || self.allowed(s) + || (sk.looks_only && !looks_like_secret(s)) + { false } else { *s = REDACTED.to_string(); true } } - Value::Array(a) => a.iter_mut().fold(false, |c, x| self.redact_strings(x) | c), + Value::Number(_) if sk.numeric => { + *v = Value::String(REDACTED.to_string()); + true + } + Value::Array(a) => a + .iter_mut() + .fold(false, |c, x| self.redact_strings(x, sk) | c), Value::Object(m) => m .values_mut() - .fold(false, |c, x| self.redact_strings(x) | c), + .fold(false, |c, x| self.redact_strings(x, sk) | c), _ => false, } } + /// How a structured key should be redacted, or `None` if it is not secret. + fn structured_key(&self, key: &str) -> Option { + if !self.structured_key_is_sensitive(key) { + return None; + } + Some(with_normalized(key, |n| StructKey { + // `key`/`sig` are also ordinary words (`{"key":"Enter"}`, cache + // keys): only redact values that look like a credential. + looks_only: n == b"key" || n == b"sig", + numeric: n == b"pin" + || n == b"otp" + || TEXT_STRONG.iter().any(|k| n.ends_with(k.as_bytes())) + || self.extra_keys.iter().any(|k| n.ends_with(k.as_bytes())), + })) + } + + fn has_card(&self, digits: &str) -> bool { + let mut spans = Vec::new(); + self.cards(digits, &mut spans); + !spans.is_empty() + } + /// Scrub a serialized JSON document. `None` when nothing changed (the /// caller keeps its original bytes). Unparseable input is scrubbed as text. pub fn scrub_json_bytes(&self, bytes: &[u8]) -> Option> { + self.scrub_json_bytes_keeping(bytes, &[]) + } + + /// [`scrub_json_bytes`](Self::scrub_json_bytes), leaving the given TOP-LEVEL + /// keys untouched (SDK-owned structure such as `crash.json`'s `signatures`, + /// which the `signature` key stem would otherwise redact). + pub fn scrub_json_bytes_keeping(&self, bytes: &[u8], keep: &[&str]) -> Option> { if !self.cfg.enabled { return None; } match serde_json::from_slice::(bytes) { Ok(mut v) => { - if self.scrub_value(&mut v) { + let held: Vec<(String, Value)> = match &mut v { + Value::Object(m) => keep + .iter() + .filter_map(|k| m.remove(*k).map(|x| ((*k).to_string(), x))) + .collect(), + _ => Vec::new(), + }; + let changed = self.scrub_value(&mut v); + if let Value::Object(m) = &mut v { + for (k, x) in held { + m.insert(k, x); + } + } + if changed { serde_json::to_vec(&v).ok() } else { None @@ -560,10 +739,15 @@ impl Scrubber { Some(out) } - /// A conservative byte-level pre-check: `false` guarantees the scrubber - /// would not change `raw` (a JSON text), so callers can skip parsing it. - /// `true` just means "look closer". Always `true` when user-defined deny - /// literals or sensitive keys are configured (they are arbitrary). + /// A conservative byte-level pre-check: `false` means the scrubber would + /// not change `raw` (a JSON text), so callers can skip parsing it. `true` + /// just means "look closer". Always `true` when user-defined deny literals + /// or sensitive keys are configured (they are arbitrary), and whenever the + /// text contains a `\u` escape (the checked substrings could be spelled + /// through it). Key stems are matched allowing `_`, `-`, space and `.` + /// between letters (`p_w_d`, `P-W-D`), which is the separator set real + /// keys use; the scrubber's normaliser would accept any punctuation, so + /// the guarantee assumes those four separators. pub fn might_match(&self, raw: &[u8]) -> bool { if !self.cfg.deny.is_empty() || !self.extra_keys.is_empty() { return true; @@ -609,7 +793,7 @@ impl Scrubber { let mut digits = 0usize; let mut gap = 0usize; for (i, &c) in raw.iter().enumerate() { - if c == b'@' || c == b'=' { + if c == b'@' || c == b'=' || (c == b'\\' && raw.get(i + 1) == Some(&b'u')) { return true; } if c.is_ascii_digit() { @@ -629,10 +813,7 @@ impl Scrubber { if lc.is_ascii_alphabetic() || matches!(lc, b'-' | b':') { let rest = &raw[i..]; for n in NEEDLES { - if n[0] == lc - && rest.len() >= n.len() - && rest[..n.len()].eq_ignore_ascii_case(n) - { + if n[0] == lc && fuzzy_starts_with(rest, n) { return true; } } @@ -689,9 +870,11 @@ impl Scrubber { fn text_key_class(&self, key: &str) -> KeyClass { with_normalized(key, |n| { - if TEXT_STRONG.iter().any(|k| n.ends_with(k.as_bytes())) - || self.extra_keys.iter().any(|k| n.ends_with(k.as_bytes())) - { + if n.ends_with(b"cookie") { + KeyClass::Cookie + } else if self.extra_keys.iter().any(|k| n.ends_with(k.as_bytes())) { + KeyClass::Explicit + } else if TEXT_STRONG.iter().any(|k| n.ends_with(k.as_bytes())) { KeyClass::Strong } else if EXACT.iter().any(|k| n == k.as_bytes()) || TEXT_AMBIGUOUS_SUFFIX @@ -738,6 +921,11 @@ impl Scrubber { if tld_len < 2 { continue; } + // `icon@2x.png` is an asset name, not an address. + let tld = &dom[dot + 1..dot + 1 + tld_len]; + if FILE_EXTENSIONS.iter().any(|x| tld.eq_ignore_ascii_case(x)) { + continue; + } let en = at + 1 + dot + 1 + tld_len; if self.allowed(&s[st..en]) { continue; @@ -892,6 +1080,10 @@ impl Scrubber { if class == KeyClass::None { continue; } + // `my_app::token: refreshed` - a path segment, not a key. + if j >= 2 && b[j - 1] == b':' && b[j - 2] == b':' { + continue; + } // Value. let mut k = i + 1; while k < b.len() && matches!(b[k], b' ' | b'\t') { @@ -910,17 +1102,22 @@ impl Scrubber { }; let vs = k; let mut ve = match quote { - Some(q) => { + Some(q) if escaped => { let e = b[vs..] .iter() .position(|&c| c == q) .map_or(b.len(), |p| vs + p); - if escaped && e > vs && b[e - 1] == b'\\' { + if e > vs && b[e - 1] == b'\\' { e - 1 } else { e } } + Some(q) => find_close(b, vs, q), + None if class == KeyClass::Cookie => b[vs..] + .iter() + .position(|&c| c == b'\r' || c == b'\n') + .map_or(b.len(), |p| vs + p), None if vs < b.len() && matches!(b[vs], b'{' | b'[') => bracketed_end(b, vs), None => { if vrun.0 <= vs && vs < vrun.1 { @@ -977,15 +1174,24 @@ impl Scrubber { } ve = e; } + // Bounded looks (the first 64 bytes), never O(len) per attempt. + let head = &b[vs..ve.min(vs + 64)]; if class == KeyClass::Ambiguous { // Ordinary words ("auth: failed", "session: started") are not - // secrets: need `=` or a quoted key and a secret-looking value. - let v = &b[vs..ve]; - // Bounded look: a digit among the first 64 bytes (never O(len) per attempt). - let secretish = v.len() >= 8 && v[..v.len().min(64)].iter().any(u8::is_ascii_digit); + // secrets: need `=` or a quoted key and a secret-looking value + // (8+ chars with a digit; `user:42:profile` cache keys excluded). + let secretish = + ve - vs >= 8 && head.iter().any(u8::is_ascii_digit) && !head.contains(&b':'); if !(sep == b'=' || quoted_key) || !secretish { continue; } + } else if class == KeyClass::Strong && sep == b':' && !quoted_key && quote.is_none() { + // Prose, not a credential: "invalid token: expected ident", + // "password: must be at least 8". A bare `token: abc123` still is. + let loose = ve - vs >= 6 && !head.iter().all(u8::is_ascii_lowercase); + if !loose { + continue; + } } if self.allowed(&s[vs..ve]) { continue; @@ -1134,6 +1340,8 @@ impl Scrubber { // `/home/feed` is a route, `/home/bob/x` a path; a bare // `/Users/bob` or `X:\Users\bob` still names the user. || (!deeper && !windows && root == 4) + // Shared system folders, not a person. + || matches!(name, "Shared" | "Public" | "Default") || self.allowed(name) { continue; @@ -1147,6 +1355,23 @@ impl Scrubber { } } +/// Position of the closing `q` of a quoted value starting at `vs`, skipping +/// backslash-escaped quotes (`"a\"b"`); the end of the input if unterminated. +fn find_close(b: &[u8], vs: usize, q: u8) -> usize { + for i in vs..b.len() { + if b[i] == q { + let mut k = 0; + while i > vs + k && b[i - 1 - k] == b'\\' { + k += 1; + } + if k % 2 == 0 { + return i; + } + } + } + b.len() +} + /// End of a `{...}` / `[...]` value starting at `vs` (matching close, or the /// end of the string when unbalanced). fn bracketed_end(b: &[u8], vs: usize) -> usize { @@ -1173,6 +1398,8 @@ struct Run { start: usize, end: usize, last_digit: Option, + last_upper: Option, + last_lower: Option, } impl Run { @@ -1180,10 +1407,14 @@ impl Run { fn end_from(&mut self, b: &[u8], t: usize) -> usize { if !(self.start <= t && t < self.end) { let mut e = t; - let mut last_digit = None; + let (mut last_digit, mut last_upper, mut last_lower) = (None, None, None); while e < b.len() && is_token_char(b[e]) { if b[e].is_ascii_digit() { last_digit = Some(e); + } else if b[e].is_ascii_uppercase() { + last_upper = Some(e); + } else if b[e].is_ascii_lowercase() { + last_lower = Some(e); } e += 1; } @@ -1191,6 +1422,8 @@ impl Run { start: t, end: e, last_digit, + last_upper, + last_lower, }; } self.end @@ -1200,19 +1433,36 @@ impl Run { fn digit_from(&self, t: usize) -> bool { self.last_digit.is_some_and(|d| d >= t) } + + /// Whether the run, from `t` on, has both upper- and lower-case letters. + fn mixed_from(&self, t: usize) -> bool { + self.last_upper.is_some_and(|d| d >= t) && self.last_lower.is_some_and(|d| d >= t) + } } /// `(start, end)` of a prefixed token starting at `i`. fn prefixed_token(b: &[u8], i: usize, run: &mut Run) -> Option<(usize, usize)> { let rest = &b[i..]; - for &(prefix, min_tail, need_digit) in TOKEN_PREFIXES { + for &(prefix, min_tail, need) in TOKEN_PREFIXES { // Compare the first byte inline; most words stop here. if rest[0] != prefix.as_bytes()[0] || !rest.starts_with(prefix.as_bytes()) { continue; } let t = i + prefix.len(); let e = run.end_from(b, t); - if e - t >= min_tail && (!need_digit || run.digit_from(t)) { + let ok = e - t >= min_tail + && match need { + Need::None => true, + Need::DigitAndMixed => run.digit_from(t) && run.mixed_from(t), + Need::DigitOrMixed => run.digit_from(t) || run.mixed_from(t), + Need::Upper16 => { + e - t == 16 + && b[t..e] + .iter() + .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) + } + }; + if ok { return Some((i, e)); } } @@ -1265,13 +1515,24 @@ fn credential_after_scheme(b: &[u8], i: usize) -> Option<(usize, usize)> { } let ts = j; // Base64/URL-safe credential chars: token chars plus `. ~ + / =`. - let mut mixed = false; + // `mixed`: a digit or one of `. ~ + / =` (a plain `-`/`_` joined word like + // "token-based" is prose); `cased`: both letter cases. + let (mut mixed, mut upper, mut lower) = (false, false, false); while j < b.len() && (is_token_char(b[j]) || matches!(b[j], b'.' | b'~' | b'+' | b'/' | b'=')) { - mixed |= !b[j].is_ascii_alphabetic(); + mixed |= b[j].is_ascii_digit() || matches!(b[j], b'.' | b'~' | b'+' | b'/' | b'='); + upper |= b[j].is_ascii_uppercase(); + lower |= b[j].is_ascii_lowercase(); j += 1; } - let min = if strict { 8 } else { 6 }; - (j - ts >= min && (!strict || mixed)).then_some((ts, j)) + let len = j - ts; + // "Bearer tokens are ..." is prose; a credential has a digit/symbol, mixed + // case, or is long. + let ok = if strict { + len >= 8 && mixed + } else { + len >= 6 && (mixed || (upper && lower) || len >= 20) + }; + ok.then_some((ts, j)) } fn is_word(c: u8) -> bool { @@ -1365,6 +1626,12 @@ mod tests { Scrubber::default().scrub_str(x).into_owned() } + /// Join halves at runtime so no whole secret-shaped literal appears in the + /// source (GitHub push protection rejects those). + fn j(a: &str, b: &str) -> String { + format!("{a}{b}") + } + #[test] fn clean_strings_are_borrowed() { let sc = Scrubber::default(); @@ -1395,26 +1662,56 @@ mod tests { #[test] fn tokens() { - assert_eq!(s("k=sk_live_abcdefgh12345678"), "k=[token]"); - assert_eq!(s("ghp_abcdefghijklmnopqrstuvwxyz0123456789"), "[token]"); - assert_eq!(s("hf_abcdefghijklmnopqrstuvwxyzABCDEF"), "[token]"); - assert_eq!(s("whsec_abcdefghijklmnopqrstuvwx"), "[token]"); - assert_eq!(s("id AKIAIOSFODNN7EXAMPLE x"), "id [token] x"); - let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk"; + assert_eq!(s(&j("k=sk_", "live_abcdefgh12345678")), "k=[token]"); + assert_eq!( + s(&j("gh", "p_abcdefghijklmnopqrstuvwxyz0123456789")), + "[token]" + ); + assert_eq!(s(&j("hf", "_abcdefghijklmnopqrstuvwxyzABCDEF")), "[token]"); + assert_eq!(s(&j("whsec", "_abcdefghijklmnopqrstuvwx")), "[token]"); + assert_eq!( + s(&format!("id {}{} x", "AKI", "AIOSFODNN7EXAMPLE")), + "id [token] x" + ); + let jwt = format!( + "{}.{}.{}", + j("ey", "JhbGciOiJIUzI1NiJ9"), + j("ey", "JzdWIiOiIxMjM0NTY3ODkwIn0"), + "dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk" + ); assert_eq!(s(&format!("t {jwt}!")), "t [token]!"); assert_eq!(s("Bearer abcdef0123456789"), "Bearer [token]"); assert_eq!(s("Bearer: abc123"), "Bearer: [token]"); assert_eq!(s("Bearer abc123"), "Bearer [token]"); - assert_eq!( - s("sk-learn-something-long-enough"), - "sk-learn-something-long-enough" - ); + } + + #[test] + fn token_shapes_are_tight() { + // Prose and identifiers that merely start like a token stay readable. + for t in [ + "bearer tokens are rotated daily", + "Bearer token-based auth", + "sk-cluster-node-01-primary-east", + "sk-learn-something-long-enough", + "AKIA is the prefix", + concat!("AKI", "AIOSFODNN7EXAMPLEXX"), + "AKIAiosfodnn7example1234", + "hf_abcdefghijklmnopqrstuvwxyz", + ] { + assert_eq!(s(t), t, "{t}"); + } + // Real shapes still match. + assert_eq!(s(&format!("{}{}", "AKI", "AIOSFODNN7EXAMPLE")), "[token]"); + assert_eq!(s(&j("sk-", "ProjAbCdEfGhIjKlMnOp0123456789xyz")), "[token]"); + assert_eq!(s(&j("hf_", "abcdEFGHijklMNOPqrstUVWX")), "[token]"); } #[test] fn pem_private_keys_and_url_userinfo() { - let pem = "x -----BEGIN RSA PRIVATE KEY-----\nMIIEow\nIBAAKCAQEA\n-----END RSA PRIVATE KEY----- y"; - assert_eq!(s(pem), "x [token] y"); + let begin = j("-----BEGIN RSA ", "PRIVATE KEY-----"); + let end = j("-----END RSA ", "PRIVATE KEY-----"); + let pem = format!("x {begin}\nMIIEow\nIBAAKCAQEA\n{end} y"); + assert_eq!(s(&pem), "x [token] y"); let cert = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----"; assert_eq!(s(cert), cert); assert_eq!( @@ -1512,9 +1809,10 @@ mod tests { #[test] fn overlapping_spans_merge_without_leaking_the_tail() { - let out = s("a@b.com-ghp_abcdefghijklmnopqrstuvwxyz0123456789 end"); + let g = j("gh", "p_abcdefghijklmnopqrstuvwxyz0123456789"); + let out = s(&format!("a@b.com-{g} end")); assert!(!out.contains("ghp_") && !out.contains("abcdefgh"), "{out}"); - let out = s("redis://u:ghp_abcdefghijklmnopqrstuvwxyz0123456789@h.example.com/x"); + let out = s(&format!("redis://u:{g}@h.example.com/x")); assert!(!out.contains("abcdefghijkl"), "{out}"); } @@ -1562,6 +1860,7 @@ mod tests { "session_ttl": 30, "signatures": ["a1b2c3"] }); + let keep = v.clone(); assert!(sc.scrub_value(&mut v)); assert_eq!(v["message"], "user [email] failed"); assert_eq!(v["headers"]["Authorization"], "[redacted]"); @@ -1573,10 +1872,18 @@ mod tests { assert_eq!(v["list"][0], "/Users/[user]/x"); assert_eq!(v["api_key"][0], "[redacted]"); assert_eq!(v["api_key"][1]["b"], "[redacted]"); - assert_eq!(v["api_key"][1]["n"], 3); + // Numbers under an unambiguous secret key are redacted too. + assert_eq!(v["api_key"][1]["n"], "[redacted]"); assert_eq!(v["api_key"][2], true); assert_eq!(v["session_ttl"], 30); - assert_eq!(v["signatures"][0], "a1b2c3"); + // `signatures` is only exempt where the SDK says so (crash.json). + assert_eq!(v["signatures"][0], "[redacted]"); + let out = sc + .scrub_json_bytes_keeping(&serde_json::to_vec(&keep).unwrap(), &["signatures"]) + .unwrap(); + let v2: Value = serde_json::from_slice(&out).unwrap(); + assert_eq!(v2["signatures"][0], "a1b2c3"); + assert_eq!(v2["message"], "user [email] failed"); assert!(sc.scrub_json_bytes(br#"{"a":"fine"}"#).is_none()); } @@ -1596,9 +1903,9 @@ mod tests { "debug_id": "41111111-1111-1111-1111-111111111111" }] }); - let mut v = crash.clone(); - assert!(!sc.scrub_value(&mut v)); - assert_eq!(v, crash); + assert!(sc + .scrub_json_bytes_keeping(&serde_json::to_vec(&crash).unwrap(), &["signatures"]) + .is_none()); assert!(sc .scrub_envelope(br#"{"version":2,"events":[{"t":"app::auth::x"}]}"#) .is_none()); @@ -1630,6 +1937,143 @@ mod tests { assert_eq!(s("/Users/Ρ‘Π»ΠΊΠ°/x"), "/Users/[user]/x"); } + #[test] + fn numeric_secrets_and_card_integers() { + let sc = Scrubber::default(); + let mut v = serde_json::json!({ + "pin": 1234, "otp": 654321, "password": 12345678, "secret": [1, 2], + "token_count": 7, "retry": 3, "key": 42, + "card": 4111111111111111u64, "ts": 1712345678901u64, "ok": true + }); + assert!(sc.scrub_value(&mut v)); + assert_eq!(v["pin"], "[redacted]"); + assert_eq!(v["otp"], "[redacted]"); + assert_eq!(v["password"], "[redacted]"); + assert_eq!(v["secret"][1], "[redacted]"); + assert_eq!(v["token_count"], 7); + assert_eq!(v["key"], 42); + assert_eq!(v["card"], "[card]"); + assert_eq!(v["ts"], 1712345678901u64); + assert_eq!(v["ok"], true); + } + + #[test] + fn colon_prose_is_not_a_secret() { + for t in [ + "invalid token: expected ident", + "unexpected token: `}`", + "password: must be at least 8 characters", + "INFO my_app::token: refreshed", + "secret: required", + ] { + assert_eq!(s(t), t, "{t}"); + } + assert_eq!(s("password: hunter2"), "password: [redacted]"); + assert_eq!(s("token: abc123def456"), "token: [redacted]"); + assert_eq!( + s(r#"{"password": "short"}"#), + r#"{"password": "[redacted]"}"# + ); + } + + #[test] + fn cookie_header_takes_the_whole_line() { + assert_eq!( + s("Cookie: a=1; sid=abcdef1234; theme=dark\nHost: x"), + "Cookie: [redacted]\nHost: x" + ); + assert_eq!(s("Set-Cookie: id=1; Path=/"), "Set-Cookie: [redacted]"); + } + + #[test] + fn escaped_quotes_inside_quoted_values() { + assert_eq!( + s(r#"{"password": "a\"bcdefgh tail", "n": 1}"#), + r#"{"password": "[redacted]", "n": 1}"# + ); + } + + #[test] + fn stem_exceptions_and_key_value_looks() { + for k in [ + "author", + "authority", + ":authority", + "tokenizer", + "token_type", + "token_count", + "method_signature", + "secretary", + "sessionStart", + "session_state", + "session_timeout", + "keyboard", + "monkey", + ] { + assert!(!is_sensitive_key(k), "{k}"); + } + for k in [ + "authorization", + "auth", + "oauth_token", + "access_token", + "client_secret", + "sessionId", + "session", + "signature", + "signatures", + "AccountKey", + "x-api-key", + "PWD", + "p_w_d_token", + ] { + assert!(is_sensitive_key(k), "{k}"); + } + let sc = Scrubber::default(); + let mut v = serde_json::json!({"key": "Enter", "sig": "ok", "author": "Bob"}); + assert!(!sc.scrub_value(&mut v)); + let mut v = serde_json::json!({"key": "abcdEFGH1234abcdEFGH1234", "other": {"key": "user:42:profile"}}); + assert!(sc.scrub_value(&mut v)); + assert_eq!(v["key"], "[redacted]"); + assert_eq!(v["other"]["key"], "user:42:profile"); + assert_eq!(s("key=user:42:profile"), "key=user:42:profile"); + assert_eq!(s("key=abcd1234efgh5678"), "key=[redacted]"); + } + + #[test] + fn file_names_and_shared_folders_are_not_pii() { + assert_eq!(s("icon@2x.png logo@3x.JPG"), "icon@2x.png logo@3x.JPG"); + assert_eq!(s("a@example.com"), "[email]"); + assert_eq!(s("/Users/Shared/Library/x"), "/Users/Shared/Library/x"); + // deny beats allow. + let sc = Scrubber::new(ScrubConfig::default().allow("acme").deny("acme")); + assert_eq!(sc.scrub_str("acme corp").as_ref(), "[redacted] corp"); + } + + /// `might_match == false` must mean "scrubbing changes nothing". + #[test] + fn prefilter_never_skips_a_record_that_would_change() { + let sc = Scrubber::default(); + for rec in [ + r#"{"m":"a@b.com"}"#, + r#"{"password":"x"}"#, + r#"{"m":"4111 1111 1111 1111"}"#, + r#"{"p_w_d":"hunter2"}"#, + r#"{"P-W-D":"x"}"#, + r#"{"p w d":"x"}"#, + r#"{"m":"\u0061@b.com"}"#, + r#"{"\u0070assword":"x"}"#, + r#"{"card":4111111111111111}"#, + r#"{"m":"/home/bob/x"}"#, + r#"{"m":"Authorization: Bearer abc123"}"#, + ] { + if sc.scrub_json_bytes(rec.as_bytes()).is_some() { + assert!(sc.might_match(rec.as_bytes()), "{rec}"); + } + } + assert!(!sc.might_match(br#"{"timestamp":1,"level":2,"message":"connection reset"}"#)); + } + /// Deterministic pseudo-random (xorshift) soup of every trigger fragment. #[test] fn fuzz_never_panics() { @@ -1669,7 +2113,7 @@ mod tests { "ζ—₯", "Bearer ", "Basic ", - "-----BEGIN PRIVATE KEY-----", + concat!("-----BEGIN ", "PRIVATE KEY-----"), "-----END ", "://", "x", @@ -1702,6 +2146,11 @@ mod tests { let limit = std::time::Duration::from_secs(if cfg!(debug_assertions) { 20 } else { 2 }); for unit in [ "sk-", + "cookie:", + "password: \"a\\\"", + "\\\"", + "hf_", + "bearer: ", "eyJ-", "eyJ.", "auth:", @@ -1717,7 +2166,7 @@ mod tests { "Bearer ", "basic ", "://", - "-----BEGIN PRIVATE KEY-----", + concat!("-----BEGIN ", "PRIVATE KEY-----"), "ghp_", "AKIA", "[", diff --git a/crates/bugsee-core/tests/scrub_ratelimit_e2e.rs b/crates/bugsee-core/tests/scrub_ratelimit_e2e.rs index 815bf7b..203d99c 100644 --- a/crates/bugsee-core/tests/scrub_ratelimit_e2e.rs +++ b/crates/bugsee-core/tests/scrub_ratelimit_e2e.rs @@ -40,28 +40,41 @@ impl Drop for TempDir { } } -// Split so the literal never appears whole in source (push protection / secret scanners). -const STRIPE_LIKE: &str = concat!("sk_", "live_", "4eC39HqLyjWDarjtT1zdp7dc"); +// Secret-shaped values are assembled from halves so no whole literal sits in the +// source (GitHub push protection rejects those). +const GH: &str = concat!("gh", "p_abcdefghijklmnopqrstuvwxyz0123456789"); +const SK: &str = concat!("sk_", "live_4eC39HqLyjWDarjtT1zdp7dc"); +const AWS: &str = concat!("AKI", "AIOSFODNN7EXAMPLE"); +const JWT: &str = concat!( + "ey", + "JhbGciOiJIUzI1NiJ9.", + "ey", + "JzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk" +); -const SECRETS: &[&str] = &[ - "alice.smith@corp-example.com", - STRIPE_LIKE, - "ghp_abcdefghijklmnopqrstuvwxyz0123456789", - "AKIAIOSFODNN7EXAMPLE", - "4111 1111 1111 1111", - "hunter2-super-pw", - "/Users/jdoe/", - "/home/jdoe/", - "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk", -]; +fn secrets() -> Vec { + [ + "alice.smith@corp-example.com", + SK, + GH, + AWS, + "4111 1111 1111 1111", + "hunter2-super-pw", + "/Users/jdoe/", + "/home/jdoe/", + JWT, + ] + .iter() + .map(|s| s.to_string()) + .collect() +} /// One string packed with every kind of secret. fn dirty() -> String { format!( - "mail alice.smith@corp-example.com key {STRIPE_LIKE} \ - gh ghp_abcdefghijklmnopqrstuvwxyz0123456789 aws AKIAIOSFODNN7EXAMPLE \ - card 4111 1111 1111 1111 password=hunter2-super-pw at /Users/jdoe/proj/a.rs \ - and /home/jdoe/b.rs jwt eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk" + "mail alice.smith@corp-example.com key {SK} gh {GH} aws {AWS} \ + card 4111 1111 1111 1111 password=hunter2-super-pw at /Users/jdoe/proj/a.rs \ + and /home/jdoe/b.rs jwt {JWT}" ) } @@ -170,9 +183,9 @@ fn seeded_secrets_in_every_report_section_do_not_survive_by_default() { if name == ".apptoken" { continue; } - for secret in SECRETS { + for secret in secrets() { assert!( - !body.contains(secret), + !body.contains(&secret), "{secret:?} survived in {name}: {body}" ); } @@ -322,15 +335,15 @@ fn in_memory_mode_reports_are_scrubbed_too() { if name == ".apptoken" { continue; } - for secret in SECRETS { - assert!(!body.contains(secret), "{secret:?} survived in {name}"); + for secret in secrets() { + assert!(!body.contains(&secret), "{secret:?} survived in {name}"); } } } for req in transport.created_requests.lock().unwrap().iter() { let body = String::from_utf8_lossy(req); - for secret in SECRETS { - assert!(!body.contains(secret), "{secret:?} in request.json"); + for secret in secrets() { + assert!(!body.contains(&secret), "{secret:?} in request.json"); } } } From 10271f3187f81b463900d5ed2be891271414b9ad Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:52:50 +0500 Subject: [PATCH 13/52] fix(core): harden the file-lock claims (second review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A launch that cannot hold .lock removes the file instead of leaving an unheld one, which every peer read as "that session died" and recovered (and deleted) the live session. Contended claims are retried briefly; an empty, fresh marker with a free lock is skipped (the pid is written after locking); on unix a probe uses F_GETLK and never takes the lock. - Filesystems that reject locks (ENOLCK, EOPNOTSUPP, ENOSYS, EINVAL) degrade to thread-only claims with a one-time message, and probes report unknown so recovery falls back to the marker pid, instead of blocking the queue forever. - An unclean shutdown (detached worker/uploader) keeps the session lock until the process exits so peers cannot recover a session still being written. - Registry keys are canonicalised; a fork child resets the registry from the fork hook; old-scheme .claim/.claim.takeover files are swept. - Tests: the real pid-namespace hazard (marker pid that does not exist here, reaped pid), crashed sessions with free lock files, the lock-race launch, unsupported-lock injection, and the probe/launch ordering cases. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 7 +- crates/bugsee-core/src/claim.rs | 293 +++++++++++++++++++---- crates/bugsee-core/src/queue.rs | 2 +- crates/bugsee-core/src/recovery.rs | 126 +++++++++- crates/bugsee-core/src/runtime.rs | 10 + crates/bugsee-core/src/session.rs | 100 +++++++- crates/bugsee-core/tests/multiprocess.rs | 60 ++++- 7 files changed, 545 insertions(+), 53 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index a2d1009..41ab698 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -204,10 +204,13 @@ Crash recovery needs the data directory to **outlive the process**: on a contain - **Pid namespaces.** Containers sharing a volume each see their app as pid 1; a pid (or pid plus start time) written in a file means nothing to the other. A lock does not depend on it, so a peer's live session is never mistaken for a crash and a dead one is never kept alive by pid collision. The `.alive` marker's pid is consulted only for a marker with no `.lock` beside it, i.e. one written by an older SDK build. - **Why `fcntl`, not `flock`/OFD locks.** `fcntl` locks are per process and are not inherited across `fork`, so a prefork child never holds its parent's claim; `flock`/OFD locks follow the open file description and would. (MSRV 1.88 predates `File::lock` anyway, hence libc/FFI.) - **Costs of per-process locks, handled in `claim.rs`.** They do not exclude threads, so a registry of held paths does (keyed by pid and spin-locked with the holder's pid stamped in, so a `fork` child that inherits it mid-update breaks the lock instead of deadlocking). POSIX drops a process's locks on a file when *any* descriptor for it closes, so a lock file is never opened twice by one process: the registry is checked before each open, and a claim closes its descriptor before leaving the registry. -- **Ordering at launch.** `Session::begin` creates the marker (`O_EXCL`), takes `.lock`, then writes the pid. Until the pid is written the marker is empty and fresh, which peers skip. +- **Ordering at launch.** `Session::begin` creates the marker (`O_EXCL`), takes `.lock`, then writes the pid. Until the pid is written the marker is empty and fresh, which peers skip even if the lock reads free. A lock that cannot be held with kernel enforcement (contended past a short retry, or the filesystem rejects locks) is *removed*: an existing but unheld lock file reads as "that session died", so peers fall back to the marker's pid instead. On unix a probe asks `F_GETLK` and never takes the lock, so it cannot steal one from a launching peer. +- **Unclean shutdown.** If the worker or uploader thread is detached at shutdown it may still be writing the session, so the session's lock is kept until the process exits (the kernel then drops it) rather than released on drop. +- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown so recovery uses the marker's pid. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. +- **Paths.** The in-process registry canonicalises the lock file's directory, so `/tmp` vs `/private/tmp` or relative vs absolute paths are one claim. A `fork` child clears the registry (`claim::reset_after_fork`, from the fork hook) before claiming anything. - **Stale scans.** After winning a session claim the marker is re-checked, since a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s: a peer's in-flight `enqueue` looks like an orphan, and reaping it lost the report. Lock files of finished items and dead sessions are swept when free. - **Compatibility.** The `.alive` marker is still a bare pid and the lock files are new names, so older builds sharing the directory keep working. They do not take part in the locking, so a mixed-version directory (a rolling upgrade) gives no exactly-once guarantee between old and new processes, and a new process treats an old build's session by its pid as before. Run one SDK version per directory for the guarantee. -- **Limits.** Advisory locks on network filesystems are best effort (NFS needs lockd; some mounts do not support them, in which case a claim fails closed: the item is skipped, never double-handled). The only timestamps relied on are the 30 s grace for an empty marker and the 60 s orphan grace, which assume clocks within seconds. Delivery is at-least-once: a holder that dies after uploading but before removing the item (or after enqueueing a recovered report but before discarding the session) makes its successor do it again. Platforms with neither unix nor Windows locks exclude threads only. +- **Limits.** Advisory locks on network filesystems are best effort (see above). The only timestamps relied on are the 30 s grace for an empty marker and the 60 s orphan grace, which assume clocks within seconds. Delivery is at-least-once: a holder that dies after uploading but before removing the item (or after enqueueing a recovered report but before discarding the session) makes its successor do it again. Platforms with neither unix nor Windows locks exclude threads only. Tests: `bugsee-core/tests/multiprocess.rs` (real child processes: racing launchers, a SIGKILLed holder, a live peer whose marker names a colliding pid) and unit tests in `claim.rs`, `process.rs`, `queue.rs`. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index ea80538..4911f0a 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -31,27 +31,48 @@ //! * They do not exclude *threads* of one process, so an in-process registry of //! held paths does that. //! * POSIX drops a process's locks on a file when it closes **any** descriptor -//! for that file. So a lock file is never opened twice by one process: the -//! registry is consulted before every open, and a claim closes its descriptor -//! *before* leaving the registry. +//! for that file. So a lock file is never opened twice by one process: every +//! open (claim or probe) registers the path first, and a claim closes its +//! descriptor *before* leaving the registry. Registry keys are canonicalised +//! (`/tmp` and `/private/tmp`, relative and absolute, are one file). //! -//! Advisory locks on network filesystems are best effort (NFS needs lockd). The -//! SDK relies on no file timestamps for claims; the only ones it reads are the -//! grace windows for half-written files in `recovery` and `queue`, which assume -//! clocks that agree to within seconds. +//! **Unsupported locks.** Some filesystems reject byte-range locks outright (NFS +//! without lockd: `ENOLCK`; FUSE/9p: `EOPNOTSUPP`/`ENOSYS`/`EINVAL`). The base +//! scheme (`O_EXCL` files) worked there, so refusing to claim would be a +//! regression: a queue that never drains. Instead such a claim degrades to +//! thread-only exclusion (announced once on stderr), and a probe reports +//! [`Probe::Unknown`] so the caller falls back to its pid check. +//! +//! **Locks that succeed but do not exclude** (Docker Desktop bind mounts, WSL2 +//! drvfs/9p, NFS mounted `nolock`) cannot be detected. Processes sharing a +//! directory over such a mount can deliver a report twice; run one SDK host per +//! directory there. +//! +//! The SDK relies on no file timestamps for claims; the only ones it reads are +//! the grace windows for half-written files in `recovery` and `queue`, which +//! assume clocks that agree to within seconds. //! //! None of this runs on the crash path; it allocates freely. use std::cell::UnsafeCell; use std::fs::{File, OpenOptions}; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; /// Holds a claim; dropping it releases the claim. #[derive(Debug)] pub struct Claim { - path: PathBuf, + key: PathBuf, file: Option, + os_locked: bool, +} + +impl Claim { + /// Whether the kernel enforces this claim against other processes. `false` + /// means the filesystem rejected locks and only threads are excluded. + pub(crate) fn is_os_locked(&self) -> bool { + self.os_locked + } } impl Drop for Claim { @@ -59,14 +80,14 @@ impl Drop for Claim { // Close (releasing the OS lock) strictly before leaving the registry, so // a thread that acquires next never has its lock dropped by our close. drop(self.file.take()); - registry().remove(&self.path); + registry().remove(&self.key); } } /// Try to claim `path` (created if absent). `None` if another process, or another /// thread of this one, holds it, or if it cannot be claimed. pub fn try_claim(path: &Path) -> Option { - match acquire(path, true) { + match acquire(path) { Acquire::Got(claim) => Some(claim), Acquire::Held | Acquire::Unavailable => None, } @@ -76,23 +97,40 @@ pub fn try_claim(path: &Path) -> Option { pub(crate) enum Probe { Held, Free, + /// The filesystem cannot say (it rejects locks): use another liveness signal. + Unknown, } /// Is `path` currently claimed? A missing file is free. A file that cannot be /// inspected is reported held: the conservative answer never disturbs a peer. +/// +/// On unix this asks (`F_GETLK`) without taking the lock, so a probe can never +/// steal the lock from a creator that has made the file but not yet locked it. pub(crate) fn probe(path: &Path) -> Probe { if !path.exists() { return Probe::Free; } - match acquire(path, false) { - Acquire::Got(claim) => { - drop(claim); - Probe::Free + let key = key_of(path); + if !registry().insert(&key) { + return Probe::Held; // held by a thread of this process, or being probed + } + let result = match OpenOptions::new().read(true).write(true).open(path) { + Ok(file) => { + let r = match test_is_locked(&file) { + Ok(Peek::Free) => Probe::Free, + Ok(Peek::Held) => Probe::Held, + Ok(Peek::Unsupported) => Probe::Unknown, + Err(_) => Probe::Held, + }; + drop(file); // close before leaving the registry + r } // Vanished since the existence check: nothing holds it. - Acquire::Unavailable if !path.exists() => Probe::Free, - Acquire::Held | Acquire::Unavailable => Probe::Held, - } + Err(_) if !path.exists() => Probe::Free, + Err(_) => Probe::Held, + }; + registry().remove(&key); + result } /// Remove the lock file at `path` if nobody holds it (a dead holder's leftover). @@ -103,55 +141,132 @@ pub(crate) fn remove_if_free(path: &Path) { } } +/// Forget every registry entry. Called in a `fork` child before it claims +/// anything: the child inherits the parent's registry (entries it does not own, +/// possibly mid-update under the parent's spin lock) and none of its locks. +pub fn reset_after_fork() { + registry().reset(); +} + enum Acquire { Got(Claim), Held, Unavailable, } -fn acquire(path: &Path, create: bool) -> Acquire { - if !registry().insert(path) { +fn acquire(path: &Path) -> Acquire { + let key = key_of(path); + if !registry().insert(&key) { return Acquire::Held; // another thread of this process } let file = match OpenOptions::new() .read(true) .write(true) - .create(create) + .create(true) .truncate(false) .open(path) { Ok(f) => f, Err(_) => { - registry().remove(path); + registry().remove(&key); return Acquire::Unavailable; } }; - match os::try_lock(&file) { - Ok(true) => Acquire::Got(Claim { - path: path.to_path_buf(), + match test_try_lock(&file) { + Ok(Lock::Got) => Acquire::Got(Claim { + key, file: Some(file), + os_locked: true, }), - Ok(false) => { + Ok(Lock::Contended) => { drop(file); - registry().remove(path); + registry().remove(&key); Acquire::Held } + Ok(Lock::Unsupported) => { + static ANNOUNCED: AtomicBool = AtomicBool::new(false); + crate::util::warn_once( + &ANNOUNCED, + "this filesystem does not support file locks; processes sharing the data \ + directory cannot be kept from delivering the same report twice", + ); + Acquire::Got(Claim { + key, + file: Some(file), + os_locked: false, + }) + } Err(_) => { drop(file); - registry().remove(path); + registry().remove(&key); Acquire::Unavailable } } } +/// The registry key for `path`: its directory canonicalised, so every spelling of +/// one file maps to one key. Falls back to the path as given. +fn key_of(path: &Path) -> PathBuf { + match (path.parent(), path.file_name()) { + (Some(parent), Some(name)) => { + let parent = if parent.as_os_str().is_empty() { + Path::new(".") + } else { + parent + }; + std::fs::canonicalize(parent) + .map(|p| p.join(name)) + .unwrap_or_else(|_| path.to_path_buf()) + } + _ => path.to_path_buf(), + } +} + +/// Outcome of an attempt to take a lock. +enum Lock { + Got, + Contended, + Unsupported, +} + +/// Outcome of asking whether a lock is held. +enum Peek { + Free, + Held, + Unsupported, +} + +#[cfg(test)] +thread_local! { + /// Test hook: make this thread's lock calls behave as on a filesystem that + /// rejects locks. + pub(crate) static FORCE_UNSUPPORTED: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +fn test_try_lock(file: &File) -> std::io::Result { + #[cfg(test)] + if FORCE_UNSUPPORTED.with(|f| f.get()) { + return Ok(Lock::Unsupported); + } + os::try_lock(file) +} + +fn test_is_locked(file: &File) -> std::io::Result { + #[cfg(test)] + if FORCE_UNSUPPORTED.with(|f| f.get()) { + return Ok(Peek::Unsupported); + } + os::is_locked(file) +} + /// The paths this process currently holds, keyed by pid so that a `fork` child, /// which inherits a copy of this memory, ignores the parent's entries. /// /// A hand-rolled spin lock rather than a `Mutex`: if another thread held a /// `Mutex` at the instant of `fork`, the child's copy would stay locked forever -/// and its first claim would deadlock. Here the lock word records the holder's -/// pid, so a child that finds it held by a different pid knows the holder does -/// not exist in this process and breaks it. +/// and its first claim would deadlock. The fork hook calls [`reset_after_fork`] +/// to clear it; as a second line the lock word records the holder's pid, so a +/// child that finds it held by a different pid breaks it. struct Registry { owner: AtomicU32, held: UnsafeCell>, @@ -221,32 +336,80 @@ impl Registry { let pid = std::process::id(); self.with(|held| held.retain(|(p, q)| !(*p == pid && q == path))); } + + /// Discard everything, including a lock word left held by a thread that does + /// not exist in this process. Only for a freshly forked, single-threaded child. + fn reset(&self) { + // SAFETY: the caller is the only thread in the process, so nothing else + // can be using the registry; the old vector is leaked, not dropped, as its + // allocator state may be mid-update from the parent's other threads. + unsafe { std::ptr::write(self.held.get(), Vec::new()) }; + self.owner.store(0, Ordering::Release); + } } #[cfg(unix)] mod os { + use super::{Lock, Peek}; use std::fs::File; use std::io; use std::os::unix::io::AsRawFd; - /// Take a non-blocking exclusive lock on byte 0. `Ok(false)` if a peer holds it. - pub fn try_lock(file: &File) -> io::Result { + fn whole() -> libc::flock { // SAFETY: an all-zero `flock` is a valid plain-data value. let mut fl: libc::flock = unsafe { std::mem::zeroed() }; fl.l_type = libc::F_WRLCK as _; fl.l_whence = libc::SEEK_SET as _; fl.l_start = 0; fl.l_len = 1; + fl + } + + /// Errors meaning "this filesystem does not do byte-range locks". + fn is_unsupported(errno: i32) -> bool { + errno == libc::ENOLCK + || errno == libc::ENOSYS + || errno == libc::EINVAL + || errno == libc::EOPNOTSUPP + || errno == libc::ENOTSUP + } + + /// Take a non-blocking exclusive lock on byte 0. + pub fn try_lock(file: &File) -> io::Result { + let fl = whole(); loop { // SAFETY: `fd` is a valid open descriptor and `fl` a valid `flock`. let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETLK, &fl) }; if ret == 0 { - return Ok(true); + return Ok(Lock::Got); } let err = io::Error::last_os_error(); match err.raw_os_error() { - Some(libc::EAGAIN) | Some(libc::EACCES) => return Ok(false), + Some(libc::EAGAIN) | Some(libc::EACCES) => return Ok(Lock::Contended), Some(libc::EINTR) => continue, + Some(e) if is_unsupported(e) => return Ok(Lock::Unsupported), + _ => return Err(err), + } + } + } + + /// Ask whether a peer holds the lock, without taking it. + pub fn is_locked(file: &File) -> io::Result { + let mut fl = whole(); + loop { + // SAFETY: as above; `F_GETLK` only writes the conflicting lock into `fl`. + let ret = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETLK, &mut fl) }; + if ret == 0 { + return Ok(if fl.l_type as i32 == libc::F_UNLCK as i32 { + Peek::Free + } else { + Peek::Held + }); + } + let err = io::Error::last_os_error(); + match err.raw_os_error() { + Some(libc::EINTR) => continue, + Some(e) if is_unsupported(e) => return Ok(Peek::Unsupported), _ => return Err(err), } } @@ -255,6 +418,7 @@ mod os { #[cfg(windows)] mod os { + use super::{Lock, Peek}; use std::ffi::c_void; use std::fs::File; use std::io; @@ -284,8 +448,8 @@ mod os { ) -> i32; } - /// Take a non-blocking exclusive lock on byte 0. `Ok(false)` if a peer holds it. - pub fn try_lock(file: &File) -> io::Result { + /// Take a non-blocking exclusive lock on byte 0. + pub fn try_lock(file: &File) -> io::Result { let mut ov = Overlapped { internal: 0, internal_high: 0, @@ -305,21 +469,35 @@ mod os { ) }; if ok != 0 { - return Ok(true); + return Ok(Lock::Got); } let err = io::Error::last_os_error(); match err.raw_os_error() { - Some(ERROR_LOCK_VIOLATION) => Ok(false), + Some(ERROR_LOCK_VIOLATION) => Ok(Lock::Contended), _ => Err(err), } } + + /// Windows has no query-only call, so this takes the lock; the caller closes + /// the handle straight after, which releases it. + pub fn is_locked(file: &File) -> io::Result { + Ok(match try_lock(file)? { + Lock::Got => Peek::Free, + Lock::Contended => Peek::Held, + Lock::Unsupported => Peek::Unsupported, + }) + } } /// No file locks: claims exclude threads of this process only. #[cfg(not(any(unix, windows)))] mod os { - pub fn try_lock(_file: &std::fs::File) -> std::io::Result { - Ok(true) + use super::{Lock, Peek}; + pub fn try_lock(_file: &std::fs::File) -> std::io::Result { + Ok(Lock::Unsupported) + } + pub fn is_locked(_file: &std::fs::File) -> std::io::Result { + Ok(Peek::Unsupported) } } @@ -338,6 +516,7 @@ mod tests { let dir = tmp(); let path = dir.join("x.lock"); let first = try_claim(&path).expect("first claim"); + assert!(first.is_os_locked()); assert!(try_claim(&path).is_none(), "already held"); assert!(matches!(probe(&path), Probe::Held)); drop(first); @@ -365,7 +544,7 @@ mod tests { } #[test] - fn dropping_one_claim_never_releases_another() { + fn probing_never_takes_or_drops_a_lock() { // POSIX drops a process's locks on a file when ANY descriptor for it // closes; the registry must keep a second open from ever happening. let dir = tmp(); @@ -378,6 +557,21 @@ mod tests { "probing must not have closed our lock" ); drop(held); + // And a probe of a free file leaves it free for the next claimer. + assert!(matches!(probe(&path), Probe::Free)); + assert!(try_claim(&path).is_some()); + } + + #[test] + fn one_file_under_two_spellings_is_one_claim() { + let dir = tmp(); + let path = dir.join("x.lock"); + let held = try_claim(&path).unwrap(); + // The same file through a `..` detour. + let detour = dir.join("sub").join("..").join("x.lock"); + std::fs::create_dir_all(dir.join("sub")).unwrap(); + assert!(try_claim(&detour).is_none()); + drop(held); } #[test] @@ -407,4 +601,19 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } } + + #[test] + fn a_filesystem_that_rejects_locks_degrades_to_thread_only_claims() { + let dir = tmp(); + let path = dir.join("x.lock"); + FORCE_UNSUPPORTED.with(|f| f.set(true)); + let first = try_claim(&path).expect("must still claim"); + assert!(!first.is_os_locked()); + assert!(try_claim(&path).is_none(), "threads are still excluded"); + drop(first); + assert!(try_claim(&path).is_some(), "and it is released on drop"); + // A probe cannot say: the caller must use another signal. + assert!(matches!(probe(&path), Probe::Unknown)); + FORCE_UNSUPPORTED.with(|f| f.set(false)); + } } diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index 804b95d..b9620b9 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -271,7 +271,7 @@ const ORPHAN_GRACE: std::time::Duration = std::time::Duration::from_secs(60); /// Whether `path` was last modified at least `grace` ago. Unknown or future-dated /// times count as recent: the conservative answer never disturbs a live peer. -fn untouched_for(path: &Path, grace: std::time::Duration) -> bool { +pub(crate) fn untouched_for(path: &Path, grace: std::time::Duration) -> bool { std::fs::metadata(path) .and_then(|m| m.modified()) .ok() diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 51c6a36..c9d2fd6 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -86,8 +86,22 @@ pub fn find_pending(data_dir: &Path, current_generation: u64) -> Vec continue, + crate::claim::Probe::Free => { + // A launching session locks BEFORE it writes its pid, and a + // marker still empty and fresh may be one that is between + // creating the lock file and locking it. + if marker_is_empty(&marker) && marker_is_fresh(&marker) { + continue; + } + } + // The filesystem rejects locks: fall back to the pid. + crate::claim::Probe::Unknown => { + if !legacy_marker_is_dead(&marker, self_pid) { + continue; + } + } } } else if !legacy_marker_is_dead(&marker, self_pid) { continue; @@ -136,6 +150,10 @@ fn legacy_marker_is_dead(marker: &Path, self_pid: u32) -> bool { } } +fn marker_is_empty(marker: &Path) -> bool { + std::fs::read_to_string(marker).is_ok_and(|s| s.trim().is_empty()) +} + /// Grace window for an empty (pid-not-yet-written) liveness marker: a launching /// peer writes its pid within microseconds of the O_EXCL create, so a marker /// still empty after this is a dead session, not a live peer mid-launch. @@ -398,6 +416,22 @@ pub fn gc_orphans(data_dir: &Path, current_generation: u64) { crate::claim::remove_if_free(&entry.path()); } } + // Leftovers of an earlier claim scheme, once they are plainly dead. + let old_gen = name + .strip_suffix(".claim.takeover") + .or_else(|| name.strip_suffix(".claim")) + .and_then(|g| g.parse::().ok()); + if let Some(generation) = old_gen { + if generation < current_generation + && !alive(generation) + && crate::queue::untouched_for( + &entry.path(), + std::time::Duration::from_secs(60), + ) + { + let _ = std::fs::remove_file(entry.path()); + } + } } } } @@ -712,3 +746,91 @@ fn crash_meta(signatures: Vec, trigger: TriggerType) -> ReportMeta { attachments: Vec::new(), } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::claim::{try_claim, FORCE_UNSUPPORTED}; + use crate::session::Session; + + fn tmp() -> PathBuf { + let p = std::env::temp_dir().join(format!("bugsee-rec-{}", crate::util::random_hex(8))); + std::fs::create_dir_all(p.join("sessions")).unwrap(); + p + } + + fn marker(dir: &Path, generation: u64, content: &str) -> PathBuf { + let m = Session::marker_path(dir, generation); + std::fs::write(&m, content).unwrap(); + m + } + + fn gens(dir: &Path) -> Vec { + find_pending(dir, u64::MAX) + .iter() + .map(|p| p.generation) + .collect() + } + + #[test] + fn a_free_lock_means_the_session_died_whatever_pid_its_marker_names() { + let dir = tmp(); + marker(&dir, 1, "1"); // a pid that is alive here + std::fs::write(Session::lock_path(&dir, 1), "").unwrap(); + assert_eq!(gens(&dir), vec![1]); + } + + #[test] + fn a_held_lock_means_alive_whatever_pid_its_marker_names() { + let dir = tmp(); + marker(&dir, 1, "4294967290"); // not a pid in this namespace + let _held = try_claim(&Session::lock_path(&dir, 1)).unwrap(); + assert!(gens(&dir).is_empty()); + } + + #[test] + fn a_launching_session_between_lock_file_and_lock_is_not_recovered() { + // Created the lock file, not yet locked it, not yet written its pid. + let dir = tmp(); + let m = marker(&dir, 1, ""); + std::fs::write(Session::lock_path(&dir, 1), "").unwrap(); + assert!(gens(&dir).is_empty(), "fresh and empty: still launching"); + let f = std::fs::OpenOptions::new().write(true).open(&m).unwrap(); + f.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(120)) + .unwrap(); + drop(f); + assert_eq!(gens(&dir), vec![1], "an old empty marker is a dead launch"); + } + + #[test] + fn without_locks_the_marker_pid_decides() { + let dir = tmp(); + marker(&dir, 1, "4294967290"); + std::fs::write(Session::lock_path(&dir, 1), "").unwrap(); + #[cfg(unix)] + { + marker(&dir, 2, "1"); + std::fs::write(Session::lock_path(&dir, 2), "").unwrap(); + } + FORCE_UNSUPPORTED.with(|f| f.set(true)); + let found = gens(&dir); + FORCE_UNSUPPORTED.with(|f| f.set(false)); + // Dead pid: recovered. Live pid (unix): left alone. + assert_eq!(found, vec![1]); + } + + #[test] + fn old_scheme_claim_files_of_dead_sessions_are_swept() { + let dir = tmp(); + let claim = dir.join("sessions").join("1.claim"); + let guard = dir.join("sessions").join("1.claim.takeover"); + for p in [&claim, &guard] { + std::fs::write(p, "").unwrap(); + let f = std::fs::OpenOptions::new().write(true).open(p).unwrap(); + f.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(300)) + .unwrap(); + } + gc_orphans(&dir, 5); + assert!(!claim.exists() && !guard.exists()); + } +} diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 232e8f8..75bba1f 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -440,6 +440,9 @@ impl Recorder { &mut self, sampler: Option>, ) -> std::io::Result<()> { + // The registry of held claims is the parent's, and may be mid-update. + crate::claim::reset_after_fork(); + // The parent's threads are not ours to join; drop their handles without // running anything. std::mem::forget(self.worker.take()); @@ -664,6 +667,13 @@ impl Drop for Recorder { if let Some(session) = &self.session { session.end(); } + } else if !(worker_acked && uploader_acked) { + // A detached thread may still be writing this session's data: keep + // the liveness lock until the process exits so no peer recovers (and + // deletes) the session underneath it. + if let Some(session) = &mut self.session { + session.keep_lock_until_exit(); + } } } } diff --git a/crates/bugsee-core/src/session.rs b/crates/bugsee-core/src/session.rs index 8d75c8c..6a8218e 100644 --- a/crates/bugsee-core/src/session.rs +++ b/crates/bugsee-core/src/session.rs @@ -30,8 +30,9 @@ pub struct Session { /// The OS lock on `.lock`, held for the session's life. Peers sharing the /// data dir (including from another pid namespace) probe it to tell a live /// session from a crashed one; the kernel drops it when this process dies. - /// `None` if the lock file could not be taken (the bare pid in the marker is - /// then all a peer has). + /// `None` when the session has no lock file (the filesystem rejects locks, or + /// the file could not be locked), in which case the pid in the marker is all + /// a peer has and `.lock` is deliberately absent. _lock: Option, } @@ -53,9 +54,8 @@ impl Session { { Ok(mut f) => { // Lock BEFORE writing the pid: the marker is empty (and fresh, - // so peers skip it) until then, which closes the window in - // which it is neither locked nor attributed. - let lock = crate::claim::try_claim(&Session::lock_path(data_dir, candidate)); + // so peers skip it) until then. + let lock = Session::take_lock(data_dir, candidate); // The marker stays a bare pid, readable by older builds. let _ = write!(f, "{}", std::process::id()); break (candidate, lock); @@ -77,6 +77,38 @@ impl Session { }) } + /// Take `.lock`, or leave no lock file at all. + /// + /// A peer's probe never takes the lock (unix), but on Windows it momentarily + /// does, so a contended claim is retried briefly. If it still cannot be held + /// with kernel enforcement the file is removed: a lock file that exists but + /// is not held reads as "that session died" to every peer, which would have + /// them recover (and delete) this live session. Without the file peers fall + /// back to the pid in the marker. + fn take_lock(data_dir: &Path, generation: u64) -> Option { + let path = Session::lock_path(data_dir, generation); + for attempt in 0..20 { + match crate::claim::try_claim(&path) { + Some(claim) if claim.is_os_locked() => return Some(claim), + // Thread-only (the filesystem rejects locks): useless to peers. + Some(_) => break, + None if attempt < 19 => std::thread::sleep(std::time::Duration::from_millis(2)), + None => break, + } + } + let _ = std::fs::remove_file(&path); + None + } + + /// Keep this session's lock until the process exits instead of releasing it + /// when the session is dropped. For an unclean shutdown, where a detached + /// worker or uploader may still be writing this session's data: releasing the + /// lock would let a peer recover the session as a crash and delete its parts + /// underneath them. The kernel drops the lock when the process ends. + pub(crate) fn keep_lock_until_exit(&mut self) { + std::mem::forget(self._lock.take()); + } + /// This session's generation number. pub fn generation(&self) -> u64 { self.generation @@ -142,3 +174,61 @@ fn max_numeric_entry(dir: &Path, strip_suffix: Option<&str>) -> u64 { } max } + +#[cfg(test)] +mod tests { + use super::*; + use crate::claim::{probe, Probe}; + + fn tmp() -> PathBuf { + let p = std::env::temp_dir().join(format!("bugsee-sess-{}", crate::util::random_hex(8))); + std::fs::create_dir_all(&p).unwrap(); + p + } + + #[test] + fn a_live_session_holds_its_lock_and_a_clean_end_removes_it() { + let dir = tmp(); + let session = Session::begin(&dir).unwrap(); + let lock = Session::lock_path(&dir, session.generation()); + assert!(matches!(probe(&lock), Probe::Held)); + session.end(); + assert!(!lock.exists()); + } + + #[test] + fn a_session_that_cannot_hold_its_lock_leaves_no_lock_file_behind() { + // A lock file that exists but is not held reads as "that session died". + let dir = tmp(); + std::fs::create_dir_all(dir.join(SESSIONS_DIR)).unwrap(); + let peer = crate::claim::try_claim(&Session::lock_path(&dir, 1)).unwrap(); + let session = Session::begin(&dir).unwrap(); + assert_eq!(session.generation(), 1); + assert!(session._lock.is_none()); + drop(peer); + assert!(!Session::lock_path(&dir, 1).exists()); + } + + #[test] + fn a_session_on_a_filesystem_without_locks_leaves_no_lock_file() { + let dir = tmp(); + crate::claim::FORCE_UNSUPPORTED.with(|f| f.set(true)); + let session = Session::begin(&dir).unwrap(); + crate::claim::FORCE_UNSUPPORTED.with(|f| f.set(false)); + assert!(session._lock.is_none()); + assert!(!Session::lock_path(&dir, session.generation()).exists()); + } + + #[test] + fn keeping_the_lock_until_exit_survives_dropping_the_session() { + let dir = tmp(); + let mut session = Session::begin(&dir).unwrap(); + let lock = Session::lock_path(&dir, session.generation()); + session.keep_lock_until_exit(); + drop(session); + assert!( + matches!(probe(&lock), Probe::Held), + "a detached thread may still be writing: peers must not recover it" + ); + } +} diff --git a/crates/bugsee-core/tests/multiprocess.rs b/crates/bugsee-core/tests/multiprocess.rs index 9b2ae1e..0e2f700 100644 --- a/crates/bugsee-core/tests/multiprocess.rs +++ b/crates/bugsee-core/tests/multiprocess.rs @@ -101,6 +101,13 @@ fn child_entry() { std::fs::rename(dir.join("READY.tmp"), dir.join("READY")).unwrap(); std::thread::sleep(Duration::from_secs(120)); } + "lockholder" => { + // Hold `sessions/1.lock`, as a peer's probe or launch would for a moment. + let claim = bugsee_core::claim::try_claim(&dir.join("sessions").join("1.lock")); + std::fs::write(dir.join("READY.tmp"), claim.is_some().to_string()).unwrap(); + std::fs::rename(dir.join("READY.tmp"), dir.join("READY")).unwrap(); + std::thread::sleep(Duration::from_secs(120)); + } other => panic!("unknown role {other}"), } } @@ -150,6 +157,8 @@ fn seed(dir: &Path, queued: usize, crashed: u64) { for g in 1..=crashed { // "0" is the portable "owner gone" sentinel. std::fs::write(sessions.join(format!("{g}.alive")), "0").unwrap(); + // The lock file a crashed session leaves behind: present, held by no one. + std::fs::write(sessions.join(format!("{g}.lock")), "").unwrap(); std::fs::create_dir_all(dir.join("parts").join(g.to_string())).unwrap(); } std::fs::write(dir.join("gen"), crashed.to_string()).unwrap(); @@ -274,7 +283,20 @@ fn a_live_peers_session_is_not_mistaken_for_a_crash_whatever_pid_its_marker_name // peer: the parent's own, then init's. Neither may make the peer look dead, // nor make it look alive once it is gone. let marker = dir.join("sessions").join(format!("{generation}.alive")); - for foreign in [std::process::id().to_string(), "1".to_string()] { + // A reaped child's pid: gone from this namespace, as a peer's own pid is. + let mut reaped = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "no_such_test"]) + .stdout(Stdio::null()) + .spawn() + .unwrap(); + let _ = reaped.wait(); + let foreign_pids = [ + std::process::id().to_string(), + "1".to_string(), + "4294967290".to_string(), + reaped.id().to_string(), + ]; + for foreign in foreign_pids { std::fs::write(&marker, &foreign).unwrap(); run_workers(&dir, &log, 3); let _ = std::fs::remove_file(dir.join("GO")); @@ -289,3 +311,39 @@ fn a_live_peers_session_is_not_mistaken_for_a_crash_whatever_pid_its_marker_name run_workers(&dir, &log, 3); assert_eq!(delivered(&log), vec!["recovered-crash".to_string()]); } + +/// A launch that finds `.lock` momentarily held by someone else must not leave +/// an unheld lock file behind: every peer would read it as "that session died" and +/// recover (and delete) the live session. +#[test] +fn a_launch_that_loses_the_lock_race_is_not_recovered_as_a_crash() { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + std::fs::create_dir_all(dir.join("sessions")).unwrap(); + let log = tmp.0.join("delivered.log"); + + let mut holder = spawn("lockholder", &dir, &log); + let ready = dir.join("READY"); + let deadline = Instant::now() + Duration::from_secs(20); + while !ready.exists() { + assert!(Instant::now() < deadline, "holder never started"); + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(std::fs::read_to_string(&ready).unwrap(), "true"); + std::fs::remove_file(&ready).unwrap(); + + // Generation 1 is ours, but its lock file is held by the other process. + let session = bugsee_core::session::Session::begin(&dir).unwrap(); + assert_eq!(session.generation(), 1); + holder.kill().unwrap(); + let _ = holder.wait(); + + // Peers now look at a live session (this process) with no usable lock file. + run_workers(&dir, &log, 3); + assert!( + delivered(&log).is_empty(), + "live session reported as a crash" + ); + assert!(dir.join("sessions").join("1.alive").exists()); + drop(session); +} From fdbb29668fd00156a9ea28978d6dece7fd1889ff Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:53:01 +0500 Subject: [PATCH 14/52] fix: keep the scheduled memory drain independent of the stop flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stop-time memory drain relies on the scheduled pass having had its one attempt; only an explicit flush aborts on stop. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/runtime.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 4abd31e..7c52aad 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1560,7 +1560,9 @@ fn drain_all( /// is set so backing-off reports are retried immediately. Returns whether any /// report was attempted (used to pace `drain_until_empty`). fn drain(shared: &Shared, data_dir: Option<&Path>, backoff_base: Duration, force: bool) -> bool { - let attempted = drain_memory(shared, backoff_base, force, true, None, true); + // Not `honor_stop`: the scheduled pass behaves as before #8, and the + // stop-time drain relies on it having had its one attempt. + let attempted = drain_memory(shared, backoff_base, force, true, None, false); match data_dir { Some(dir) => drain_disk(shared, dir, backoff_base, force) | attempted, None => attempted, @@ -1716,8 +1718,7 @@ fn notify_dropped(shared: &Shared, reason: DropReason) { /// to pick it and to settle it, never across the network), so the queue length is /// the true backlog and [`MAX_MEMORY_REPORTS`] holds. Stops at `deadline`, like /// the disk loop in [`drain_until_empty`]. With `honor_stop`, also stops as soon -/// as the recorder is being stopped (only the stop-time drain itself passes -/// `false`). +/// as the recorder is being stopped (only an explicit flush passes `true`). fn drain_memory( shared: &Shared, backoff_base: Duration, From 716c363a05c83c25f79cbb728bbe0a5c15256875 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:54:48 +0500 Subject: [PATCH 15/52] test: gate unix-only helpers so Windows clippy stays clean (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/tests/storage_failures.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index 175de28..bf04627 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -433,6 +433,7 @@ fn a_failed_put_retries_at_the_put_and_does_not_mint_a_duplicate_issue() { ); } +#[cfg(unix)] /// Lay down what a previous run left behind: a queued report nobody delivered and /// a session that died (marker left, owner gone). fn seed_leftovers(dir: &Path) { @@ -470,6 +471,7 @@ fn seed_leftovers(dir: &Path) { std::fs::write(dir.join("gen"), "1").unwrap(); } +#[cfg(unix)] fn wait_for_bundles(transport: &MockTransport, n: usize) -> usize { let start = std::time::Instant::now(); while transport.uploaded_bundles.lock().unwrap().len() < n From 6e71a459f404207a8db56ed81172a841b2a60b22 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 15:55:54 +0500 Subject: [PATCH 16/52] feat(native): best-effort flush of in-flight events at crash time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A producer only queues an entry; the capture worker appends it later, so the last events before a fault (still in the channel) were lost. Measured: a 1500-event burst followed by SIGSEGV recovered ~50-60 events. After the marker is complete, the fatal-crash handler now does one atomic ticket, one non-blocking write(2) (Windows: SetEvent) to a pre-created wake handle and polls an atomic for at most 50 ms. A helper thread made at launch posts a Flush through the normal FIFO channel and publishes the worker's ack. No allocation or lock on the crashing thread; a stuck worker costs only the budget. A main-thread panic nothing will catch runs the same bounded flush. Tests: core handshake/stuck-worker/deregistration, torn part-tail recovery, and subprocess crash tests (burst then SIGSEGV recovers every event; wedged worker still dies promptly and reports). fork_safety's FileTransport now writes atomically since a dying worker can linger mid-upload. Closes #13 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- PROGRESS.md | 4 +- crates/bugsee-core/src/crash_flush.rs | 367 ++++++++++++++++++++ crates/bugsee-core/src/lib.rs | 1 + crates/bugsee-core/src/runtime.rs | 29 ++ crates/bugsee-core/tests/crash_flush_e2e.rs | 235 +++++++++++++ crates/bugsee-core/tests/recovery_e2e.rs | 66 ++++ crates/bugsee-native/src/lib.rs | 24 ++ crates/bugsee-panic/src/lib.rs | 9 + crates/bugsee/Cargo.toml | 6 + crates/bugsee/tests/crash_inflight.rs | 262 ++++++++++++++ crates/bugsee/tests/fork_safety.rs | 7 +- 12 files changed, 1008 insertions(+), 4 deletions(-) create mode 100644 crates/bugsee-core/src/crash_flush.rs create mode 100644 crates/bugsee-core/tests/crash_flush_e2e.rs create mode 100644 crates/bugsee/tests/crash_inflight.rs diff --git a/DESIGN.md b/DESIGN.md index f2f03f6..a2c41b0 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -273,7 +273,7 @@ That last case previously produced **no report whatsoever** β€” the SDK shut dow ## 12. Crash-time safety, best-effort flush & edge cases -**Best-effort in-flight flush:** at crash, secure the guaranteed artifact first (`PanicSnapshot` / minidump), then attempt flush, then terminate. The crashing thread never touches the queue lock/allocator β€” it does an async-signal-safe wake (atomic flag + `sem_post`/eventfd/self-pipe) and waits ≀~50 ms for the worker to drainβ†’appendβ†’`write_all`β†’ack; no ack β†’ give up. (This best-effort in-flight flush is a **planned** hardening step; the current worker already durably appends each record as it drains, so completed records survive without it.) Panic(unwind): usually succeeds. panic=abort: hook wakes worker before abort. Native fault: defensive only. +**Best-effort in-flight flush (implemented, `bugsee_core::crash_flush`):** a producer only queues; the worker appends later, so entries still in the channel (or mid-append) when the process dies are lost. At crash the handler first secures the guaranteed artifact (marker + frames), then does one atomic ticket increment, one non-blocking `write(2)` (Windows: `SetEvent`) to a pre-created wake handle, and polls an atomic ack with `nanosleep` for at most 50 ms (`INFLIGHT_FLUSH_BUDGET`). A helper thread created at launch (never in the handler) is blocked on that handle; it posts a `Flush` message through the normal FIFO channel and, when the worker acks, publishes the ticket. FIFO order means an ack implies every entry queued before the crash is appended. The crashing thread never takes the queue lock, allocates or serializes, and a stuck/dead worker or poisoned allocator costs exactly the budget (the 5 s watchdog is not involved). The pid recorded at registration makes a fork child skip the flush until its revived recorder re-registers. Panic: the hook runs the same bounded flush for a main-thread panic no guard will catch (covers `panic = "abort"` builds with no native handler; with one, the SIGABRT handler flushes too); an unwinding panic is also drained by the launch guard's `Drop`. Native fault: flushed from every fatal exit of the signal handler / the Windows exception filter, after the marker is complete. **Tradeoffs:** zero hot-path cost and no new on-disk format (no spill file: a torn append is the already-tolerated torn part tail, pinned by a recovery test), but the flush needs a live worker and allocator, so a fault that corrupts the heap or hits the worker thread forfeits it, and what is drained is bounded by throughput (about 3.5k ~450-byte events in 50 ms on a debug build; the remainder is lost). A producer-side pre-serialized ring spilled by the handler would survive a dead worker but taxes every event; not built. **Recursion guard:** `NORMALβ†’PANICKINGβ†’HANDLING_FATALβ†’SECONDARY`; second fatal β†’ tiny marker + terminate, no re-entry. **Quarantine:** Class A (continue) / B (restart) / C (rebuild instance) / D (disable); restart budget (1 immediate, 3/10 min, then disable) β€” never infinite. diff --git a/PROGRESS.md b/PROGRESS.md index 6f6bbeb..53bfda2 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -32,7 +32,7 @@ in [`DESIGN.md`](./DESIGN.md); the wire contract is `bugsee/report-bundle-struct | 0 β€” Harness | βœ… | Workspace, mock 3-step API server, bundle validator, subprocess crash runner, CI **matrix across ubuntu/macOS/windows** (build/test/clippy per platform, fmt once) + pinned-MSRV job. | | 1 β€” Ingestible bundle (core) | βœ… | Sliding-window capture (time/byte/event caps), length-prefixed part streams, envelope/manifest, Zstd method-93 ZIP, signatures. | | 2 β€” Handled errors + caught panics | βœ… | `catch_unwind` guards, panic snapshot, error/cause chains. Follow-ups: `tracing` Layer + `log` adapter done. Subsystem quarantine still open. | -| 3 β€” Fatal + next-launch + correlation | βœ… (core) | Native handler, session generations + liveness markers, next-launch recovery, panic↔SIGABRT correlation, client-side native dedup signature from frame offsets. **New:** Windows native capture β€” SEH/VEH handler, PE CodeView module map, marker + frame capture β€” so all three desktop platforms report fatal crashes. Open: best-effort in-flight flush, full out-of-process minidump. | +| 3 β€” Fatal + next-launch + correlation | βœ… (core) | Native handler, session generations + liveness markers, next-launch recovery, panic↔SIGABRT correlation, client-side native dedup signature from frame offsets. **New:** Windows native capture β€” SEH/VEH handler, PE CodeView module map, marker + frame capture β€” so all three desktop platforms report fatal crashes. In-flight events are now flushed at crash (bounded 50 ms, signal-safe wake of the capture worker; #13). Open: full out-of-process minidump. | | 4 β€” Integrations & APM | βœ… (core) | APM transactions/spans, sysinfo telemetry, reqwest network capture. **New:** the environment is actually populated β€” OS version, kernel, memory, disk, locale, and app identity (`package_id`/`version`/`build`) via `LaunchOptions::app_version`. Open: sessions / release-health. | | 5 β€” Hardening | βœ… (core) | Durable retry queue (backoff, retry cap, blacklist), `before_send`/`before_breadcrumb`, event sampling. **New:** `on_report_dropped` β€” delivery failure is observable at last; `flush()` alone never could distinguish delivered from discarded. Open: general PII scrubber, rate limits. | | 6 β€” Mobile/FFI | βœ… (core) | `bugsee-ffi` C ABI (`include/bugsee.h`). Open: actual iOS/Android target builds + Swift/Kotlin wrappers. | @@ -514,7 +514,7 @@ Rust and are worth evaluating before building one. ### Remaining phase work - Phase 2: subsystem quarantine (auto-disable a repeatedly-faulting integration). -- Phase 3: best-effort in-flight flush at crash; full out-of-process minidump (desktop) β€” see the backend note below, which is the same machinery. +- Phase 3: full out-of-process minidump (desktop) β€” see the backend note below, which is the same machinery. - Phase 4: sessions / release-health. - Phase 5: general PII scrubber, rate limits. - Phase 6: iOS/Android target builds; Swift/Kotlin language wrappers. diff --git a/crates/bugsee-core/src/crash_flush.rs b/crates/bugsee-core/src/crash_flush.rs new file mode 100644 index 0000000..a2a37f6 --- /dev/null +++ b/crates/bugsee-core/src/crash_flush.rs @@ -0,0 +1,367 @@ +// +// crash_flush.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// + +//! Best-effort flush of in-flight capture entries at crash time (DESIGN Β§12). +//! +//! # What is lost without this +//! +//! A producer thread never writes to disk: it pushes the entry into the capture +//! worker's channel and returns. The worker serializes and appends it to the +//! part file a little later. Anything still in the channel (or in the worker's +//! hands) when the process dies is gone, and those are exactly the last +//! breadcrumbs, log lines and network events before the fault, the ones a crash +//! report is read for. The window is normally microseconds, but it grows to +//! many milliseconds under a burst, or when the worker is descheduled. +//! +//! # What this does +//! +//! A fatal-crash handler cannot take the channel lock, allocate, or serialize +//! (the faulting thread may hold the allocator or the lock). So it does not +//! touch the queue at all. It performs one `write(2)` / `SetEvent` on a +//! pre-created wake handle and then **polls an atomic** for a bounded time: +//! +//! 1. a dedicated, ordinary thread (created at launch, long before any crash) +//! is blocked reading that handle; +//! 2. woken, it posts a `Flush` marker to the capture worker through the +//! normal channel. The channel is FIFO, so by the time the worker +//! acknowledges the marker every entry queued before the crash has been +//! serialized and appended to its part file; +//! 3. the helper publishes the acknowledgement in an atomic, which the waiting +//! crash handler observes and then proceeds to terminate. +//! +//! The crash handler's side ([`flush_in_flight`]) uses only an atomic load and +//! `fetch_add`, `getpid`, one non-blocking `write`, `clock_gettime` and +//! `nanosleep` (Windows: `SetEvent`, `Sleep`, `GetTickCount64`) β€” all +//! async-signal-safe. There is no allocation and no lock, and the wait is capped +//! by the caller's budget, so a stuck or dead worker (or a corrupted heap that +//! blocks the helper in the allocator) costs the budget and no more. It is a +//! best effort: when the budget runs out we give up and the process dies +//! exactly as it would have without this module. +//! +//! The expensive work (JSON, file I/O) runs on the worker, with the process +//! otherwise healthy; the crashing thread is only parked. Other threads keep +//! running while a *synchronous* fault is handled, which is what makes +//! this possible, and also why events emitted by other threads in the meantime +//! may also land. +//! +//! # Limits (deliberate, documented tradeoffs) +//! +//! * Costs nothing on the happy path (no extra copy or serialization on the +//! producer) but depends on the worker and allocator being alive. A fault that +//! corrupts the heap, or kills the worker thread itself, forfeits the flush. +//! * Entries are flushed to the OS page cache like every other append; this +//! survives the process dying, not the machine. +//! * A fork child has none of the parent's threads: the pid recorded at +//! registration lets the handler skip the flush there until the child's +//! revived recorder registers again. + +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::thread::JoinHandle; +use std::time::Duration; + +/// Flush requests issued by crash handlers (a ticket counter). +static REQUESTED: AtomicU64 = AtomicU64::new(0); +/// Highest ticket the helper has seen acknowledged by the worker. +static ACKED: AtomicU64 = AtomicU64::new(0); + +/// Owns the helper thread and the wake handle for one capture pipeline. +/// Dropping it deregisters (if it is still the registered one) and stops the +/// thread. +pub struct CrashFlush { + stop: Arc, + thread: Option>, + handle: imp::Handle, +} + +impl CrashFlush { + /// Start the helper thread and make this the process's registered flusher. + /// + /// `flush` is called on the helper thread each time a crash handler asks, and + /// must return `true` once everything queued before the call is on disk. It + /// must bound its own waiting. + pub fn install(flush: impl Fn() -> bool + Send + 'static) -> std::io::Result { + let stop = Arc::new(AtomicBool::new(false)); + let (handle, waiter) = imp::create()?; + let thread_stop = Arc::clone(&stop); + let thread = std::thread::Builder::new() + .name("bugsee-crash-flush".into()) + .spawn(move || { + // `waiter` is closed by its own Drop when the loop ends. + while imp::wait(&waiter) { + if thread_stop.load(Ordering::Acquire) { + break; + } + let target = REQUESTED.load(Ordering::Acquire); + if target <= ACKED.load(Ordering::Acquire) { + continue; + } + let ok = std::panic::catch_unwind(std::panic::AssertUnwindSafe(&flush)) + .unwrap_or(false); + if ok { + ACKED.fetch_max(target, Ordering::AcqRel); + } + } + })?; + imp::register(&handle); + Ok(CrashFlush { + stop, + thread: Some(thread), + handle, + }) + } +} + +impl Drop for CrashFlush { + fn drop(&mut self) { + imp::deregister(&self.handle); + self.stop.store(true, Ordering::Release); + imp::wake(&self.handle); + if let Some(thread) = self.thread.take() { + // The helper polls `stop` only between flushes; a flush is bounded by + // the closure's own timeout, so this join is too. + let _ = thread.join(); + } + imp::close(&mut self.handle); + } +} + +/// Ask the registered capture pipeline to drain everything queued so far, and +/// wait at most `budget` for it to finish. Returns whether it confirmed. +/// +/// **Async-signal-safe**: callable from a signal handler or exception filter. +/// No allocation, no locks, bounded time. Returns `false` immediately when no +/// pipeline is registered or the registration belongs to another process (a +/// `fork()` child). +pub fn flush_in_flight(budget: Duration) -> bool { + imp::request_and_wait(budget) +} + +#[cfg(unix)] +mod imp { + use super::{ACKED, REQUESTED}; + use std::sync::atomic::{AtomicI32, Ordering}; + use std::time::Duration; + + /// Write end of the wake pipe the handler writes to (`-1` when none). + static WAKE_FD: AtomicI32 = AtomicI32::new(-1); + /// Pid that created the registration (a fork child must not use it). + static OWNER_PID: AtomicI32 = AtomicI32::new(0); + + pub struct Handle { + write_fd: i32, + } + + /// The read end, closed when the helper thread drops it. + pub struct Waiter(i32); + + impl Drop for Waiter { + fn drop(&mut self) { + // SAFETY: closing a descriptor this value owns. + unsafe { libc::close(self.0) }; + } + } + + pub fn create() -> std::io::Result<(Handle, Waiter)> { + let mut fds = [0i32; 2]; + // SAFETY: `fds` is a valid two-element array. + if unsafe { libc::pipe(fds.as_mut_ptr()) } != 0 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: both descriptors were just created and are ours. + unsafe { + for fd in fds { + libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC); + } + // The handler must never block on a full pipe. + let fl = libc::fcntl(fds[1], libc::F_GETFL); + libc::fcntl(fds[1], libc::F_SETFL, fl | libc::O_NONBLOCK); + } + Ok((Handle { write_fd: fds[1] }, Waiter(fds[0]))) + } + + /// Block until woken. `false` means the pipe is gone: stop. + pub fn wait(w: &Waiter) -> bool { + let mut buf = [0u8; 64]; + loop { + // SAFETY: reads into a valid buffer. + let n = unsafe { libc::read(w.0, buf.as_mut_ptr().cast(), buf.len()) }; + if n > 0 { + return true; + } + if n < 0 && std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted { + continue; + } + return false; + } + } + + pub fn register(h: &Handle) { + // SAFETY: getpid has no preconditions. + OWNER_PID.store(unsafe { libc::getpid() }, Ordering::Release); + WAKE_FD.store(h.write_fd, Ordering::Release); + } + + pub fn deregister(h: &Handle) { + let _ = WAKE_FD.compare_exchange(h.write_fd, -1, Ordering::AcqRel, Ordering::Acquire); + } + + pub fn wake(h: &Handle) { + let b = 1u8; + // SAFETY: writes one byte from a valid buffer; non-blocking. + unsafe { libc::write(h.write_fd, (&b as *const u8).cast(), 1) }; + } + + pub fn close(h: &mut Handle) { + if h.write_fd >= 0 { + // SAFETY: closing the descriptor this handle owns, once. + unsafe { libc::close(h.write_fd) }; + h.write_fd = -1; + } + } + + fn now_ns() -> u64 { + let mut ts = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + // SAFETY: clock_gettime is async-signal-safe and `ts` is valid. + unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) }; + (ts.tv_sec as u64) + .saturating_mul(1_000_000_000) + .saturating_add(ts.tv_nsec as u64) + } + + pub fn request_and_wait(budget: Duration) -> bool { + let fd = WAKE_FD.load(Ordering::Acquire); + // SAFETY: getpid is async-signal-safe. + if fd < 0 || OWNER_PID.load(Ordering::Acquire) != unsafe { libc::getpid() } { + return false; + } + let ticket = REQUESTED.fetch_add(1, Ordering::AcqRel) + 1; + let b = 1u8; + // SAFETY: async-signal-safe write of one byte. A full pipe (EAGAIN) means + // wakeups are already pending, and the helper reads the ticket counter. + unsafe { libc::write(fd, (&b as *const u8).cast(), 1) }; + let deadline = now_ns().saturating_add(budget.as_nanos().min(u64::MAX as u128) as u64); + while ACKED.load(Ordering::Acquire) < ticket { + if now_ns() >= deadline { + return false; + } + let ts = libc::timespec { + tv_sec: 0, + tv_nsec: 200_000, + }; + // SAFETY: nanosleep is async-signal-safe. + unsafe { libc::nanosleep(&ts, core::ptr::null_mut()) }; + } + true + } +} + +#[cfg(windows)] +mod imp { + use super::{ACKED, REQUESTED}; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + type Hnd = usize; + + extern "system" { + fn CreateEventW(attrs: *const u8, manual: i32, initial: i32, name: *const u16) -> Hnd; + fn SetEvent(h: Hnd) -> i32; + fn WaitForSingleObject(h: Hnd, ms: u32) -> u32; + fn CloseHandle(h: Hnd) -> i32; + fn Sleep(ms: u32); + fn GetTickCount64() -> u64; + } + + /// The auto-reset event the handler signals (`0` when none). + static WAKE: AtomicUsize = AtomicUsize::new(0); + + pub struct Handle(Hnd); + pub struct Waiter(Hnd); + + pub fn create() -> std::io::Result<(Handle, Waiter)> { + // SAFETY: plain Win32 call; auto-reset, initially unsignalled, unnamed. + let h = unsafe { CreateEventW(core::ptr::null(), 0, 0, core::ptr::null()) }; + if h == 0 { + return Err(std::io::Error::last_os_error()); + } + Ok((Handle(h), Waiter(h))) + } + + /// Block until signalled (`WAIT_OBJECT_0`). + pub fn wait(w: &Waiter) -> bool { + // SAFETY: waits on an event handle that outlives the thread (it is closed + // only after the thread is joined). + unsafe { WaitForSingleObject(w.0, u32::MAX) == 0 } + } + + pub fn register(h: &Handle) { + WAKE.store(h.0, Ordering::Release); + } + + pub fn deregister(h: &Handle) { + let _ = WAKE.compare_exchange(h.0, 0, Ordering::AcqRel, Ordering::Acquire); + } + + pub fn wake(h: &Handle) { + // SAFETY: signals a live event. + unsafe { SetEvent(h.0) }; + } + + pub fn close(h: &mut Handle) { + if h.0 != 0 { + // SAFETY: closes the handle this value owns, once. + unsafe { CloseHandle(h.0) }; + h.0 = 0; + } + } + + pub fn request_and_wait(budget: Duration) -> bool { + let h = WAKE.load(Ordering::Acquire); + if h == 0 { + return false; + } + let ticket = REQUESTED.fetch_add(1, Ordering::AcqRel) + 1; + // SAFETY: SetEvent/Sleep/GetTickCount64 take no locks we could be holding. + unsafe { + SetEvent(h); + let deadline = GetTickCount64().saturating_add(budget.as_millis() as u64); + while ACKED.load(Ordering::Acquire) < ticket { + if GetTickCount64() >= deadline { + return false; + } + Sleep(1); + } + } + true + } +} + +#[cfg(not(any(unix, windows)))] +mod imp { + use std::time::Duration; + + pub struct Handle; + pub struct Waiter; + + pub fn create() -> std::io::Result<(Handle, Waiter)> { + Err(std::io::Error::from(std::io::ErrorKind::Unsupported)) + } + pub fn wait(_: &Waiter) -> bool { + false + } + pub fn register(_: &Handle) {} + pub fn deregister(_: &Handle) {} + pub fn wake(_: &Handle) {} + pub fn close(_: &mut Handle) {} + pub fn request_and_wait(_: Duration) -> bool { + false + } +} diff --git a/crates/bugsee-core/src/lib.rs b/crates/bugsee-core/src/lib.rs index cdbc126..fc37236 100644 --- a/crates/bugsee-core/src/lib.rs +++ b/crates/bugsee-core/src/lib.rs @@ -19,6 +19,7 @@ pub mod bundle; pub mod capture; pub mod claim; +pub mod crash_flush; pub mod errors; pub mod fork; pub mod model; diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 232e8f8..d42242f 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -35,6 +35,7 @@ use std::time::{Duration, Instant}; use serde_json::Value; use crate::capture::{export, PartStore, WindowCaps}; +use crate::crash_flush::CrashFlush; use crate::fork::{ForkLock, ForkLockable}; use crate::model::entry::{Breadcrumb, CaptureEntry, TraceEntry}; use crate::model::enums::IssueType; @@ -314,6 +315,9 @@ pub struct Recorder { backoff_base: Duration, worker: Option>, uploader: Option>, + /// Lets a crash handler ask the worker to drain in-flight entries + /// ([`crate::crash_flush`]). `None` in memory mode (nothing to flush to). + crash_flush: Option, } /// The two pipeline threads and the channels that feed them. @@ -322,6 +326,7 @@ struct Pipeline { upload_tx: Sender, worker: JoinHandle<()>, uploader: JoinHandle<()>, + crash_flush: Option, } impl Recorder { @@ -415,6 +420,7 @@ impl Recorder { backoff_base: config.upload_backoff_base, worker: Some(pipeline.worker), uploader: Some(pipeline.uploader), + crash_flush: pipeline.crash_flush, }) } @@ -444,6 +450,9 @@ impl Recorder { // running anything. std::mem::forget(self.worker.take()); std::mem::forget(self.uploader.take()); + // Likewise the parent's crash-flush helper thread (and its wake handle, + // which the parent still uses). + std::mem::forget(self.crash_flush.take()); // Entries queued to the dead worker will never be processed, so the // back-pressure counters they incremented would stay high forever and @@ -496,6 +505,7 @@ impl Recorder { std::mem::forget(std::mem::replace(&mut self.upload_tx, pipeline.upload_tx)); self.worker = Some(pipeline.worker); self.uploader = Some(pipeline.uploader); + self.crash_flush = pipeline.crash_flush; Ok(()) } @@ -622,6 +632,8 @@ impl Recorder { impl Drop for Recorder { fn drop(&mut self) { + // Stop answering crash handlers before the worker goes away. + drop(self.crash_flush.take()); // Signal both threads; JOIN only if they acknowledge within the timeout. // A thread wedged in a host callback with no timeout is DETACHED (its // JoinHandle dropped un-joined) rather than hanging the stopping thread @@ -857,6 +869,7 @@ fn spawn_pipeline( let worker_shared = Arc::clone(shared); let worker_data_dir = data_dir.map(Path::to_path_buf); let worker_upload_tx = upload_tx.clone(); + let crash_flush = store.is_some().then(|| install_crash_flush(&tx)).flatten(); let worker = std::thread::Builder::new() .name("bugsee-capture".into()) @@ -906,9 +919,25 @@ fn spawn_pipeline( upload_tx, worker, uploader, + crash_flush, }) } +/// How long the helper waits for the worker to acknowledge a crash-time flush. +/// The crash handler gives up on its own, sooner; this only bounds the helper. +const CRASH_FLUSH_ACK_TIMEOUT: Duration = Duration::from_millis(250); + +/// Register this pipeline as the process's crash-time flush target. A failure +/// (no pipe/event available) just forgoes the best-effort flush. +fn install_crash_flush(tx: &Sender) -> Option { + let tx = tx.clone(); + CrashFlush::install(move || { + let (ack, done) = channel(); + tx.send(Msg::Flush(ack)).is_ok() && done.recv_timeout(CRASH_FLUSH_ACK_TIMEOUT).is_ok() + }) + .ok() +} + /// Recover any prior generation that ended abnormally, queueing it for delivery. /// The crashed session's state is discarded ONLY after the report is durably /// enqueued β€” a failure leaves it on disk for a later attempt (no crash loss). diff --git a/crates/bugsee-core/tests/crash_flush_e2e.rs b/crates/bugsee-core/tests/crash_flush_e2e.rs new file mode 100644 index 0000000..c03abfc --- /dev/null +++ b/crates/bugsee-core/tests/crash_flush_e2e.rs @@ -0,0 +1,235 @@ +// +// crash_flush_e2e.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! The crash-time flush handshake against a real capture pipeline (issue #13). +//! The crash handler's side is exactly `flush_in_flight`; the subprocess crash +//! tests in the `bugsee` crate prove it from a real fault. +//! +//! The registration is process-global, so these tests run one at a time. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use bugsee_core::capture::record::RecordIter; +use bugsee_core::crash_flush::{flush_in_flight, CrashFlush}; +use bugsee_core::model::entry::{CaptureEntry, EventEntry}; +use bugsee_core::runtime::TelemetrySampler; +use bugsee_core::{MockTransport, Recorder, RecorderConfig}; +use serde_json::{Map, Value}; + +static SERIAL: Mutex<()> = Mutex::new(()); + +struct TempDir(PathBuf); +impl TempDir { + fn new() -> Self { + let p = std::env::temp_dir().join(format!( + "bugsee-cflush-{}", + bugsee_core::util::random_hex(8) + )); + std::fs::create_dir_all(&p).unwrap(); + TempDir(p) + } +} +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn event(i: usize) -> CaptureEntry { + let mut params = Map::new(); + params.insert("pad".into(), Value::String("x".repeat(300))); + CaptureEntry::UserEvent(EventEntry { + timestamp: 1_000 + i as i64, + name: Some(format!("e{i}")), + params, + custom: Map::new(), + }) +} + +/// Count the records the part files hold RIGHT NOW (no flush of ours involved). +fn records_on_disk(data: &Path) -> usize { + let mut n = 0; + let mut stack = vec![data.join("parts")]; + while let Some(dir) = stack.pop() { + let Ok(read) = std::fs::read_dir(&dir) else { + continue; + }; + for e in read.flatten() { + let p = e.path(); + if p.is_dir() { + stack.push(p); + } else if p.to_string_lossy().ends_with("events.user.part") { + n += RecordIter::new(&std::fs::read(&p).unwrap()).count(); + } + } + } + n +} + +#[test] +fn everything_queued_before_the_request_is_on_disk_when_it_confirms() { + let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let dir = TempDir::new(); + let recorder = Recorder::launch( + RecorderConfig::new(&dir.0, "TOKEN"), + Arc::new(MockTransport::default()), + ) + .unwrap(); + + const N: usize = 1500; + for i in 0..N { + recorder.capture(event(i)); + } + // The producer only queued; the worker is still behind. + let behind = records_on_disk(&dir.0); + + assert!( + flush_in_flight(Duration::from_secs(5)), + "a healthy worker confirms" + ); + assert_eq!(records_on_disk(&dir.0), N, "all {N} are durable on confirm"); + eprintln!("on disk before the request: {behind}/{N}"); + drop(recorder); +} + +struct Blocker(Arc); +impl TelemetrySampler for Blocker { + fn sample(&mut self) -> Vec<(String, Value)> { + while !self.0.load(Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(5)); + } + Vec::new() + } +} + +#[test] +fn a_stuck_worker_costs_only_the_budget() { + let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let dir = TempDir::new(); + let release = Arc::new(AtomicBool::new(false)); + let mut cfg = RecorderConfig::new(&dir.0, "TOKEN"); + cfg.rotate_interval = Duration::from_millis(20); + cfg.sampler = Some(Box::new(Blocker(Arc::clone(&release)))); + let recorder = Recorder::launch(cfg, Arc::new(MockTransport::default())).unwrap(); + // Let a tick start and wedge the worker inside the sampler. + std::thread::sleep(Duration::from_millis(150)); + for i in 0..100 { + recorder.capture(event(i)); + } + + let t = Instant::now(); + assert!(!flush_in_flight(Duration::from_millis(50))); + let took = t.elapsed(); + assert!(took < Duration::from_millis(500), "bounded, took {took:?}"); + + // Un-wedge: the same request now succeeds (the helper is not poisoned). + release.store(true, Ordering::SeqCst); + assert!(flush_in_flight(Duration::from_secs(5))); + assert_eq!(records_on_disk(&dir.0), 100); + drop(recorder); +} + +#[test] +fn dropping_the_recorder_deregisters() { + let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let dir = TempDir::new(); + let recorder = Recorder::launch( + RecorderConfig::new(&dir.0, "TOKEN"), + Arc::new(MockTransport::default()), + ) + .unwrap(); + assert!(flush_in_flight(Duration::from_secs(2))); + drop(recorder); + let t = Instant::now(); + assert!(!flush_in_flight(Duration::from_secs(2))); + assert!(t.elapsed() < Duration::from_millis(200), "no wait at all"); +} + +// The handshake itself, against stand-in flushers (no pipeline). + +#[test] +fn unregistered_returns_false_immediately() { + let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let t = Instant::now(); + assert!(!flush_in_flight(Duration::from_millis(500))); + assert!(t.elapsed() < Duration::from_millis(100)); +} + +#[test] +fn a_responsive_flusher_is_confirmed_promptly() { + let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let calls = Arc::new(AtomicUsize::new(0)); + let c = Arc::clone(&calls); + let cf = CrashFlush::install(move || { + c.fetch_add(1, Ordering::SeqCst); + true + }) + .unwrap(); + let t = Instant::now(); + assert!(flush_in_flight(Duration::from_millis(500))); + assert!(t.elapsed() < Duration::from_millis(250)); + assert_eq!(calls.load(Ordering::SeqCst), 1); + drop(cf); + assert!(!flush_in_flight(Duration::from_millis(50)), "deregistered"); +} + +#[test] +fn a_stuck_flusher_costs_the_budget_and_no_more() { + let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let release = Arc::new(AtomicBool::new(false)); + let r = Arc::clone(&release); + let cf = CrashFlush::install(move || { + while !r.load(Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(5)); + } + true + }) + .unwrap(); + let t = Instant::now(); + assert!(!flush_in_flight(Duration::from_millis(50))); + let took = t.elapsed(); + assert!(took >= Duration::from_millis(45), "{took:?}"); + assert!(took < Duration::from_millis(400), "{took:?}"); + release.store(true, Ordering::SeqCst); + drop(cf); +} + +#[test] +fn a_panicking_flusher_does_not_kill_the_helper() { + let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let n = Arc::new(AtomicUsize::new(0)); + let c = Arc::clone(&n); + let cf = CrashFlush::install(move || { + if c.fetch_add(1, Ordering::SeqCst) == 0 { + panic!("boom"); + } + true + }) + .unwrap(); + assert!(!flush_in_flight(Duration::from_millis(50))); + // Let the helper finish the first request: two wakeups that land before it + // reads are served by ONE flush call (which here panics). + while n.load(Ordering::SeqCst) == 0 { + std::thread::sleep(Duration::from_millis(5)); + } + std::thread::sleep(Duration::from_millis(50)); + assert!(flush_in_flight(Duration::from_millis(500))); + drop(cf); +} + +#[test] +fn memory_mode_has_nothing_to_flush() { + let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let dir = TempDir::new(); + let mut cfg = RecorderConfig::new(&dir.0, "TOKEN"); + cfg.memory_only = Some("test".into()); + let recorder = Recorder::launch(cfg, Arc::new(MockTransport::default())).unwrap(); + assert!(!flush_in_flight(Duration::from_millis(50))); + drop(recorder); +} diff --git a/crates/bugsee-core/tests/recovery_e2e.rs b/crates/bugsee-core/tests/recovery_e2e.rs index 4e88f14..456d219 100644 --- a/crates/bugsee-core/tests/recovery_e2e.rs +++ b/crates/bugsee-core/tests/recovery_e2e.rs @@ -363,3 +363,69 @@ fn clean_shutdown_is_not_recovered() { "a cleanly-closed session is never recovered" ); } + +/// A process that dies mid-`write` (or whose in-flight flush was cut short by the +/// crash handler's budget) leaves a torn tail on the part it was appending to. +/// Recovery must deliver every complete record before it and ignore whatever +/// follows β€” never panic, never loop, never read past the buffer. +#[test] +fn a_torn_or_garbage_part_tail_is_ignored_on_recovery() { + let header = |len: u32| { + let mut t = Vec::new(); + t.extend_from_slice(&300u64.to_le_bytes()); + t.extend_from_slice(&len.to_le_bytes()); + t + }; + let mut cut = header(64); + cut.extend_from_slice(br#"{"timestamp":300,"lev"#); + let mut absurd = header(u32::MAX); + absurd.extend_from_slice(b"junk"); + let tails: Vec<(&str, Vec)> = vec![ + ("partial header", vec![0xAB; 5]), + ("header only, payload missing", header(64)), + ("payload cut mid-way", cut), + ("absurd length prefix", absurd), + ("zeroed tail of a grown file", vec![0u8; 4096]), + ( + "random garbage", + (0..777u32).map(|i| (i * 31 + 7) as u8).collect(), + ), + ]; + + for (what, tail) in tails { + let dir = TempDir::new(); + seed_crashed_session(&dir.path, 1); + let log = dir.path.join("parts/1/0/log.part"); + let mut bytes = std::fs::read(&log).unwrap(); + bytes.extend_from_slice(&tail); + std::fs::write(&log, &bytes).unwrap(); + + let transport = Arc::new(MockTransport::default()); + let recorder = + Recorder::launch(RecorderConfig::new(&dir.path, "TOKEN"), transport.clone()).unwrap(); + assert!(recorder.flush(Duration::from_secs(5)), "{what}"); + + let bundles = transport.uploaded_bundles.lock().unwrap(); + assert_eq!(bundles.len(), 1, "{what}: the session was still delivered"); + let mut zip = ZipArchive::new(Cursor::new(bundles[0].clone())).unwrap(); + let log_name = (0..zip.len()) + .map(|i| zip.by_index(i).unwrap().name().to_string()) + .find(|n| n.ends_with(".log.json")) + .unwrap_or_else(|| panic!("{what}: no log file")); + let mut raw = Vec::new(); + zip.by_name(&log_name) + .unwrap() + .read_to_end(&mut raw) + .unwrap(); + let log: Value = serde_json::from_slice(&raw).unwrap(); + let msgs: Vec<&str> = log["events"] + .as_array() + .unwrap() + .iter() + .map(|e| e["message"].as_str().unwrap()) + .collect(); + assert_eq!(msgs, vec!["pre-crash line A", "pre-crash line B"], "{what}"); + drop(bundles); + drop(recorder); + } +} diff --git a/crates/bugsee-native/src/lib.rs b/crates/bugsee-native/src/lib.rs index b6e9620..56815e7 100644 --- a/crates/bugsee-native/src/lib.rs +++ b/crates/bugsee-native/src/lib.rs @@ -194,6 +194,23 @@ mod imp { #[cfg(any(unix, windows))] const HANDLER_BUDGET_SECS: u32 = 5; + /// How long a dying process waits for the capture worker to write out the + /// entries still queued when it faulted (see `bugsee_core::crash_flush`). + /// Small against [`HANDLER_BUDGET_SECS`] on purpose: it is a courtesy, never + /// something the process may be held for, and it runs only AFTER the marker is + /// complete, so giving up costs the last few events and nothing else. + #[cfg(any(unix, windows))] + const INFLIGHT_FLUSH_BUDGET: std::time::Duration = std::time::Duration::from_millis(50); + + /// Best-effort drain of in-flight capture entries. Async-signal-safe (one + /// atomic, one non-blocking write, a bounded sleep loop); the heavy lifting + /// happens on the capture worker. Call only once the marker is on disk. + #[cfg(any(unix, windows))] + #[inline(never)] + fn flush_inflight_events() { + let _ = bugsee_core::crash_flush::flush_in_flight(INFLIGHT_FLUSH_BUDGET); + } + /// A `SIGALRM`-based deadline for the crash handler. Only async-signal-safe /// calls (`sigaction`, `pthread_sigmask`, `sigpending`, `sigwait`, `alarm`) are /// used. @@ -834,6 +851,9 @@ mod imp { if n > 0 { unsafe { append_frames(path_cbytes, &frames[..n]) }; } + // The marker is complete; now give the worker a moment to write out what + // was still queued. + flush_inflight_events(); } /// Open the marker for writing, creating/truncating it. Returns the raw handle @@ -2365,6 +2385,8 @@ mod imp { report_unrecorded(tid, &*path, sig, info, regs); } } + // Only now, with the marker complete and the latch released. + flush_inflight_events(); } /// The interrupted registers, to tell whether a host changed anything. @@ -2616,6 +2638,7 @@ mod imp { unsafe { if generation != 0 { complete_report(tid, &*path, regs, generation); + flush_inflight_events(); return; } // The original fault was not the first in flight, so it wrote no @@ -2637,6 +2660,7 @@ mod imp { watchdog.disarm(); } WRITING.store(0, Ordering::Release); + flush_inflight_events(); } } diff --git a/crates/bugsee-panic/src/lib.rs b/crates/bugsee-panic/src/lib.rs index e0e285d..c4ca1d1 100644 --- a/crates/bugsee-panic/src/lib.rs +++ b/crates/bugsee-panic/src/lib.rs @@ -215,6 +215,11 @@ pub fn install(reporter: Arc) { // emit the same panic twice. let worker_thread = worker_thread_name(); let will_be_caught = guard_active(); + // A main-thread panic nothing will catch is the process's last act under + // `panic = "abort"` (and usually under unwind too): drain the capture + // worker so the events emitted just before it are on disk. Bounded and + // best effort; a no-op when the SDK has no disk pipeline. + let likely_fatal = !will_be_caught && worker_thread.is_none(); if !will_be_caught && (worker_thread.is_some() || REPORT_FROM_HOOK.load(Ordering::SeqCst)) { @@ -247,6 +252,10 @@ pub fn install(reporter: Arc) { frames, }); }); + if likely_fatal { + let _ = + bugsee_core::crash_flush::flush_in_flight(std::time::Duration::from_millis(50)); + } // Preserve prior hook behavior (default abort message, other SDKs). previous(info); })); diff --git a/crates/bugsee/Cargo.toml b/crates/bugsee/Cargo.toml index 3051090..e2183cb 100644 --- a/crates/bugsee/Cargo.toml +++ b/crates/bugsee/Cargo.toml @@ -43,5 +43,11 @@ name = "crash_e2e_matrix" harness = false required-features = ["native"] +# Dies for real after a burst of events; needs its own `main`, like the above. +[[test]] +name = "crash_inflight" +harness = false +required-features = ["native"] + [target.'cfg(unix)'.dev-dependencies] libc = "0.2" diff --git a/crates/bugsee/tests/crash_inflight.rs b/crates/bugsee/tests/crash_inflight.rs new file mode 100644 index 0000000..78a8c8d --- /dev/null +++ b/crates/bugsee/tests/crash_inflight.rs @@ -0,0 +1,262 @@ +// +// crash_inflight.rs +// bugsee +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! Best-effort flush of in-flight events at crash time (issue #13). +//! +//! A producer only queues an entry; the capture worker appends it to disk a +//! little later. These tests die for real, in a child process, immediately after +//! emitting a burst of events, and read what the next launch recovers: +//! +//! * `burst_then_segv` β€” every event emitted before the fault, including the +//! ones still queued when it hit, is in the recovered report. +//! * `wedged_worker_then_segv` β€” the worker is stuck in a host callback, so the +//! flush cannot succeed. The process must still die promptly (the handler is +//! bounded; the 5 s watchdog must not be what ends it) and the crash is still +//! reported, with the events that had already reached disk. +//! +//! `harness = false` (see `Cargo.toml`): the child must fault from the real main +//! thread. + +#![cfg(all(feature = "native", unix))] + +use std::io::{BufRead, BufReader, Cursor, Read, Write}; +use std::process::{Command, Stdio}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use bugsee::core::MockTransport; +use bugsee::{Bugsee, LaunchOptions, LogLevel}; +use serde_json::{Map, Value}; +use zip::ZipArchive; + +const KIND_ENV: &str = "BUGSEE_INFLIGHT_KIND"; +const DIR_ENV: &str = "BUGSEE_INFLIGHT_DIR"; +/// Events in the burst. Sized so the producer (which only queues) finishes well +/// before the worker (which serializes and writes) has drained, yet the worker +/// can drain within the 50 ms handler budget. +const BURST: usize = 1000; + +#[inline(never)] +fn fault_null() { + // SAFETY: deliberately faulting. + unsafe { std::ptr::write_volatile(std::hint::black_box(std::ptr::null_mut::()), 1) } +} + +fn emit_burst(prefix: &str) { + let pad = "x".repeat(400); + for i in 0..BURST { + let mut params = Map::new(); + params.insert("pad".into(), Value::String(pad.clone())); + Bugsee::event_with(format!("{prefix}-{i}"), params); + } +} + +fn run_child(kind: &str, dir: &str) -> ! { + let wedge = kind == "wedged_worker_then_segv"; + let _guard = Bugsee::launch_with( + LaunchOptions::new("APP_TOKEN") + .data_dir(dir) + .native_crash_capture(true) + // Runs on the capture worker: wedging it stands in for a worker blocked + // on a dead disk or a host callback that never returns. + .before_send(move |_| { + if wedge { + loop { + std::thread::sleep(Duration::from_secs(60)); + } + } + #[allow(unreachable_code)] + true + }) + .with_transport(Arc::new(MockTransport::default())), + ) + .expect("launch"); + std::thread::sleep(Duration::from_millis(300)); + + match kind { + "burst_then_segv" => emit_burst("burst"), + "wedged_worker_then_segv" => { + // These reach disk before the worker is wedged... + for i in 0..20 { + Bugsee::event(format!("early-{i}")); + } + Bugsee::flush(Duration::from_secs(2)); + // ...then a report wedges it in `before_send`, and everything after + // is stuck in the queue. + Bugsee::capture_message(LogLevel::Error, "wedge the worker"); + std::thread::sleep(Duration::from_millis(300)); + emit_burst("stuck"); + } + other => panic!("unknown kind {other}"), + } + println!("FAULTING"); + let _ = std::io::stdout().flush(); + fault_null(); + std::process::exit(0); +} + +fn event_names(bundle: &[u8]) -> Vec { + let mut zip = ZipArchive::new(Cursor::new(bundle)).expect("bundle is a zip"); + let mut names = Vec::new(); + for i in 0..zip.len() { + let mut entry = zip.by_index(i).expect("zip entry"); + if !entry.name().contains("events.user") { + continue; + } + let mut raw = Vec::new(); + entry.read_to_end(&mut raw).expect("read events"); + let v: Value = serde_json::from_slice(&raw).expect("events parse"); + for e in v["events"].as_array().cloned().unwrap_or_default() { + if let Some(n) = e["name"].as_str() { + names.push(n.to_string()); + } + } + } + names +} + +fn crash_json(bundle: &[u8]) -> Value { + let mut zip = ZipArchive::new(Cursor::new(bundle)).expect("bundle is a zip"); + let mut entry = zip.by_name("crash.json").expect("crash.json"); + let mut raw = Vec::new(); + entry.read_to_end(&mut raw).unwrap(); + serde_json::from_slice(&raw).unwrap() +} + +fn recover(dir: &std::path::Path) -> Vec> { + let mock = Arc::new(MockTransport::default()); + let guard = Bugsee::launch_with( + LaunchOptions::new("APP_TOKEN") + .data_dir(dir) + .native_crash_capture(false) + .with_transport(mock.clone()), + ) + .expect("relaunch"); + std::thread::sleep(Duration::from_millis(1500)); + Bugsee::flush(Duration::from_secs(10)); + let bundles = mock.uploaded_bundles.lock().unwrap().clone(); + drop(guard); + bundles +} + +/// Run the child to its death; returns the time from its "about to fault" line to +/// its exit. +fn crash_child(kind: &str, dir: &std::path::Path) -> Result { + let mut child = Command::new(std::env::current_exe().unwrap()) + .env(KIND_ENV, kind) + .env(DIR_ENV, dir) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| format!("spawn: {e}"))?; + let mut line = String::new(); + BufReader::new(child.stdout.take().unwrap()) + .read_line(&mut line) + .map_err(|e| e.to_string())?; + let faulted = Instant::now(); + if line.trim() != "FAULTING" { + let _ = child.kill(); + return Err(format!("child never reached the fault: {line:?}")); + } + let status = child.wait().map_err(|e| e.to_string())?; + let took = faulted.elapsed(); + if status.success() { + return Err("child should have died".into()); + } + Ok(took) +} + +fn with_dir( + kind: &str, + f: impl FnOnce(&std::path::Path) -> Result, +) -> Result { + let dir = std::env::temp_dir().join(format!( + "bugsee-inflight-{kind}-{}", + bugsee::core::util::random_hex(6) + )); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + let r = f(&dir); + let _ = std::fs::remove_dir_all(&dir); + r +} + +fn burst_then_segv() -> Result<(), String> { + with_dir("burst", |dir| { + crash_child("burst_then_segv", dir)?; + let bundles = recover(dir); + if bundles.len() != 1 { + return Err(format!("expected one report, got {}", bundles.len())); + } + let crash = crash_json(&bundles[0]); + if crash["signal"]["name"] != "SIGSEGV" { + return Err(format!("not a SIGSEGV report: {crash}")); + } + let names = event_names(&bundles[0]); + let last = format!("burst-{}", BURST - 1); + let have = names.iter().filter(|n| n.starts_with("burst-")).count(); + if !names.contains(&last) { + return Err(format!( + "the last event before the crash was lost ({have}/{BURST} recovered)" + )); + } + if have != BURST { + return Err(format!("only {have}/{BURST} events recovered")); + } + Ok(()) + }) +} + +fn wedged_worker_then_segv() -> Result<(), String> { + with_dir("wedged", |dir| { + let took = crash_child("wedged_worker_then_segv", dir)?; + if took > Duration::from_secs(2) { + return Err(format!("the dying process hung for {took:?}")); + } + let bundles = recover(dir); + if bundles.len() != 1 { + return Err(format!("expected one report, got {}", bundles.len())); + } + let crash = crash_json(&bundles[0]); + let has_frames = crash["frames"].as_array().is_some_and(|f| !f.is_empty()); + if crash["signal"]["name"] != "SIGSEGV" || !has_frames { + return Err(format!("crash report incomplete: {crash}")); + } + let names = event_names(&bundles[0]); + if !names.contains(&"early-19".to_string()) { + return Err(format!("events already on disk were lost: {names:?}")); + } + Ok(()) + }) +} + +type Case = fn() -> Result<(), String>; + +fn main() { + if let Ok(kind) = std::env::var(KIND_ENV) { + run_child(&kind, &std::env::var(DIR_ENV).expect("dir")); + } + let filter = std::env::args().skip(1).find(|a| !a.starts_with('-')); + let cases: [(&str, Case); 2] = [ + ("burst_then_segv", burst_then_segv), + ("wedged_worker_then_segv", wedged_worker_then_segv), + ]; + let mut failed = Vec::new(); + for (name, case) in cases { + if filter.as_deref().is_some_and(|f| !name.contains(f)) { + continue; + } + match case() { + Ok(()) => println!("test crash_inflight::{name} ... ok"), + Err(e) => { + println!("test crash_inflight::{name} ... FAILED\n {e}"); + failed.push(name); + } + } + } + if !failed.is_empty() { + std::process::exit(1); + } +} diff --git a/crates/bugsee/tests/fork_safety.rs b/crates/bugsee/tests/fork_safety.rs index a27b0f5..e72ef5a 100644 --- a/crates/bugsee/tests/fork_safety.rs +++ b/crates/bugsee/tests/fork_safety.rs @@ -64,7 +64,12 @@ impl Transport for FileTransport { fn upload_bundle(&self, _: &str, zip: &[u8]) -> Result<(), TransportError> { let n = self.n.fetch_add(1, Ordering::SeqCst); let name = format!("{}-{n}.zip", std::process::id()); - std::fs::write(self.dir.join(name), zip) + // Write-then-rename: a process killed mid-upload (a crashing worker now + // lingers briefly to flush in-flight events, so one can die with an upload + // under way) must not leave a half-written `.zip` for the test to read. + let tmp = self.dir.join(format!(".{name}.part")); + std::fs::write(&tmp, zip) + .and_then(|()| std::fs::rename(&tmp, self.dir.join(name))) .map_err(|e| TransportError::Transient(e.to_string())) } } From 9ae8a050da2abe6065e20330fc5a93bf2a445447 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:02:16 +0500 Subject: [PATCH 17/52] fix(core): queue the previous crash on the launch path before launch returns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A host that crashes again right after launch (crash loop) could die before the worker thread recovered the previous crash. launch now runs a bounded, newest-first recovery pass (8 sessions, 500 ms budget via LaunchOptions::launch_recovery_budget, 4096-entry scan cap) that enqueues the reports durably before returning; upload stays async and the worker's pass picks up the rest, skipping what the launch pass attempted. Closes #10 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 + crates/bugsee-core/src/recovery.rs | 52 ++++- crates/bugsee-core/src/runtime.rs | 133 ++++++++++-- crates/bugsee-core/tests/launch_recovery.rs | 211 ++++++++++++++++++++ crates/bugsee/src/api.rs | 1 + crates/bugsee/src/options.rs | 19 ++ 6 files changed, 394 insertions(+), 24 deletions(-) create mode 100644 crates/bugsee-core/tests/launch_recovery.rs diff --git a/DESIGN.md b/DESIGN.md index d0d81e2..99a0fef 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -217,6 +217,8 @@ Tests: `bugsee-core/tests/multiprocess.rs` (real child processes: racing launche Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, a memory-mode case in `bugsee/tests/fork_safety.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised, since mounting one needs root; failures are injected by blocking path components and removing the directory mid-run. +**Early-crash recovery on the launch path (#10).** A host that dies again within milliseconds of launch (a crash loop under systemd/Kubernetes) must not outrun the report of the crash before it. `Recorder::launch` therefore runs a bounded recovery pass on the calling thread, after this session's marker exists and before the pipeline threads start: the newest crashed sessions are built, passed through `before_send`/blacklist, assembled and enqueued, and only then is their evidence discarded. Upload stays asynchronous and resumes from the queue. Bounds: at most 8 sessions (`launch_recovery_max_sessions`), a time budget of 500 ms (`LaunchOptions::launch_recovery_budget`, zero disables the pass), and a scan of at most 4096 directory entries, names sorted before any file is touched. The budget is checked between sessions, so one huge session can overrun it, and the newest is always attempted. **Newest first**: the latest crash is the one most likely to belong to the build now running, and a crash loop makes only the most recent evidence reachable; the rest is not lost, since the worker's unbounded pass recovers whatever the launch pass left. Claims (#14) are unchanged: the pass takes the same per-session claim, so it composes with peers and the worker pass, and it runs only for a fresh launch (a fork child keeps recovering on its worker, and holds no lock the fork hook needs). A crash *inside* the pass (or before the native handler is installed right after it) loses nothing: the session's evidence is still on disk, retried next launch up to `MAX_RECOVERY_ATTEMPTS`. Not done: collapsing repeated identical crash-on-launch reports into one (each crash is queued; the backend deduplicates by signature and the TooManySimilar blacklist suppresses them once the server says so). Tests: `bugsee-core/tests/launch_recovery.rs` (a real abort-after-launch crash loop, time cap, zero budget). + **Generations:** each launch bumps `gen`; next-launch recovery reads the **previous** gen's parts before advancing. **Snapshot (no seal, no copy):** pin `[start,end]` (`end`=snapshot instant, `start`=`endβˆ’window`) + **hard-link current parts including the active one** into the report dir (`PartStore::snapshot_into`, after a `flush`). At export, read linked parts and include only entries with `start ≀ timestamp ≀ end` (monotonic order β†’ early-stop at the tail; the self-terminating `RecordIter` cleanly ignores any torn trailing write on the active part). Bounds become `manifest.time.start/end`. Cost: 2 timestamps + O(#parts) `link` syscalls. Fallback to copy on FAT/exFAT/cross-volume (capability-probed once). Reflink (APFS `clonefile` / Linux `FICLONE`) a noted future tier. diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index ad23596..245c7e3 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -61,6 +61,22 @@ pub struct RecoveredReport { /// because it launched earlier) has a live marker that is *not* a crash. Its /// marker is left in place for a future launch to recover once the peer is gone. pub fn find_pending(data_dir: &Path, current_generation: u64) -> Vec { + find_pending_bounded(data_dir, current_generation, usize::MAX, usize::MAX) +} + +/// Like [`find_pending`], for the launch path: newest generation first, and +/// bounded. At most `scan_cap` directory entries are read and at most `limit` +/// sessions returned, so a directory a crash loop has filled does not make launch +/// scan, probe and stat without end. Names are sorted before any file is touched, +/// so only the newest candidates cost a probe. A scan cut off by `scan_cap` may +/// miss the newest session; the worker's unbounded pass picks up whatever the +/// bounded one left. +pub fn find_pending_bounded( + data_dir: &Path, + current_generation: u64, + scan_cap: usize, + limit: usize, +) -> Vec { let mut pending = Vec::new(); let sessions = data_dir.join("sessions"); let read = match std::fs::read_dir(&sessions) { @@ -68,8 +84,8 @@ pub fn find_pending(data_dir: &Path, current_generation: u64) -> Vec return pending, }; let self_pid = std::process::id(); - for entry in read.flatten() { - let marker = entry.path(); + let mut candidates: Vec = Vec::new(); + for entry in read.flatten().take(scan_cap) { let name = entry.file_name().to_string_lossy().into_owned(); let Some(gen_str) = name.strip_suffix(".alive") else { continue; @@ -77,9 +93,16 @@ pub fn find_pending(data_dir: &Path, current_generation: u64) -> Vec() else { continue; }; - if generation >= current_generation { - continue; + if generation < current_generation { + candidates.push(generation); } + } + candidates.sort_unstable_by(|a, b| b.cmp(a)); + for generation in candidates { + if pending.len() >= limit { + break; + } + let marker = sessions.join(format!("{generation}.alive")); // A session whose `.lock` exists is arbitrated by that OS lock: held // means a live process (in any pid namespace), free means it died. The // marker's pid is not consulted, since it is only meaningful inside the @@ -779,6 +802,27 @@ mod tests { .collect() } + #[test] + fn the_bounded_scan_is_newest_first_and_capped() { + let dir = tmp(); + for g in [3u64, 9, 1, 7, 5] { + marker(&dir, g, "0"); + } + let got = |current, scan, limit| -> Vec { + find_pending_bounded(&dir, current, scan, limit) + .iter() + .map(|p| p.generation) + .collect() + }; + assert_eq!(got(u64::MAX, 100, 100), vec![9, 7, 5, 3, 1]); + assert_eq!(got(u64::MAX, 100, 2), vec![9, 7]); + // Only older generations than the caller's own are candidates. + assert_eq!(got(7, 100, 100), vec![5, 3, 1]); + // A scan cut short reads fewer entries (which ones is directory order). + assert!(got(u64::MAX, 2, 100).len() <= 2); + assert!(got(u64::MAX, 0, 100).is_empty()); + } + #[test] fn a_free_lock_means_the_session_died_whatever_pid_its_marker_names() { let dir = tmp(); diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index b771812..2362638 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -53,6 +53,10 @@ const UPLOAD_RETRY_CAP: u32 = 60; /// Maximum next-launch recovery attempts for one pending session before it is /// abandoned (guards against a corrupt-parts poison pill re-running every launch). const MAX_RECOVERY_ATTEMPTS: u32 = 3; +/// Default time `launch` may spend queueing the previous run's crashes. +pub const DEFAULT_LAUNCH_RECOVERY_BUDGET: Duration = Duration::from_millis(500); +/// Default cap on crashed sessions queued on the launch path. +pub const DEFAULT_LAUNCH_RECOVERY_SESSIONS: usize = 8; /// Default in-flight capture-entry cap (drop-newest beyond this). const DEFAULT_MAX_QUEUED: usize = 8192; /// Default in-flight *report* cap. Reports are far heavier than capture entries @@ -179,6 +183,12 @@ pub struct RecorderConfig { /// crashed sessions and deliver its queue. For a host layer that moved its /// default location and must not strand reports left at the old one. pub legacy_data_dir: Option, + /// Time `launch` may spend queueing the previous run's crashes before it + /// returns (see [`Recorder::launch`]). Zero skips the launch-path pass: all + /// recovery then happens on the worker thread after `launch` returns. + pub launch_recovery_budget: Duration, + /// Most crashed sessions the launch-path pass recovers. + pub launch_recovery_max_sessions: usize, } impl RecorderConfig { @@ -201,6 +211,8 @@ impl RecorderConfig { max_queued_entries: DEFAULT_MAX_QUEUED, memory_only: None, legacy_data_dir: None, + launch_recovery_budget: DEFAULT_LAUNCH_RECOVERY_BUDGET, + launch_recovery_max_sessions: DEFAULT_LAUNCH_RECOVERY_SESSIONS, } } } @@ -338,6 +350,14 @@ struct Pipeline { impl Recorder { /// Start the workers and begin capturing. Recovers any prior session that /// ended abnormally, queueing it for delivery. + /// + /// The newest crashed sessions (at most + /// [`RecorderConfig::launch_recovery_max_sessions`], newest first, within + /// [`RecorderConfig::launch_recovery_budget`]) are turned into durable queue + /// entries before this returns, so a host that crashes again immediately has + /// already got the previous crash's report on disk. Upload is asynchronous and + /// resumes from the queue. Anything beyond the caps is recovered by the worker + /// thread right after. pub fn launch(config: RecorderConfig, transport: Arc) -> std::io::Result { let env = Environment::detect_with( &config.sdk_version, @@ -403,6 +423,26 @@ impl Recorder { // recovering them only reads and deletes. So an existing directory is // still drained and recovered from in memory mode. let drain_dir = (session.is_some() || data_dir.is_dir()).then_some(data_dir.as_path()); + // Queue the previous run's crashes before returning (issue #10): a host + // that crashes again right after launch must not outrun their report. + // This runs after this session's marker exists and before the native + // handler is installed; a session's evidence stays on disk until its + // report is queued, so a crash inside this pass just retries next launch + // (bounded by `MAX_RECOVERY_ATTEMPTS`). + let mut attempted = BTreeSet::new(); + if let (Some(dir), false) = (drain_dir, config.launch_recovery_budget.is_zero()) { + let generation = session.as_ref().map_or(u64::MAX, Session::generation); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run_launch_recovery( + &shared, + dir, + generation, + config.launch_recovery_budget, + config.launch_recovery_max_sessions, + &mut attempted, + ) + })); + } let pipeline = spawn_pipeline( &shared, store.is_some().then_some(data_dir.as_path()), @@ -415,6 +455,7 @@ impl Recorder { config.sampler, true, config.legacy_data_dir, + attempted, )?; Ok(Recorder { @@ -508,6 +549,7 @@ impl Recorder { sampler, false, None, + BTreeSet::new(), )?; // The parent's session is still the parent's: leave its marker alone. @@ -861,6 +903,7 @@ fn spawn_pipeline( sampler: Option>, fresh_launch: bool, legacy_data_dir: Option, + launch_attempted: BTreeSet, ) -> std::io::Result { // Uploader thread: drains the durable queue with retry. let (upload_tx, upload_rx) = channel(); @@ -913,13 +956,13 @@ fn spawn_pipeline( // it must, or that crash waits for the master to restart. Claims // keep siblings from recovering the same session twice. if let Some(dir) = &worker_drain_dir { - run_recovery(&worker_shared, dir, generation); + run_recovery(&worker_shared, dir, generation, &launch_attempted); } // Best effort, once: sessions a previous default location left // behind. No generation of ours lives there, so none is skipped. if let Some(dir) = &legacy_data_dir { queue::gc_orphans(dir); - run_recovery(&worker_shared, dir, u64::MAX); + run_recovery(&worker_shared, dir, u64::MAX, &BTreeSet::new()); } })); legacy_ready.store(true, Ordering::SeqCst); @@ -948,31 +991,81 @@ fn spawn_pipeline( /// Recover any prior generation that ended abnormally, queueing it for delivery. /// The crashed session's state is discarded ONLY after the report is durably /// enqueued β€” a failure leaves it on disk for a later attempt (no crash loss). -fn run_recovery(shared: &Shared, data_dir: &Path, current_generation: u64) { +/// +/// `skip` holds generations the launch-path pass already attempted: one it could +/// only hold in memory still has its evidence on disk, and doing it again would +/// report it twice. +fn run_recovery(shared: &Shared, data_dir: &Path, current_generation: u64, skip: &BTreeSet) { for pending in recovery::find_pending(data_dir, current_generation) { - // Several processes share this directory (a prefork master and its - // workers, above all): exactly one recovers a given session. - let Some(_claim) = recovery::claim_pending(data_dir, &pending) else { - continue; - }; - // Count the attempt *before* processing: a corrupt session that panics - // the build/assemble path is a poison pill that would otherwise re-run β€” - // and abort the whole loop β€” on every launch. After the cap, abandon it. - let attempts = recovery::bump_attempts(data_dir, &pending); - if attempts > MAX_RECOVERY_ATTEMPTS { - recovery::discard(data_dir, &pending); - continue; + if !skip.contains(&pending.generation) { + recover_claimed(shared, data_dir, &pending); } - // Isolate each pending so one corrupt session cannot block recovering the - // rest (a panic here would otherwise unwind the entire loop). - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - recover_one(shared, data_dir, &pending); - })); } // Reap parts/attempts left by cleanly-exited or already-recovered generations. recovery::gc_orphans(data_dir, current_generation); } +/// Claim one pending session and recover it. A session another process holds is +/// skipped. +fn recover_claimed(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSession) { + // Several processes share this directory (a prefork master and its + // workers, above all): exactly one recovers a given session. + let Some(_claim) = recovery::claim_pending(data_dir, pending) else { + return; + }; + // Count the attempt *before* processing: a corrupt session that panics + // the build/assemble path is a poison pill that would otherwise re-run β€” + // and abort the whole loop β€” on every launch. After the cap, abandon it. + let attempts = recovery::bump_attempts(data_dir, pending); + if attempts > MAX_RECOVERY_ATTEMPTS { + recovery::discard(data_dir, pending); + return; + } + // Isolate each pending so one corrupt session cannot block recovering the + // rest (a panic here would otherwise unwind the entire loop). + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + recover_one(shared, data_dir, pending); + })); +} + +/// Directory entries the launch-path pass reads before giving up on finding more. +const LAUNCH_SCAN_CAP: usize = 4096; + +/// The launch-path recovery pass (issue #10): turn the newest crashed sessions +/// into durable queue entries BEFORE `launch` returns, so a host that dies again +/// within milliseconds (a crash loop under a supervisor) cannot outrun the report +/// of the crash before it. Only the queueing is synchronous; upload stays on the +/// uploader thread. +/// +/// Bounded three ways: at most `max_sessions` are recovered, the scan reads at +/// most [`LAUNCH_SCAN_CAP`] directory entries, and no new session is started once +/// `budget` has elapsed. The budget is checked between sessions, so one session +/// can overrun it; the newest session is always attempted whatever the budget, +/// since it is the one a crash loop is about to make unreachable. Newest first: +/// the latest crash is the one most likely to be on the build now running, and +/// what is left is not lost, since the worker's unbounded pass recovers the rest. +/// +/// Fills `attempted` with the generations it tried, for the worker's pass to skip. +fn run_launch_recovery( + shared: &Shared, + data_dir: &Path, + current_generation: u64, + budget: Duration, + max_sessions: usize, + attempted: &mut BTreeSet, +) { + let started = Instant::now(); + for pending in + recovery::find_pending_bounded(data_dir, current_generation, LAUNCH_SCAN_CAP, max_sessions) + { + if !attempted.is_empty() && started.elapsed() >= budget { + break; + } + attempted.insert(pending.generation); + recover_claimed(shared, data_dir, &pending); + } +} + /// Build, filter, assemble and enqueue one pending session's crash report, /// discarding its on-disk evidence only once it is safely queued. fn recover_one(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSession) { diff --git a/crates/bugsee-core/tests/launch_recovery.rs b/crates/bugsee-core/tests/launch_recovery.rs new file mode 100644 index 0000000..7d25c5b --- /dev/null +++ b/crates/bugsee-core/tests/launch_recovery.rs @@ -0,0 +1,211 @@ +// +// launch_recovery.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! Early-crash recovery finishes on the launch path (issue #10): by the time +//! `launch` returns, the previous run's crash is a durable queue entry, so a host +//! that crashes again right away (a crash loop) cannot outrun its report. +//! +//! The crash loop is real: this test binary re-executes itself as children that +//! launch the SDK and `abort()` immediately. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bugsee_core::transport::{Transport, TransportError}; +use bugsee_core::{queue, MockTransport, Recorder, RecorderConfig}; + +const ROLE: &str = "BUGSEE_LR_ROLE"; +const DIR: &str = "BUGSEE_LR_DIR"; +const OUT: &str = "BUGSEE_LR_OUT"; + +/// A transport that never delivers anything, so the queue keeps its entries. +struct Offline; +impl Transport for Offline { + fn register_session(&self, _: &str, _: &[u8]) -> Result { + Err(TransportError::Transient("offline".into())) + } + fn create_issue(&self, _: &str, _: Option<&str>, _: &[u8]) -> Result { + Err(TransportError::Transient("offline".into())) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), TransportError> { + Err(TransportError::Transient("offline".into())) + } +} + +/// Child entry point. A no-op in a normal test run. +#[test] +fn child_entry() { + if std::env::var(ROLE).as_deref() != Ok("crash_after_launch") { + return; + } + let dir = PathBuf::from(std::env::var(DIR).unwrap()); + let out = PathBuf::from(std::env::var(OUT).unwrap()); + let _rec = Recorder::launch(RecorderConfig::new(&dir, "tok"), Arc::new(Offline)).unwrap(); + // The instant `launch` returns: record what is durably queued, then die the + // way a crash does, with no destructor and no flush. + std::fs::write(out, queue::list_pending(&dir).len().to_string()).unwrap(); + std::process::abort(); +} + +struct TempDir(PathBuf); +impl TempDir { + fn new() -> Self { + let p = + std::env::temp_dir().join(format!("bugsee-lr-{}", bugsee_core::util::random_hex(8))); + std::fs::create_dir_all(&p).unwrap(); + TempDir(p) + } +} +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn seed_crashed(dir: &Path, generations: std::ops::RangeInclusive) { + let sessions = dir.join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + for g in generations.clone() { + // "0" is the portable "owner gone" sentinel. + std::fs::write(sessions.join(format!("{g}.alive")), "0").unwrap(); + std::fs::create_dir_all(dir.join("parts").join(g.to_string())).unwrap(); + } + std::fs::write(dir.join("gen"), generations.end().to_string()).unwrap(); +} + +#[test] +fn the_previous_crash_is_queued_before_launch_returns() { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + seed_crashed(&dir, 1..=1); + + let rec = Recorder::launch(RecorderConfig::new(&dir, "tok"), Arc::new(Offline)).unwrap(); + // No flush, no waiting: it is on disk the moment launch returns. + assert_eq!(queue::list_pending(&dir).len(), 1); + drop(rec); +} + +/// Several pending crashes, a time cap, and newest-first: the hook observes which +/// thread recovers each. With a budget shorter than one session only the newest +/// is recovered on the launch path; the rest follow on the worker. +#[test] +fn the_launch_path_is_capped_by_time_and_the_rest_follows_in_the_background() { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + seed_crashed(&dir, 1..=5); + + let threads: Arc>> = Arc::default(); + let seen = Arc::clone(&threads); + let mut config = RecorderConfig::new(&dir, "tok"); + config.launch_recovery_budget = Duration::from_millis(150); + config.before_send = Some(Box::new(move |_meta| { + let t = std::thread::current(); + seen.lock() + .unwrap() + .push(t.name().unwrap_or("launch").to_string()); + std::thread::sleep(Duration::from_millis(100)); + true + })); + let transport = Arc::new(MockTransport::default()); + let me = std::thread::current().name().map(String::from); + let rec = Recorder::launch(config, transport.clone()).unwrap(); + + let after_launch = threads.lock().unwrap().clone(); + // 100 ms per session against a 150 ms budget: two start, a third does not. + let on_launch_path = after_launch + .iter() + .filter(|n| Some(n.as_str()) != Some("bugsee-capture")) + .count(); + assert_eq!( + on_launch_path, 2, + "hook threads: {after_launch:?}, test thread {me:?}" + ); + + assert!(rec.flush(Duration::from_secs(10))); + assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 5); + let all = threads.lock().unwrap().clone(); + assert_eq!( + all.len(), + 5, + "each crash was processed exactly once: {all:?}" + ); +} + +#[test] +fn a_zero_budget_leaves_recovery_to_the_worker() { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + seed_crashed(&dir, 1..=2); + + let threads: Arc>> = Arc::default(); + let seen = Arc::clone(&threads); + let mut config = RecorderConfig::new(&dir, "tok"); + config.launch_recovery_budget = Duration::ZERO; + config.before_send = Some(Box::new(move |_meta| { + let t = std::thread::current(); + seen.lock() + .unwrap() + .push(t.name().unwrap_or("launch").to_string()); + true + })); + let transport = Arc::new(MockTransport::default()); + let rec = Recorder::launch(config, transport.clone()).unwrap(); + assert!(rec.flush(Duration::from_secs(10))); + assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 2); + assert!(threads + .lock() + .unwrap() + .iter() + .all(|n| n == "bugsee-capture")); +} + +/// The product scenario: the host crashes right after every launch. Each launch +/// must already hold the previous crash in the queue, and a later healthy run +/// delivers every one of them exactly once. +#[test] +fn a_crash_loop_loses_no_crash_and_reports_each_exactly_once() { + const CRASHES: usize = 5; + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + std::fs::create_dir_all(&dir).unwrap(); + + for i in 0..CRASHES { + let out = tmp.0.join(format!("queued-{i}")); + let status = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "child_entry", "--nocapture"]) + .env(ROLE, "crash_after_launch") + .env(DIR, &dir) + .env(OUT, &out) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .unwrap(); + assert!(!status.success(), "child {i} should have crashed"); + let queued: usize = std::fs::read_to_string(&out) + .unwrap_or_else(|_| panic!("child {i} never got past launch")) + .parse() + .unwrap(); + // Child i found the crashes of children 0..i already queued, with + // the previous child's among them: none waited for an async worker. + assert_eq!(queued, i, "child {i} saw {queued} queued crashes"); + } + + // A healthy run: everything is delivered, once. + let transport = Arc::new(MockTransport::default()); + let rec = Recorder::launch(RecorderConfig::new(&dir, "tok"), transport.clone()).unwrap(); + assert!(rec.flush(Duration::from_secs(20))); + assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), CRASHES); + assert!(queue::list_pending(&dir).is_empty()); + drop(rec); + + // And not again. + let again = Arc::new(MockTransport::default()); + let rec = Recorder::launch(RecorderConfig::new(&dir, "tok"), again.clone()).unwrap(); + assert!(rec.flush(Duration::from_secs(5))); + assert!(again.uploaded_bundles.lock().unwrap().is_empty()); +} diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 985b567..1ecf9aa 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -140,6 +140,7 @@ impl Bugsee { config.on_report_dropped = options.on_report_dropped; config.before_breadcrumb = options.before_breadcrumb; config.sample_rate = options.sample_rate; + config.launch_recovery_budget = options.launch_recovery_budget; config.app_identity = bugsee_core::model::environment::AppIdentity { package_id: options.app_package_id.clone(), version: options.app_version.clone(), diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index a77619c..998569c 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -39,6 +39,7 @@ pub struct LaunchOptions { pub(crate) before_send: Option, pub(crate) before_breadcrumb: Option, pub(crate) sample_rate: f64, + pub(crate) launch_recovery_budget: Duration, } impl LaunchOptions { @@ -63,6 +64,7 @@ impl LaunchOptions { before_send: None, before_breadcrumb: None, sample_rate: 1.0, + launch_recovery_budget: bugsee_core::runtime::DEFAULT_LAUNCH_RECOVERY_BUDGET, } } @@ -212,6 +214,23 @@ impl LaunchOptions { self } + /// Time `launch` may spend queueing the previous run's crash reports before + /// it returns. Default 500 ms. + /// + /// A host that crashes again right after launch (a crash loop under a + /// supervisor) still gets the previous crash reported, because it is made + /// durable on this path; only the upload is asynchronous. The newest crashed + /// session is always queued, and at most 8 are queued here, newest first; + /// the budget stops further ones from starting once it has elapsed, and + /// whatever is left is recovered on the background worker. It is checked + /// between sessions, so one very large session can exceed it (the cost of a + /// session grows with its capture window, `max_window`/`max_bytes`). `Duration::ZERO` skips the launch-path + /// pass and recovers everything in the background. + pub fn launch_recovery_budget(mut self, budget: Duration) -> Self { + self.launch_recovery_budget = budget; + self + } + pub fn endpoint(mut self, endpoint: impl Into) -> Self { self.endpoint = Some(endpoint.into()); self From 0b78a1000d30068b7c195aebee31072937dbda93 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:05:35 +0500 Subject: [PATCH 18/52] fix: drain every trusted unwritable default tree, not just one; stale unlinkable claims proceed (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - recovery_dirs: all rejected existing candidates (incl. the private temp tree) are drained/recovered even when a later default wins; core takes extra_data_dirs - a stale claim that cannot be unlinked is Unavailable, not Taken - seed the in-memory blacklist from the on-disk one πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/claim.rs | 28 ++++++- crates/bugsee-core/src/queue.rs | 5 ++ crates/bugsee-core/src/runtime.rs | 78 +++++++++---------- crates/bugsee-core/tests/storage_failures.rs | 43 +++++++++++ crates/bugsee/src/api.rs | 5 +- crates/bugsee/src/options.rs | 80 ++++++++++++++++---- 7 files changed, 179 insertions(+), 62 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 4d06151..6b3cdbe 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -187,7 +187,7 @@ Global base scope behind `RwLock`; `with_scope` pushes a thread-local overlay. * |---|---| | Default candidate unusable, a later default works | falls through; one stderr line; `StorageStatus::skipped` lists why | | Requested directory unusable, or no default usable, or `Session::begin`/`PartStore::new` fails (also in a fork child) | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from an in-memory backlog capped at 16 (an entry counts until its delivery settles; overflow drops the newest with `DropReason::StorageUnavailable`); a `TooManySimilar` verdict is remembered in memory; a clean stop (guard drop) gives the backlog one last bounded (1.5 s) attempt, but a report still unsent when the process dies is lost; a fork child starts with an empty backlog (the parent's is the parent's to send) | -| Directory exists but cannot take NEW files (full, read-only remount) | memory mode for new reports, but the directory is still **drained and recovered from**: its queued bundles are delivered and its crashed sessions recovered (claims are best-effort there; a crash that cannot be re-queued on disk is delivered from memory). Bundles delivered but undeletable are remembered and not resent in this process; a later launch with a writable disk may resend them and the backend deduplicates. Only a missing or non-directory path is skipped | +| Directory exists but cannot take NEW files (full, read-only remount) | memory mode for new reports, but the directory is still **drained and recovered from**: its queued bundles are delivered and its crashed sessions recovered (claims are best-effort there; a crash that cannot be re-queued on disk is delivered from memory). Bundles delivered but undeletable are remembered and not resent in this process; a later launch with a writable disk may resend them and the backend deduplicates. Every such rejected candidate is drained, including a full default that lost to a later writable one and the private temp tree (which must be a real directory of ours); a stale claim that cannot be unlinked counts as "no claim possible", not "held elsewhere"; the on-disk `.blacklist` seeds the in-memory one. Only a missing or non-directory path is skipped | | Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and dropped; the rotation tick rewrites a missing `sessions/.alive` (a wiped-and-recreated tree would otherwise make the next launch's orphan sweep drop this session's crash); a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out; `StorageStatus::degraded` turns true and `crash_recovery_available()` false | | Disk full / permission denied at enqueue, snapshot, marker | same as above; the `reports/` snapshot dir is reclaimed by `SnapshotHandle::drop` | | Crash handler cannot write its marker | the raw `open`/`write` return values are ignored; the process still dies with the original signal (watchdog bounds the handler) | diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index 12ab003..341f1e0 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -86,7 +86,14 @@ fn claim_state(path: &Path) -> ClaimState { // Take over a claim whose owner is gone, then retry the create. // Losing the race to another taker just means `create_new` fails // again and we report the claim as held. - let _ = std::fs::remove_file(path); + match std::fs::remove_file(path) { + Ok(()) => {} + // Someone else took it over first: retry the create. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + // A read-only or full volume will not let us clear it, so no + // claim can ever be made here: not "held by someone else". + Err(_) => return ClaimState::Unavailable, + } } // Directory missing / unwritable: nothing can be claimed, and the // caller's own I/O will fail more usefully than we can here. @@ -150,6 +157,25 @@ mod tests { assert!(try_claim(&path).is_some()); } + #[cfg(unix)] + #[test] + fn a_stale_claim_that_cannot_be_removed_is_unavailable_not_taken() { + use std::os::unix::fs::PermissionsExt; + let dir = tmp(); + let path = dir.join("x.claim"); + std::fs::write(&path, "4294967290").unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o555)).unwrap(); + if std::fs::remove_file(dir.join("nonexistent-probe")).is_ok() + || std::fs::write(dir.join("probe"), b"x").is_ok() + { + let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)); + return; // permission bits not enforced (root) + } + // Proceed unclaimed instead of skipping the work as "held elsewhere". + assert!(matches!(try_claim_lenient(&path), Some(None))); + let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)); + } + #[test] fn a_freshly_created_unowned_claim_is_not_stolen() { let dir = tmp(); diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index ea6c033..2a1cbd2 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -186,6 +186,11 @@ fn blacklist_path(data_dir: &Path) -> PathBuf { queue_dir(data_dir).join(".blacklist") } +/// The locally blacklisted signatures in `data_dir` (empty if none or unreadable). +pub fn blacklisted_signatures(data_dir: &Path) -> BTreeSet { + load_blacklist(data_dir) +} + fn load_blacklist(data_dir: &Path) -> BTreeSet { std::fs::read_to_string(blacklist_path(data_dir)) .map(|s| { diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 57a9864..49fe3dc 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -179,6 +179,10 @@ pub struct RecorderConfig { /// crashed sessions and deliver its queue. For a host layer that moved its /// default location and must not strand reports left at the old one. pub legacy_data_dir: Option, + /// More existing directories to recover from and drain once, like + /// `legacy_data_dir`: trusted trees that could not take new files (a full or + /// read-only default) even though a later default was writable. + pub extra_data_dirs: Vec, } impl RecorderConfig { @@ -201,6 +205,7 @@ impl RecorderConfig { max_queued_entries: DEFAULT_MAX_QUEUED, memory_only: None, legacy_data_dir: None, + extra_data_dirs: Vec::new(), } } } @@ -414,7 +419,11 @@ impl Recorder { config.upload_backoff_base, config.sampler, true, - config.legacy_data_dir, + config + .legacy_data_dir + .into_iter() + .chain(config.extra_data_dirs) + .collect(), )?; Ok(Recorder { @@ -504,7 +513,7 @@ impl Recorder { self.backoff_base, sampler, false, - None, + Vec::new(), )?; // The parent's session is still the parent's: leave its marker alone. @@ -850,16 +859,16 @@ fn spawn_pipeline( backoff_base: Duration, sampler: Option>, fresh_launch: bool, - legacy_data_dir: Option, + legacy_data_dirs: Vec, ) -> std::io::Result { // Uploader thread: drains the durable queue with retry. let (upload_tx, upload_rx) = channel(); let uploader_shared = Arc::clone(shared); let uploader_data_dir = drain_dir.map(Path::to_path_buf); - let uploader_legacy = legacy_data_dir.clone(); + let uploader_legacy = legacy_data_dirs.clone(); // Set once the worker has finished recovering the legacy directory, so the // uploader does not retire it while recovery is still about to fill its queue. - let legacy_ready = Arc::new(AtomicBool::new(legacy_data_dir.is_none())); + let legacy_ready = Arc::new(AtomicBool::new(legacy_data_dirs.is_empty())); let uploader_ready = Arc::clone(&legacy_ready); let uploader = std::thread::Builder::new() .name("bugsee-uploader".into()) @@ -897,6 +906,14 @@ fn spawn_pipeline( if let (true, Some(dir)) = (fresh_launch, &worker_drain_dir) { queue::gc_orphans(dir); } + // With no writable disk the in-memory set is the only blacklist + // `process_report` consults, so seed it from what is on disk. + if let Some(dir) = &worker_drain_dir { + let known = queue::blacklisted_signatures(dir); + if !known.is_empty() { + worker_shared.mem_blacklist.lock().extend(known); + } + } // A fork child recovers too. A prefork master recovers only at ITS // launch, and its own generation is the lowest, so it can never // recover a worker that died later; the next worker started from @@ -907,7 +924,7 @@ fn spawn_pipeline( } // Best effort, once: sessions a previous default location left // behind. No generation of ours lives there, so none is skipped. - if let Some(dir) = &legacy_data_dir { + for dir in &legacy_data_dirs { queue::gc_orphans(dir); run_recovery(&worker_shared, dir, u64::MAX); } @@ -1422,39 +1439,27 @@ fn uploader_loop( rx: Receiver, shared: Arc, data_dir: Option, - legacy_data_dir: Option, + legacy_data_dirs: Vec, legacy_ready: Arc, backoff_base: Duration, fresh_launch: bool, ) { let poll = Duration::from_secs(5); - let mut legacy = legacy_data_dir; + let mut legacy = legacy_data_dirs; // A fresh launch retries any queued report immediately (force), ignoring // backoff scheduled by the prior session. A fork child skips this: the // parent's uploader is already working the shared queue. Guarded so a // panicking Transport impl can't kill the uploader. if fresh_launch { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_all( - &shared, - data_dir.as_deref(), - legacy.as_deref(), - backoff_base, - true, - ) + drain_all(&shared, data_dir.as_deref(), &legacy, backoff_base, true) })); } loop { match rx.recv_timeout(poll) { Ok(UploadMsg::Wake) => { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_all( - &shared, - data_dir.as_deref(), - legacy.as_deref(), - backoff_base, - false, - ) + drain_all(&shared, data_dir.as_deref(), &legacy, backoff_base, false) })); } Ok(UploadMsg::Drain { deadline, ack }) => { @@ -1462,7 +1467,7 @@ fn uploader_loop( let dirs: Vec<&Path> = data_dir .as_deref() .into_iter() - .chain(legacy.as_deref()) + .chain(legacy.iter().map(PathBuf::as_path)) .collect(); drain_until_empty(&shared, &dirs, backoff_base, deadline) })) @@ -1487,13 +1492,7 @@ fn uploader_loop( } Err(RecvTimeoutError::Timeout) => { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - drain_all( - &shared, - data_dir.as_deref(), - legacy.as_deref(), - backoff_base, - false, - ) + drain_all(&shared, data_dir.as_deref(), &legacy, backoff_base, false) })); } Err(RecvTimeoutError::Disconnected) => break, @@ -1515,31 +1514,26 @@ fn pending_reports(shared: &Shared, dir: &Path) -> Vec { /// Stop polling the legacy directory once recovery has run (it runs once per /// launch, so nothing new can appear) and its queue is empty. -fn retire_legacy(shared: &Shared, legacy: &mut Option, ready: &AtomicBool) { +fn retire_legacy(shared: &Shared, legacy: &mut Vec, ready: &AtomicBool) { if !ready.load(Ordering::SeqCst) { return; } - if legacy - .as_deref() - .is_some_and(|dir| pending_reports(shared, dir).is_empty()) - { - *legacy = None; - } + legacy.retain(|dir| !pending_reports(shared, dir).is_empty()); } /// [`drain`] the data directory, then the legacy one a launch was asked to adopt. fn drain_all( shared: &Shared, data_dir: Option<&Path>, - legacy: Option<&Path>, + legacy: &[PathBuf], backoff_base: Duration, force: bool, ) -> bool { - let attempted = drain(shared, data_dir, backoff_base, force); - match legacy { - Some(dir) => drain_disk(shared, dir, backoff_base, force) | attempted, - None => attempted, + let mut attempted = drain(shared, data_dir, backoff_base, force); + for dir in legacy { + attempted |= drain_disk(shared, dir, backoff_base, force); } + attempted } /// Attempt delivery of every due queued report once. On the flush path `force` diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index bf04627..616fb67 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -656,3 +656,46 @@ fn a_directory_wiped_and_recreated_mid_run_gets_its_liveness_marker_back() { assert!(alive(), "a crash after the wipe must still be recoverable"); assert!(recorder.is_degraded(), "the lost window is a degradation"); } + +/// A later default can win while an earlier, full one still holds leftovers. +#[cfg(unix)] +#[test] +fn extra_directories_are_recovered_alongside_a_working_data_dir() { + let dir = TempDir::new(); + let data = dir.path.join("data"); + let full = dir.path.join("full-default"); + std::fs::create_dir_all(&full).unwrap(); + seed_leftovers(&full); + + let transport = Arc::new(MockTransport::default()); + let mut config = RecorderConfig::new(&data, "T"); + config.extra_data_dirs = vec![full.clone()]; + let recorder = Recorder::launch(config, transport.clone()).expect("launch"); + assert_eq!(recorder.storage_mode(), StorageMode::Disk); + assert_eq!(wait_for_bundles(&transport, 2), 2); + assert!(bugsee_core::queue::list_pending(&full).is_empty()); +} + +#[test] +fn signatures_blacklisted_on_disk_are_honoured_in_memory_mode() { + let dir = TempDir::new(); + let data = dir.path.join("data"); + std::fs::create_dir_all(data.join("queue")).unwrap(); + std::fs::write(data.join("queue").join(".blacklist"), "sig-known\n").unwrap(); + + let transport = Arc::new(MockTransport::default()); + let mut config = RecorderConfig::new(&data, "T"); + config.memory_only = Some("write probe failed".into()); + let recorder = Recorder::launch(config, transport.clone()).expect("launch"); + + let mut known = manual_upload_meta(); + known.signatures = vec!["sig-known".into()]; + recorder.report(known, None); + recorder.report(manual_upload_meta(), None); + assert!(recorder.flush(Duration::from_secs(10))); + assert_eq!( + transport.created_requests.lock().unwrap().len(), + 1, + "only the report without the blacklisted signature is sent" + ); +} diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 985b567..6c1bfa2 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -88,7 +88,7 @@ impl Bugsee { source, skipped, explicit_rejected, - recovery_dir, + recovery_dirs, legacy_dir, } = options.resolve_data_dir(); let memory_only = dir.is_none().then(|| { @@ -99,7 +99,7 @@ impl Bugsee { }); // In memory mode no new file is written there, but an existing directory // is still drained and recovered from; with none, an empty path. - let data_dir = dir.or(recovery_dir).unwrap_or_default(); + let data_dir = dir.unwrap_or_default(); if memory_only.is_none() && !skipped.is_empty() { static WARNED: AtomicBool = AtomicBool::new(false); bugsee_core::util::warn_once( @@ -134,6 +134,7 @@ impl Bugsee { let mut config = RecorderConfig::new(data_dir, options.app_token.clone()); config.memory_only = memory_only; config.legacy_data_dir = legacy_dir; + config.extra_data_dirs = recovery_dirs; config.caps = options.caps(); config.rotate_interval = options.rotate_interval; config.before_send = options.before_send; diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index a77619c..6688638 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -292,10 +292,11 @@ pub(crate) struct DataDirResolution { pub skipped: Vec, /// Why an explicit / environment directory was rejected (it is final). pub explicit_rejected: Option, - /// A rejected candidate that nevertheless EXISTS as a directory (full or - /// read-only). It may hold reports and crashes a previous run persisted, so - /// it is still drained and recovered from; no new files are written to it. - pub recovery_dir: Option, + /// Rejected candidates that nevertheless EXIST as directories we may trust + /// (full or read-only). They may hold reports and crashes a previous run + /// persisted, so each is still drained and recovered from, whether or not a + /// later candidate won; no new files are written to them. + pub recovery_dirs: Vec, /// The location earlier SDK versions used by default /// (`/bugsee/`), if it exists, is trustworthy and is not `dir`. /// Its leftovers are recovered once at launch. @@ -326,12 +327,12 @@ pub(crate) fn resolve_data_dir( source, skipped: Vec::new(), explicit_rejected: None, - recovery_dir: None, + recovery_dirs: Vec::new(), legacy_dir: None, }, Err(e) => DataDirResolution { explicit_rejected: Some(format!("{}: {e}", dir.display())), - recovery_dir: dir.is_dir().then_some(dir), + recovery_dirs: if dir.is_dir() { vec![dir] } else { Vec::new() }, dir: None, source: DataDirSource::None, skipped: Vec::new(), @@ -358,7 +359,7 @@ pub(crate) fn resolve_data_dir( let mut skipped = Vec::new(); let mut found = None; - let mut recovery_dir = None; + let mut recovery_dirs = Vec::new(); for (source, dir, private) in candidates { match prepare_dir(&dir, private) { Ok(()) => { @@ -367,10 +368,16 @@ pub(crate) fn resolve_data_dir( } Err(e) => { skipped.push(format!("{}: {e}", dir.display())); - // Only the user's own default location: the shared temp area is - // exactly where a planted directory would be refused. - if !private && recovery_dir.is_none() && dir.is_dir() { - recovery_dir = Some(dir); + // The shared temp area is where a planted directory would be + // refused, so there the tree must be a real directory of ours + // (and so must its parent) to be trusted for recovery. + let trusted = if private { + trusted_existing_dir(&dir) && dir.parent().is_some_and(trusted_existing_dir) + } else { + dir.is_dir() + }; + if trusted { + recovery_dirs.push(dir); } } } @@ -387,7 +394,7 @@ pub(crate) fn resolve_data_dir( source, skipped, explicit_rejected: None, - recovery_dir: None, + recovery_dirs, legacy_dir, }, None => DataDirResolution { @@ -395,7 +402,7 @@ pub(crate) fn resolve_data_dir( source: DataDirSource::None, skipped, explicit_rejected: None, - recovery_dir, + recovery_dirs, legacy_dir, }, } @@ -842,8 +849,8 @@ mod tests { let r = resolve_data_dir(Some(&ro), "tok", &env_of(&[]), &|| s.join("tmp")); assert!(r.dir.is_none() && r.explicit_rejected.is_some()); assert_eq!( - r.recovery_dir.as_deref(), - Some(ro.as_path()), + r.recovery_dirs, + vec![ro.clone()], "its queue and crashed sessions must still be drained" ); @@ -851,7 +858,7 @@ mod tests { let r = resolve_data_dir(Some(&blocked(&s, "f5")), "tok", &env_of(&[]), &|| { s.join("tmp") }); - assert!(r.recovery_dir.is_none()); + assert!(r.recovery_dirs.is_empty()); let _ = std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o755)); let _ = std::fs::remove_dir_all(&s); } @@ -871,6 +878,47 @@ mod tests { let _ = std::fs::remove_dir_all(&s); } + #[cfg(all(unix, not(target_vendor = "apple")))] + #[test] + fn a_full_default_that_loses_to_a_later_one_is_still_recovered_from() { + use std::os::unix::fs::PermissionsExt; + let s = scratch(); + let home = s.join("home"); + let default = home.join("bugsee").join("tok"); + std::fs::create_dir_all(&default).unwrap(); + std::fs::set_permissions(&default, std::fs::Permissions::from_mode(0o555)).unwrap(); + if std::fs::write(default.join("p"), b"x").is_ok() { + let _ = std::fs::remove_dir_all(&s); + return; // permission bits not enforced (root) + } + // The platform default is full/read-only; the temp fallback is writable. + let r = resolve_data_dir(None, "tok", &env_of(&home_env(&home)), &|| s.join("tmp")); + assert_eq!(r.source, DataDirSource::TempFallback); + assert_eq!(r.recovery_dirs, vec![default.clone()]); + let _ = std::fs::set_permissions(&default, std::fs::Permissions::from_mode(0o755)); + let _ = std::fs::remove_dir_all(&s); + } + + #[cfg(unix)] + #[test] + fn a_full_private_temp_tree_is_recovered_from_when_nothing_is_writable() { + use std::os::unix::fs::PermissionsExt; + let s = scratch(); + let tmp = s.join("tmp"); + let tree = tmp.join(private_temp_name()).join("tok"); + std::fs::create_dir_all(&tree).unwrap(); + std::fs::set_permissions(&tree, std::fs::Permissions::from_mode(0o500)).unwrap(); + if std::fs::write(tree.join("p"), b"x").is_ok() { + let _ = std::fs::remove_dir_all(&s); + return; // permission bits not enforced (root) + } + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); + assert!(r.dir.is_none()); + assert_eq!(r.recovery_dirs, vec![tree.clone()]); + let _ = std::fs::set_permissions(&tree, std::fs::Permissions::from_mode(0o755)); + let _ = std::fs::remove_dir_all(&s); + } + #[test] fn an_explicit_directory_never_asks_for_the_temp_location() { let s = scratch(); From 3bf7ac78054306269921c875c5c612a7f0ebc971 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:08:41 +0500 Subject: [PATCH 19/52] fix(scrub,runtime): second re-review fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stem exclusions for cancellation_token, www-authenticate/authentication, token_endpoint/url/uri/expiry/ttl; the colon-prose leniency applies only to bare 'token', so password/passwd/secret/api_key values are always redacted; the rate limiter now runs before the on-disk blacklist read so a denied storm is lock + hash only (test counts disk reads). Refs #11 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/queue.rs | 10 +++ crates/bugsee-core/src/runtime.rs | 19 +++-- crates/bugsee-core/src/scrub.rs | 83 +++++++++++++++---- crates/bugsee-core/tests/ratelimit_no_disk.rs | 51 ++++++++++++ crates/bugsee-reqwest/src/lib.rs | 17 ++++ 5 files changed, 155 insertions(+), 25 deletions(-) create mode 100644 crates/bugsee-core/tests/ratelimit_no_disk.rs diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index ea6c033..4749e38 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -186,7 +186,17 @@ fn blacklist_path(data_dir: &Path) -> PathBuf { queue_dir(data_dir).join(".blacklist") } +static BLACKLIST_READS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// How many times the on-disk blacklist has been read since process start. +/// Exists so tests can prove a rate-limited storm performs no disk reads. +#[doc(hidden)] +pub fn blacklist_read_count() -> usize { + BLACKLIST_READS.load(std::sync::atomic::Ordering::Relaxed) +} + fn load_blacklist(data_dir: &Path) -> BTreeSet { + BLACKLIST_READS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); std::fs::read_to_string(blacklist_path(data_dir)) .map(|s| { s.lines() diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 5e50d8c..63456fe 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1412,18 +1412,23 @@ fn process_report( return Ok(()); } - // Suppress reports the server blacklisted (12004 TooManySimilar), from the - // in-memory set (the only one with no usable disk) or the on-disk one. - if mem_blacklisted(shared, &meta.signatures) - || data_dir.is_some_and(|dir| queue::any_blacklisted(dir, &meta.signatures)) - { + // Suppress reports the server blacklisted (12004 TooManySimilar). Order + // matters for a storm: the in-memory set is a lock + hash, so it goes first; + // the rate limiter comes next (also lock + hash), so a flood of denied + // reports never touches the disk; only the reports the limiter ADMITS pay + // for the on-disk `.blacklist` read (which, unlike the in-memory set, also + // sees blacklistings written by sibling processes sharing the directory). + if mem_blacklisted(shared, &meta.signatures) { return Ok(()); } - - // Rate limit before any disk work (snapshot / ZIP / durable queue). let Some(carried) = admit(shared, &mut meta) else { return Ok(()); }; + if data_dir.is_some_and(|dir| queue::any_blacklisted(dir, &meta.signatures)) { + // Dropped after all: hand the suppressed-count summary back. + shared.limiter.restore_suppressed(carried); + return Ok(()); + } let window_start = window_end.saturating_sub(caps.max_window_ms); let window = TimeWindow { diff --git a/crates/bugsee-core/src/scrub.rs b/crates/bugsee-core/src/scrub.rs index e5e744a..a48fd7f 100644 --- a/crates/bugsee-core/src/scrub.rs +++ b/crates/bugsee-core/src/scrub.rs @@ -315,19 +315,37 @@ fn stem_hit(n: &[u8], stem: &[u8]) -> bool { let at = from + p; let (before, rest) = (&n[..at], &n[at + stem.len()..]); let excluded = match stem { - b"auth" => rest.starts_with(b"or") && !rest.starts_with(b"oriz"), - b"token" => [ - &b"iz"[..], - b"type", - b"count", - b"len", - b"limit", - b"usage", - b"used", - b"budget", - ] - .iter() - .any(|x| rest.starts_with(x)), + // `author`/`authority` (but not `authorization`), and + // `www-authenticate`/`authentication`. + b"auth" => { + (rest.starts_with(b"or") && !rest.starts_with(b"oriz")) + || rest.starts_with(b"enticat") + } + // `tokenizer`, `token_type`/`_count`/`_ttl`/`_endpoint`/`_url`, and + // `cancellation_token` (a cooperative-cancel handle, not a secret). + b"token" => { + [ + &b"iz"[..], + b"type", + b"count", + b"len", + b"limit", + b"usage", + b"used", + b"budget", + b"endpoint", + b"url", + b"uri", + b"expir", + b"ttl", + b"scope", + b"kind", + ] + .iter() + .any(|x| rest.starts_with(x)) + || before.ends_with(b"cancellation") + || before.ends_with(b"cancel") + } b"secret" => rest.starts_with(b"ary"), b"session" => [ &b"start"[..], @@ -1185,9 +1203,16 @@ impl Scrubber { if !(sep == b'=' || quoted_key) || !secretish { continue; } - } else if class == KeyClass::Strong && sep == b':' && !quoted_key && quote.is_none() { - // Prose, not a credential: "invalid token: expected ident", - // "password: must be at least 8". A bare `token: abc123` still is. + } else if class == KeyClass::Strong + && sep == b':' + && !quoted_key + && quote.is_none() + && with_normalized(&s[j..key_end], |n| n == b"token") + { + // Compiler-error wording, not a credential: "invalid token: + // expected ident". ONLY the bare word `token` gets this leniency: + // `password: secret` / `invalid password: letmein` are redacted + // however plain the value looks. A real `token: abc123` still is. let loose = ve - vs >= 6 && !head.iter().all(u8::is_ascii_lowercase); if !loose { continue; @@ -1962,13 +1987,22 @@ mod tests { for t in [ "invalid token: expected ident", "unexpected token: `}`", - "password: must be at least 8 characters", "INFO my_app::token: refreshed", - "secret: required", + "token: expected one of `,` or `)`", ] { assert_eq!(s(t), t, "{t}"); } assert_eq!(s("password: hunter2"), "password: [redacted]"); + // Only bare `token` gets the prose leniency. + assert_eq!(s("password: secret"), "password: [redacted]"); + assert_eq!( + s("invalid password: letmein"), + "invalid password: [redacted]" + ); + assert_eq!(s("passwd: letmein"), "passwd: [redacted]"); + assert_eq!(s("secret: required"), "secret: [redacted]"); + assert_eq!(s("api_key: abcdefgh"), "api_key: [redacted]"); + assert_eq!(s("access_token: abcdefgh"), "access_token: [redacted]"); assert_eq!(s("token: abc123def456"), "token: [redacted]"); assert_eq!( s(r#"{"password": "short"}"#), @@ -2009,6 +2043,19 @@ mod tests { "session_timeout", "keyboard", "monkey", + "cancellation_token", + "CancellationToken", + "www-authenticate", + "WWW-Authenticate", + "token_endpoint", + "token_url", + "token_uri", + "token_expiry", + "token_expires_at", + "token_ttl", + "refresh_token_count", + "max_token_count", + "cancel_token", ] { assert!(!is_sensitive_key(k), "{k}"); } diff --git a/crates/bugsee-core/tests/ratelimit_no_disk.rs b/crates/bugsee-core/tests/ratelimit_no_disk.rs new file mode 100644 index 0000000..9df2262 --- /dev/null +++ b/crates/bugsee-core/tests/ratelimit_no_disk.rs @@ -0,0 +1,51 @@ +// +// ratelimit_no_disk.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! A rate-limited storm must be cheap: reports the limiter denies never read the +//! on-disk blacklist. Own test binary, because the read counter is process-wide. + +use std::sync::Arc; +use std::time::Duration; + +use bugsee_core::model::enums::IssueType; +use bugsee_core::queue::blacklist_read_count; +use bugsee_core::ratelimit::{Rate, RateLimitConfig}; +use bugsee_core::reporting::manual_upload_meta; +use bugsee_core::{MockTransport, Recorder, RecorderConfig}; + +#[test] +fn denied_storm_reports_never_touch_the_disk_blacklist() { + let mut dir = std::env::temp_dir(); + dir.push(format!( + "bugsee-nodisk-{}", + bugsee_core::util::random_hex(8) + )); + std::fs::create_dir_all(&dir).unwrap(); + + let transport = Arc::new(MockTransport::default()); + let mut cfg = RecorderConfig::new(&dir, "T"); + cfg.rate_limit = RateLimitConfig::default() + .per_signature(Rate::new(2, Duration::from_secs(3600))) + .global(Rate::new(1000, Duration::from_secs(1))); + let r = Recorder::launch(cfg, transport.clone()).unwrap(); + assert!(r.flush(Duration::from_secs(10))); // launch-time recovery is done + + let before = blacklist_read_count(); + for _ in 0..300 { + let mut m = manual_upload_meta(); + m.issue_type = IssueType::Error; + m.signatures = vec!["hot".into()]; + r.report(m, None); + } + assert!(r.flush(Duration::from_secs(30))); + let reads = blacklist_read_count() - before; + + assert_eq!(r.suppressed_reports(), 298); + // Only the two admitted reports may pay for the disk read. + assert!(reads <= 2, "{reads} blacklist reads for a 300-report storm"); + drop(r); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/bugsee-reqwest/src/lib.rs b/crates/bugsee-reqwest/src/lib.rs index 50cdb58..a353b11 100644 --- a/crates/bugsee-reqwest/src/lib.rs +++ b/crates/bugsee-reqwest/src/lib.rs @@ -407,6 +407,23 @@ fn redact_json(value: &mut Value) { mod tests { use super::*; + #[test] + fn benign_token_like_names_are_not_redacted() { + let url = reqwest::Url::parse( + "https://api.example.com/v1?token_endpoint=a&cancellation_token=b&token_ttl=9&token=zzz", + ) + .unwrap(); + let s = sanitize_url(&url); + assert!( + s.contains("token_endpoint=a") && s.contains("cancellation_token=b"), + "{s}" + ); + assert!(s.contains("token_ttl=9") && !s.contains("zzz"), "{s}"); + assert!(!is_sensitive_header("WWW-Authenticate")); + assert!(!is_sensitive_header("token_url")); + assert!(is_sensitive_header("Authorization")); + } + #[test] fn redacts_sensitive_query_params() { let url = reqwest::Url::parse("https://api.example.com/v1?token=secret&page=2").unwrap(); From e73983974494f946f2e426116e72892911d531d1 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:08:45 +0500 Subject: [PATCH 20/52] fix: PR #31 review and CI fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Android: errno via libc::__errno (and the BSD variants), not __errno_location. - Gate the flush_at_exit statics and import on unix (Windows clippy). - SA_RESETHAND: the handler decides "terminate" before emulating the reset and ships it in the pipe byte; the watcher no longer re-reads the slot. New test with a one-shot host handler. - Windows Ctrl+C/Break now run shutdown like close/logoff/shutdown; document that a host which swallows the event registers its handler after ours. - DESIGN section 6 no longer says guard Drop flushes. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 6 +-- crates/bugsee/src/api.rs | 16 ++++++-- crates/bugsee/src/termination.rs | 64 ++++++++++++++++++-------------- crates/bugsee/tests/shutdown.rs | 22 +++++++++-- 4 files changed, 70 insertions(+), 38 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index bc6daca..21dae98 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -85,7 +85,7 @@ A Cargo workspace. Integrations are **separate crates** re-exported by an umbrel ## 6. Public API surface -Global singleton (mobile parity); `launch` returns a guard whose `Drop` flushes. +Global singleton (mobile parity); `launch` returns a guard whose `Drop` persists and stops cleanly (see Β§8; it does not upload). ```rust // Lifecycle @@ -210,10 +210,10 @@ Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.r | `process::exit` / `exit()` | left (default) | as already written | one abnormal-exit report; queue delivered once | | same, with `LaunchOptions::flush_at_exit` (unix only; ignored on Windows, call `shutdown`) | removed by an `atexit` handler (not run in `fork` children) | persisted | as the first row | | `SIGTERM`/`SIGINT` + `Bugsee::install_termination_flush` | removed if the signal would have terminated the process | flushed, bounded | as the first row; host handler still called | -| Windows Ctrl+C/Break with the hook and no host handler | left (flush only) | flushed, bounded | one abnormal-exit report; queue delivered | +| Windows Ctrl+C/Break/close/logoff/shutdown + hook | removed (`shutdown`) | flushed, bounded | as the first row | | `SIGTERM`/`SIGINT` without the hook, `SIGKILL`, OOM, power loss | left | records appended as drained survive | one abnormal-exit report | -*Decision: `exit` without `stop` stays an abnormal exit.* From inside the process an `exit` is indistinguishable from a runtime tearing down after a fault the host swallowed, so the default errs towards reporting; hosts that exit on purpose opt in with `flush_at_exit` (or call `shutdown`). *Decision: the signal hook is opt-in and chaining.* The handler only writes the signal number to a non-blocking self-pipe and then calls the previously installed handler; a watcher thread does the flush. With a default previous disposition it runs `shutdown`, restores `SIG_DFL` and re-raises, so the exit status is unchanged; a second signal during the shutdown terminates at once. With a host handler it flushes only and the host decides; a host handler installed AFTER ours (tokio, `signal-hook`) is detected by asking the kernel what is in force, and is treated the same, so we never kill a host mid-graceful-shutdown. A signal ignored at install is left alone. A second signal while the first is served, or a dead watcher, terminates at once. The chained call keeps the host's `sa_mask`, `SA_ONSTACK`, `SA_NODEFER`, `SA_SIGINFO`, and `SA_RESETHAND` (emulated). Do not call `exit` from a host handler with `flush_at_exit` on: the exit-time `shutdown` would run in signal context. A `fork` child inherits the handler but not the watcher: it chains/terminates as if the hook were absent until `install_termination_flush` is called again there. Windows: a console control handler (Ctrl+C/Break flush only, since an earlier handler may swallow them; close/logoff/shutdown flush and stop) that returns `FALSE` to pass the event on; service-control stop is not a console event and needs a `shutdown` call from the host's stop handler. **Not compiled or run on Windows** (a macOS cross-check stops at the C dependencies of `zstd-sys`/`ring`); the Windows half was reviewed by reading only. +*Decision: `exit` without `stop` stays an abnormal exit.* From inside the process an `exit` is indistinguishable from a runtime tearing down after a fault the host swallowed, so the default errs towards reporting; hosts that exit on purpose opt in with `flush_at_exit` (or call `shutdown`). *Decision: the signal hook is opt-in and chaining.* The handler only writes the signal number to a non-blocking self-pipe and then calls the previously installed handler; a watcher thread does the flush. With a default previous disposition it runs `shutdown`, restores `SIG_DFL` and re-raises, so the exit status is unchanged; a second signal during the shutdown terminates at once. With a host handler it flushes only and the host decides; a host handler installed AFTER ours (tokio, `signal-hook`) is detected by asking the kernel what is in force, and is treated the same, so we never kill a host mid-graceful-shutdown. A signal ignored at install is left alone. A second signal while the first is served, or a dead watcher, terminates at once. The chained call keeps the host's `sa_mask`, `SA_ONSTACK`, `SA_NODEFER`, `SA_SIGINFO`, and `SA_RESETHAND` (emulated). Do not call `exit` from a host handler with `flush_at_exit` on: the exit-time `shutdown` would run in signal context. A `fork` child inherits the handler but not the watcher: it chains/terminates as if the hook were absent until `install_termination_flush` is called again there. Windows: a console control handler that runs `shutdown` for every event (a host that swallows Ctrl+C and keeps running must register its handler AFTER the SDK's, so it runs first) that returns `FALSE` to pass the event on; service-control stop is not a console event and needs a `shutdown` call from the host's stop handler. **Not compiled or run on Windows** (a macOS cross-check stops at the C dependencies of `zstd-sys`/`ring`); the Windows half was reviewed by reading only. Tests: `bugsee/tests/shutdown.rs` (a real child process per exit path, then a relaunch asserting exactly-once delivery and the abnormal-exit count). diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 15fd7b4..fce0515 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -11,7 +11,9 @@ //! and after stop. use std::io; -use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +#[cfg(unix)] +use std::sync::atomic::AtomicU32; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::Duration; use bugsee_core::errors::{self, Cause}; @@ -340,9 +342,13 @@ impl Bugsee { /// Signals covered: `SIGTERM`, `SIGINT`. A `fork` child inherits the /// handler but not its thread; it terminates by default as usual, and /// calling this again in the child re-arms it there. - /// - **Windows.** A console control handler that flushes (and, for close, - /// logoff and shutdown, which end the process regardless, also stops) - /// then returns `FALSE` so the next handler still runs. Windows gives + /// - **Windows.** A console control handler that runs [`Bugsee::shutdown`] + /// (flush, then a clean stop) for Ctrl+C, Ctrl+Break, close, logoff and + /// shutdown, then returns `FALSE` so the next handler still runs. Windows + /// cannot tell it whether that next handler will end the process, so a + /// host that handles Ctrl+C itself and keeps running must register its + /// handler **after** this call (handlers run last-registered first, so + /// its swallowing the event means ours never runs). Windows gives /// close/logoff/shutdown handlers only a few seconds: use a short /// `timeout`. A service-control stop is not a console event and is not /// covered; call [`Bugsee::shutdown`] from your stop handler. @@ -580,10 +586,12 @@ impl Bugsee { } /// Exit-time flush timeout in milliseconds, `0` when not requested. +#[cfg(unix)] static EXIT_FLUSH_MS: AtomicU64 = AtomicU64::new(0); /// The process that registered the `atexit` handler. A `fork` child inherits /// the registration, and running `shutdown` there would revive the SDK in a /// process that may be mid-`fork`-and-`exec`; the child must do its own stop. +#[cfg(unix)] static EXIT_FLUSH_PID: AtomicU32 = AtomicU32::new(0); #[cfg(unix)] diff --git a/crates/bugsee/src/termination.rs b/crates/bugsee/src/termination.rs index c3bc176..b9372bb 100644 --- a/crates/bugsee/src/termination.rs +++ b/crates/bugsee/src/termination.rs @@ -37,10 +37,10 @@ //! A console control handler. Windows runs it on a thread of its own, so it //! flushes in place, then returns `FALSE` so the next handler (the host's, or //! the default) still runs. Close, logoff and shutdown also stop the SDK, as -//! the process ends regardless. Ctrl+C and Ctrl+Break only flush: with no host -//! handler the default one ends the process without a clean stop, so the next -//! launch reports an abnormal exit (an earlier handler may instead swallow the -//! event and keep running, which a stop here would wrongly cut short). +//! the process ends regardless. Ctrl+C and Ctrl+Break do the same: Windows +//! cannot say whether a later handler will end the process, so a host that +//! swallows the event and keeps running must register its handler AFTER the +//! SDK's (it then runs first, and our handler never sees a swallowed event). use std::io; use std::time::Duration; @@ -78,6 +78,12 @@ mod imp { }; static SLOTS: [Slot; 2] = [SLOT; 2]; + /// Set in the pipe byte when the handler found "default disposition, ours in + /// force" and so wants the watcher to terminate after flushing. Decided in + /// the handler, BEFORE a host `SA_RESETHAND` handler has been emulated away, + /// so the one signal that handler was meant for is not turned into a kill. + const TERMINATE: u8 = 0x80; + /// Write end of the self-pipe; `-1` until installed. static PIPE_W: AtomicI32 = AtomicI32::new(-1); /// The process that owns the watcher thread. A `fork` child inherits the @@ -127,12 +133,22 @@ mod imp { } } - #[cfg(any(target_os = "linux", target_os = "android"))] + #[cfg(all(target_os = "linux", not(target_os = "android")))] unsafe fn errno_location() -> *mut c_int { unsafe { libc::__errno_location() } } - #[cfg(not(any(target_os = "linux", target_os = "android")))] + #[cfg(any(target_os = "android", target_os = "openbsd", target_os = "netbsd"))] + unsafe fn errno_location() -> *mut c_int { + unsafe { libc::__errno() } + } + #[cfg(not(any( + target_os = "linux", + target_os = "android", + target_os = "openbsd", + target_os = "netbsd" + )))] unsafe fn errno_location() -> *mut c_int { + // Apple, FreeBSD, DragonFly. unsafe { libc::__error() } } @@ -147,10 +163,12 @@ mod imp { let slot = &SLOTS[idx]; let prev = slot.handler.load(Ordering::SeqCst); let prev_default = prev == libc::SIG_DFL; + let mut terminate = false; let owner = OWNER_PID.load(Ordering::Acquire) == libc::getpid() as u32; let live = owner && WATCHER_ALIVE.load(Ordering::SeqCst); if prev_default && current_is_ours(sig) { + terminate = true; // Nobody to flush (a fork child, or the watcher is gone), or a // second signal while the first is still being served: behave // as if we were not installed. @@ -161,7 +179,7 @@ mod imp { if live { let fd = PIPE_W.load(Ordering::Acquire); if fd >= 0 { - let byte = sig as u8; + let byte = sig as u8 | if terminate { TERMINATE } else { 0 }; // A full pipe means wake-ups are already pending: ignore. libc::write(fd, (&byte as *const u8).cast(), 1); } @@ -210,11 +228,12 @@ mod imp { } return; } - let sig = byte as c_int; - let Some(idx) = index(sig) else { continue }; + let sig = (byte & !TERMINATE) as c_int; + if index(sig).is_none() { + continue; + } let timeout = Duration::from_millis(TIMEOUT_MS.load(Ordering::Relaxed)); - let prev_default = SLOTS[idx].handler.load(Ordering::SeqCst) == libc::SIG_DFL; - if prev_default && current_is_ours(sig) { + if byte & TERMINATE != 0 && current_is_ours(sig) { // The process is about to end because of this signal: finish // cleanly, then let it end the way it would have. let _ = std::panic::catch_unwind(|| Bugsee::shutdown(timeout)); @@ -355,10 +374,6 @@ mod imp { use crate::Bugsee; - const CTRL_CLOSE_EVENT: u32 = 2; - const CTRL_LOGOFF_EVENT: u32 = 5; - const CTRL_SHUTDOWN_EVENT: u32 = 6; - type HandlerRoutine = unsafe extern "system" fn(u32) -> i32; #[link(name = "kernel32")] @@ -373,19 +388,12 @@ mod imp { /// Runs on a thread Windows creates for the event, so blocking is fine. unsafe extern "system" fn on_event(kind: u32) -> i32 { let timeout = Duration::from_millis(TIMEOUT_MS.load(Ordering::Relaxed)); - let _ = std::panic::catch_unwind(|| match kind { - // Windows ends the process once the handlers return, whatever they - // answer: this exit is deliberate, so finish cleanly. - CTRL_CLOSE_EVENT | CTRL_LOGOFF_EVENT | CTRL_SHUTDOWN_EVENT => { - Bugsee::shutdown(timeout); - } - // Ctrl+C / Ctrl+Break: a handler registered before ours may swallow - // it and keep running, so only flush. The host ends the session by - // dropping the launch guard. - _ => { - Bugsee::flush(timeout); - } - }); + // Every event is treated as deliberate: close, logoff and shutdown end + // the process whatever the handlers answer, and for Ctrl+C / Ctrl+Break + // the default routine that follows ours does too. A host that swallows + // them registers after us (see `install_termination_flush`). + let _ = kind; + let _ = std::panic::catch_unwind(|| Bugsee::shutdown(timeout)); // FALSE: pass the event on to the next handler (the host's, or the // default one). We never swallow it. 0 diff --git a/crates/bugsee/tests/shutdown.rs b/crates/bugsee/tests/shutdown.rs index 1993060..34995b0 100644 --- a/crates/bugsee/tests/shutdown.rs +++ b/crates/bugsee/tests/shutdown.rs @@ -175,6 +175,15 @@ fn child_entry() { "sigterm_host_handler" => { set_action(libc::SIGTERM, host_handler as *const () as usize, 0); } + "sigterm_oneshot_handler" => { + // A host's one-shot handler: it must run for THIS signal and not be + // turned into a kill by the SDK's own default-disposition logic. + set_action( + libc::SIGTERM, + host_handler as *const () as usize, + libc::SA_RESETHAND, + ); + } "sigterm_siginfo_handler" => { set_action( libc::SIGTERM, @@ -239,7 +248,10 @@ fn child_entry() { std::thread::sleep(Duration::from_secs(60)); std::process::exit(3); // the signal should have ended us } - "sigterm_host_handler" | "sigterm_siginfo_handler" | "sigterm_late_handler" => { + "sigterm_host_handler" + | "sigterm_siginfo_handler" + | "sigterm_oneshot_handler" + | "sigterm_late_handler" => { ready(); exit_via_host(guard) } @@ -525,8 +537,12 @@ fn sigterm_with_the_hook_delivers_when_the_backend_is_up() { #[test] fn a_hosts_own_handler_still_runs_and_decides() { - // Plain handler, and an SA_SIGINFO one (which must get a real siginfo). - for kind in ["sigterm_host_handler", "sigterm_siginfo_handler"] { + // Plain, SA_SIGINFO (must get a real siginfo) and one-shot SA_RESETHAND. + for kind in [ + "sigterm_host_handler", + "sigterm_siginfo_handler", + "sigterm_oneshot_handler", + ] { let (out, pending, abnormal) = run_case(kind, FAIL, &[libc::SIGTERM]); assert!( out.dir.join("host-handler-ran").exists(), From 0cb980b6506f358fa931faf4fa338606ec87641f Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:17:07 +0500 Subject: [PATCH 21/52] fix(core): run launch-path recovery on a bounded helper, after the crash handlers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review of the launch-path recovery pass: - the pass runs on a helper thread and launch waits at most the budget, so a blocked before_send cannot stall or deadlock launch; the worker holds its own recovery until the pass is done (bounded) and skips what the pass claimed - Recorder::recover_on_launch_path is called by launch_with after the native handler and panic hook are installed, outside all locks - dedup repeated identical recovered crashes against the queue and cap it - recovered crashes carry no scope on either path; claimed-only attempted set; scan cap counts session markers only; docs, latency numbers, deterministic tests, facade test πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 13 +- crates/bugsee-core/src/queue.rs | 91 +++++++ crates/bugsee-core/src/recovery.rs | 11 +- crates/bugsee-core/src/runtime.rs | 256 ++++++++++++++----- crates/bugsee-core/tests/launch_recovery.rs | 260 +++++++++++++++----- crates/bugsee/src/api.rs | 8 + crates/bugsee/src/options.rs | 40 ++- crates/bugsee/tests/launch_recovery.rs | 56 +++++ 8 files changed, 597 insertions(+), 138 deletions(-) create mode 100644 crates/bugsee/tests/launch_recovery.rs diff --git a/DESIGN.md b/DESIGN.md index 99a0fef..c85632b 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -217,7 +217,18 @@ Tests: `bugsee-core/tests/multiprocess.rs` (real child processes: racing launche Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.rs`, a memory-mode case in `bugsee/tests/fork_safety.rs`, `options.rs` unit tests. A real ENOSPC (size-capped tmpfs) is not exercised, since mounting one needs root; failures are injected by blocking path components and removing the directory mid-run. -**Early-crash recovery on the launch path (#10).** A host that dies again within milliseconds of launch (a crash loop under systemd/Kubernetes) must not outrun the report of the crash before it. `Recorder::launch` therefore runs a bounded recovery pass on the calling thread, after this session's marker exists and before the pipeline threads start: the newest crashed sessions are built, passed through `before_send`/blacklist, assembled and enqueued, and only then is their evidence discarded. Upload stays asynchronous and resumes from the queue. Bounds: at most 8 sessions (`launch_recovery_max_sessions`), a time budget of 500 ms (`LaunchOptions::launch_recovery_budget`, zero disables the pass), and a scan of at most 4096 directory entries, names sorted before any file is touched. The budget is checked between sessions, so one huge session can overrun it, and the newest is always attempted. **Newest first**: the latest crash is the one most likely to belong to the build now running, and a crash loop makes only the most recent evidence reachable; the rest is not lost, since the worker's unbounded pass recovers whatever the launch pass left. Claims (#14) are unchanged: the pass takes the same per-session claim, so it composes with peers and the worker pass, and it runs only for a fresh launch (a fork child keeps recovering on its worker, and holds no lock the fork hook needs). A crash *inside* the pass (or before the native handler is installed right after it) loses nothing: the session's evidence is still on disk, retried next launch up to `MAX_RECOVERY_ATTEMPTS`. Not done: collapsing repeated identical crash-on-launch reports into one (each crash is queued; the backend deduplicates by signature and the TooManySimilar blacklist suppresses them once the server says so). Tests: `bugsee-core/tests/launch_recovery.rs` (a real abort-after-launch crash loop, time cap, zero budget). +**Early-crash recovery on the launch path (#10).** A host that dies again within milliseconds of launch (a crash loop under systemd/Kubernetes) must not outrun the report of the crash before it. `Bugsee::launch_with` therefore ends with `Recorder::recover_on_launch_path()`: a bounded pass that turns the newest crashed sessions into queue entries (build, `before_send`/blacklist, assemble, enqueue, then discard the evidence) before launch returns. Upload stays asynchronous and resumes from the queue. A bundle is queued with plain file writes and a rename, no `fsync`, so it survives process death (the case here) but not power loss, as everywhere in Β§8. + +- **Threading contract.** The pass runs on a helper thread (`bugsee-launch-recovery`) and `launch` waits at most `launch_recovery_budget` (500 ms default, `LaunchOptions::launch_recovery_budget`; zero disables the pass) by polling a flag (never parking: a parker used before a `fork` is inherited by the child, and on macOS the child then traps in libdispatch when it parks). That is a hard upper bound whatever a `before_send` callback does: on timeout `launch` returns and the pass finishes in the background, so a host that holds its own lock while calling `launch` cannot deadlock against a callback that takes it. The callback can run on an SDK thread before `launch` returns; this is documented on `LaunchOptions::before_send`. +- **Ordering with the crash handlers.** The pass is the last thing `launch_with` does, after the native handler and the panic hook are installed and outside every lock, so a fault inside it is recorded with a stack against THIS session (its marker and `crash.info` path are already this generation's) and reported by the next launch. The previous session's evidence is untouched until its report is queued, and `bump_attempts` runs before processing, so a session that crashes the assemble path is abandoned after `MAX_RECOVERY_ATTEMPTS` launches instead of looping. On a re-launch within one process the native handler is replaced with one for the new generation before the pass runs; a fault in the pass is therefore the new generation's. +- **Worker coordination.** With the pass armed, the worker holds its own recovery back until the pass is done (bounded by budget plus 5 s, so a wedged callback cannot stall capture for good), then skips generations the pass claimed (`launch_attempted`, recorded only after winning the #14 claim). Claims keep the two exclusive in any interleaving. A host using `Recorder::launch` directly with a non-zero budget must call `recover_on_launch_path` or the worker waits that grace out; the core default budget is zero, i.e. no pass. +- **Bounds.** At most 8 sessions (`launch_recovery_max_sessions`), no new session after the budget (checked between sessions, so one huge session can overrun it, and the newest is always attempted), a scan of at most 4096 session markers (and four times as many directory entries; `.lock` and `.attempts` files do not use the cap). **Newest first**: the latest crash is the one most likely to belong to the build now running, and a crash loop makes only recent evidence reachable. What the pass leaves is recovered by the worker's unbounded pass, so nothing is lost; recovered reports may therefore upload newest first, which is harmless because the server deduplicates by signature. +- **Crash-loop escalation.** A fast loop never lets the uploader finish, so the server's `12004` verdict and the local blacklist are never reached. In the pass, a recovered crash whose signature set equals one already queued is dropped (the first copy is kept; once it has been delivered a recurrence is queued again), and recovered crashes stop being queued when the queue holds 64 reports or 64 MiB (`queue::admit_recovered`; the newest is dropped). Not done: a repeat counter on the kept report. The worker pass does not dedup or cap. +- **Scope.** A recovered crash carries no scope (email, labels, attributes) from the new process, on either path: the scope describes the new run, and merging it made the two paths disagree. Persisting scope per session is possible, not done. +- **Not covered.** Sessions of the legacy data directory are recovered by the worker only. A fork child (prefork worker boot loop) recovers on its worker, not on a launch path, so a prefork worker that dies before its worker thread finishes recovery leaves the evidence for a later launch (nothing is lost, it is reported late); the master's pass covers only the master's own launch. In memory mode with a recovery directory (#9), a crash that cannot be queued on disk is delivered from memory and its evidence stays, so a later launch reports it again (the backend deduplicates). +- **Measured** (`launch_recovery.rs::measure_launch_latency`, Apple silicon, synthetic log-only windows at the default 8 MiB / 60 s caps, a pass that waits for completion): empty directory about 0.1 to 0.3 ms; one crashed session about 7 ms release, 21 ms debug; eight identical sessions about 17 ms release, 43 ms debug. The eight-session figure undercounts assembly, since seven of them are dropped as duplicates before being assembled; scrubbing (#11) will add per-session cost. `Recorder::launch` itself is about 0.5 to 0.8 ms of that. + +Tests: `bugsee-core/tests/launch_recovery.rs` (a real abort-after-launch crash loop, time and count caps, a blocked callback, zero budget), `queue.rs` and `recovery.rs` unit tests. **Generations:** each launch bumps `gen`; next-launch recovery reads the **previous** gen's parts before advancing. diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index 90f1b1e..8dbbd9e 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -102,6 +102,63 @@ pub fn list_pending(data_dir: &Path) -> Vec { out } +/// Most reports the launch-path recovery will leave queued before it stops adding +/// recovered crashes (see [`admit_recovered`]). +pub const MAX_QUEUED_REPORTS: usize = 64; +/// Most queued bundle bytes before the same. +pub const MAX_QUEUED_BYTES: u64 = 64 << 20; + +/// Whether the launch-path recovery should queue a recovered crash. +#[derive(Debug, PartialEq, Eq)] +pub enum Admission { + Admit, + /// A queued report already carries exactly this signature set. + Duplicate, + /// The queue is at its entry or byte cap. + Full, +} + +/// Decide whether a recovered crash with `signatures` may be queued. +/// +/// A host in a crash loop re-reports the same crash on every start, and the loop +/// is faster than the uploader, so neither the server's duplicate verdict nor the +/// local blacklist is ever reached. Instead the first copy stays queued and +/// later copies with the identical, non-empty signature set are dropped; once it +/// has been delivered a recurrence is queued again. The queue is also capped +/// (entries and bytes): beyond it the newest recovered crash is dropped rather +/// than grow the directory without bound. Reports without signatures are never +/// treated as duplicates. +pub fn admit_recovered(data_dir: &Path, signatures: &[String]) -> Admission { + let wanted: BTreeSet<&String> = signatures.iter().collect(); + let mut count = 0usize; + let mut bytes = 0u64; + for report in list_pending(data_dir) { + count += 1; + bytes += std::fs::metadata(&report.bundle).map_or(0, |m| m.len()); + if count >= MAX_QUEUED_REPORTS || bytes >= MAX_QUEUED_BYTES { + return Admission::Full; + } + if wanted.is_empty() { + continue; + } + let queued: BTreeSet = std::fs::read(report.req_path()) + .ok() + .and_then(|b| serde_json::from_slice::(&b).ok()) + .and_then(|v| { + v.get("signatures").and_then(|s| s.as_array()).map(|a| { + a.iter() + .filter_map(|x| x.as_str().map(String::from)) + .collect() + }) + }) + .unwrap_or_default(); + if queued.len() == wanted.len() && wanted.iter().all(|s| queued.contains(*s)) { + return Admission::Duplicate; + } + } + Admission::Admit +} + /// Load a queued report's bundle bytes, request body, retry count, and the /// earliest next-attempt time (epoch ms). pub fn load(report: &QueuedReport) -> std::io::Result<(Vec, Vec, u32, i64)> { @@ -298,6 +355,40 @@ pub fn any_blacklisted(data_dir: &Path, sigs: &[String]) -> bool { mod tests { use super::*; + fn queue_report(data: &Path, sigs: &[&str]) { + let list: Vec = sigs.iter().map(|s| format!("\"{s}\"")).collect(); + enqueue( + data, + &AssembledReport { + bundle_name: format!("{}.bundle.zip", crate::util::random_hex(10)), + zip: b"z".to_vec(), + request_json: format!("{{\"signatures\":[{}]}}", list.join(",")).into_bytes(), + }, + ) + .unwrap(); + } + + #[test] + fn recovered_crashes_are_deduplicated_against_the_queue_and_capped() { + let data = std::env::temp_dir().join(format!("bugsee-qadm-{}", crate::util::random_hex(8))); + let sig = |s: &str| vec![s.to_string()]; + assert_eq!(admit_recovered(&data, &sig("a")), Admission::Admit); + queue_report(&data, &["a"]); + assert_eq!(admit_recovered(&data, &sig("a")), Admission::Duplicate); + assert_eq!(admit_recovered(&data, &sig("b")), Admission::Admit); + // A different signature SET is not a duplicate; neither is no signature. + assert_eq!( + admit_recovered(&data, &["a".to_string(), "b".to_string()]), + Admission::Admit + ); + assert_eq!(admit_recovered(&data, &[]), Admission::Admit); + for i in 1..MAX_QUEUED_REPORTS { + queue_report(&data, &[&format!("s{i}")]); + } + assert_eq!(admit_recovered(&data, &sig("new")), Admission::Full); + let _ = std::fs::remove_dir_all(&data); + } + #[test] fn gc_spares_a_peers_in_flight_enqueue_and_reaps_old_orphans() { let data = std::env::temp_dir().join(format!("bugsee-qgc-{}", crate::util::random_hex(8))); diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 245c7e3..76f7b1a 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -85,7 +85,12 @@ pub fn find_pending_bounded( }; let self_pid = std::process::id(); let mut candidates: Vec = Vec::new(); - for entry in read.flatten().take(scan_cap) { + // The cap counts session markers only: a session also leaves `.lock` and + // `.attempts` files that must not use it up. + // Entries read are bounded too (a session leaves about three files), so a + // directory full of unrelated files cannot make the scan long either. + let mut seen = 0usize; + for entry in read.flatten().take(scan_cap.saturating_mul(4)) { let name = entry.file_name().to_string_lossy().into_owned(); let Some(gen_str) = name.strip_suffix(".alive") else { continue; @@ -93,6 +98,10 @@ pub fn find_pending_bounded( let Ok(generation) = gen_str.parse::() else { continue; }; + if seen >= scan_cap { + break; + } + seen += 1; if generation < current_generation { candidates.push(generation); } diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 2362638..2c7f303 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -183,9 +183,13 @@ pub struct RecorderConfig { /// crashed sessions and deliver its queue. For a host layer that moved its /// default location and must not strand reports left at the old one. pub legacy_data_dir: Option, - /// Time `launch` may spend queueing the previous run's crashes before it - /// returns (see [`Recorder::launch`]). Zero skips the launch-path pass: all - /// recovery then happens on the worker thread after `launch` returns. + /// Time [`Recorder::recover_on_launch_path`] waits for the launch-path pass + /// (a hard bound; the pass then finishes in the background) and after which + /// the pass starts no further session. Zero, the default here, disables the + /// pass: all recovery happens on the worker thread. The facade sets + /// [`DEFAULT_LAUNCH_RECOVERY_BUDGET`]. With a non-zero budget the worker holds + /// its own recovery back until the pass has run, so the host layer must call + /// [`Recorder::recover_on_launch_path`] (or the worker waits out a few seconds). pub launch_recovery_budget: Duration, /// Most crashed sessions the launch-path pass recovers. pub launch_recovery_max_sessions: usize, @@ -211,7 +215,7 @@ impl RecorderConfig { max_queued_entries: DEFAULT_MAX_QUEUED, memory_only: None, legacy_data_dir: None, - launch_recovery_budget: DEFAULT_LAUNCH_RECOVERY_BUDGET, + launch_recovery_budget: Duration::ZERO, launch_recovery_max_sessions: DEFAULT_LAUNCH_RECOVERY_SESSIONS, } } @@ -269,6 +273,26 @@ struct Shared { /// Set the first time a disk write fails while running in disk mode: the /// recorder is no longer fully durable (see [`Recorder::is_degraded`]). degraded: AtomicBool, + /// The launch-path recovery pass to run, taken by + /// [`Recorder::recover_on_launch_path`]; `None` when not armed. + launch_plan: ForkLock>, + /// Generations the launch-path pass claimed, which the worker's pass skips: + /// one it could only hold in memory still has its evidence on disk, and doing + /// it again would report it twice. + launch_attempted: ForkLock>, + /// Whether the worker holds its own recovery until the launch-path pass is + /// over (it is armed and the pass has not finished). + launch_armed: AtomicBool, + launch_done: AtomicBool, + launch_wait_ms: AtomicU64, +} + +/// What the launch-path recovery pass needs, fixed at launch. +struct LaunchPlan { + dir: PathBuf, + generation: u64, + budget: Duration, + max_sessions: usize, } /// The recorder's own locks join the `fork()` lock protocol, so a forked child @@ -281,8 +305,12 @@ impl ForkLockable for Shared { self.mem_queue.fork_lock(); self.mem_blacklist.fork_lock(); self.delivered_stuck.fork_lock(); + self.launch_plan.fork_lock(); + self.launch_attempted.fork_lock(); } fn fork_unlock(&self) { + self.launch_attempted.fork_unlock(); + self.launch_plan.fork_unlock(); self.delivered_stuck.fork_unlock(); self.mem_blacklist.fork_unlock(); self.mem_queue.fork_unlock(); @@ -351,13 +379,10 @@ impl Recorder { /// Start the workers and begin capturing. Recovers any prior session that /// ended abnormally, queueing it for delivery. /// - /// The newest crashed sessions (at most - /// [`RecorderConfig::launch_recovery_max_sessions`], newest first, within - /// [`RecorderConfig::launch_recovery_budget`]) are turned into durable queue - /// entries before this returns, so a host that crashes again immediately has - /// already got the previous crash's report on disk. Upload is asynchronous and - /// resumes from the queue. Anything beyond the caps is recovered by the worker - /// thread right after. + /// Recovery of the newest crashed sessions can be made to finish on the + /// launch path with [`Recorder::recover_on_launch_path`], which the host layer + /// calls after installing its crash handlers (see there). Whatever it does not + /// cover is recovered by the worker thread. pub fn launch(config: RecorderConfig, transport: Arc) -> std::io::Result { let env = Environment::detect_with( &config.sdk_version, @@ -386,6 +411,11 @@ impl Recorder { mem_blacklist: ForkLock::new(BTreeSet::new()), delivered_stuck: ForkLock::new(BTreeSet::new()), degraded: AtomicBool::new(false), + launch_plan: ForkLock::new(None), + launch_attempted: ForkLock::new(BTreeSet::new()), + launch_armed: AtomicBool::new(false), + launch_done: AtomicBool::new(false), + launch_wait_ms: AtomicU64::new(0), }); crate::fork::install_fork_hook(); @@ -423,25 +453,20 @@ impl Recorder { // recovering them only reads and deletes. So an existing directory is // still drained and recovered from in memory mode. let drain_dir = (session.is_some() || data_dir.is_dir()).then_some(data_dir.as_path()); - // Queue the previous run's crashes before returning (issue #10): a host - // that crashes again right after launch must not outrun their report. - // This runs after this session's marker exists and before the native - // handler is installed; a session's evidence stays on disk until its - // report is queued, so a crash inside this pass just retries next launch - // (bounded by `MAX_RECOVERY_ATTEMPTS`). - let mut attempted = BTreeSet::new(); + // Arm the launch-path recovery pass (issue #10). It does not run here: + // the host layer runs it with `recover_on_launch_path` once its crash + // handlers are installed, so a fault inside the pass is captured with a + // stack. Until then the worker holds its own recovery back. if let (Some(dir), false) = (drain_dir, config.launch_recovery_budget.is_zero()) { - let generation = session.as_ref().map_or(u64::MAX, Session::generation); - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - run_launch_recovery( - &shared, - dir, - generation, - config.launch_recovery_budget, - config.launch_recovery_max_sessions, - &mut attempted, - ) - })); + *shared.launch_plan.lock() = Some(LaunchPlan { + dir: dir.to_path_buf(), + generation: session.as_ref().map_or(u64::MAX, Session::generation), + budget: config.launch_recovery_budget, + max_sessions: config.launch_recovery_max_sessions, + }); + let wait_ms = config.launch_recovery_budget.as_millis() as u64; + shared.launch_wait_ms.store(wait_ms, Ordering::SeqCst); + shared.launch_armed.store(true, Ordering::SeqCst); } let pipeline = spawn_pipeline( &shared, @@ -455,7 +480,6 @@ impl Recorder { config.sampler, true, config.legacy_data_dir, - attempted, )?; Ok(Recorder { @@ -549,7 +573,6 @@ impl Recorder { sampler, false, None, - BTreeSet::new(), )?; // The parent's session is still the parent's: leave its marker alone. @@ -566,6 +589,24 @@ impl Recorder { Ok(()) } + /// Run the launch-path recovery pass (issue #10): turn the newest crashed + /// sessions of earlier runs into queued reports before returning. + /// + /// Call it once, after the crash handlers are installed (so a fault inside the + /// pass is captured with a stack, attributed to this session) and without + /// holding any lock a `before_send` hook might take. The pass runs on a helper + /// thread and this waits at most [`RecorderConfig::launch_recovery_budget`] + /// for it: on timeout it returns and the pass finishes in the background, so + /// `launch` has a hard upper bound whatever the host's hook does. `before_send` + /// can therefore run on an SDK thread before this returns. + /// + /// Does nothing when the budget is zero or it already ran. A recorder whose + /// budget is non-zero holds its worker's own recovery back until this has + /// finished, for at most the budget plus a few seconds. + pub fn recover_on_launch_path(&self) { + self.handle().recover_on_launch_path(); + } + /// Enqueue an entry. Never blocks; drops (newest) under back-pressure or if /// the worker is gone. pub fn capture(&self, entry: CaptureEntry) { @@ -689,6 +730,9 @@ impl Recorder { impl Drop for Recorder { fn drop(&mut self) { + // A worker still holding its recovery for the launch-path pass must not + // make the stop below time out. + self.shared.launch_done.store(true, Ordering::SeqCst); // Signal both threads; JOIN only if they acknowledge within the timeout. // A thread wedged in a host callback with no timeout is DETACHED (its // JoinHandle dropped un-joined) rather than hanging the stopping thread @@ -754,6 +798,36 @@ pub struct RecorderHandle { } impl RecorderHandle { + /// See [`Recorder::recover_on_launch_path`]. + pub fn recover_on_launch_path(&self) { + let Some(plan) = self.shared.launch_plan.lock().take() else { + return; + }; + let budget = plan.budget; + let shared = Arc::clone(&self.shared); + let spawned = std::thread::Builder::new() + .name("bugsee-launch-recovery".into()) + .spawn(move || { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run_launch_recovery(&shared, &plan); + })); + shared.launch_done.store(true, Ordering::SeqCst); + }); + match spawned { + // Poll, never park: this thread's parker (a libdispatch semaphore on + // macOS) would then be inherited in whatever state it was left by a + // later `fork`, whose child traps in libdispatch when it parks. + Ok(_) => { + let deadline = Instant::now() + budget; + while !self.shared.launch_done.load(Ordering::SeqCst) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(1)); + } + } + // No thread: the worker's own pass recovers everything. + Err(_) => self.shared.launch_done.store(true, Ordering::SeqCst), + } + } + /// Enqueue a capture entry (bounded, drop-newest under back-pressure). pub fn capture(&self, entry: CaptureEntry) { capture_into(&self.shared, &self.tx, entry); @@ -903,7 +977,6 @@ fn spawn_pipeline( sampler: Option>, fresh_launch: bool, legacy_data_dir: Option, - launch_attempted: BTreeSet, ) -> std::io::Result { // Uploader thread: drains the durable queue with retry. let (upload_tx, upload_rx) = channel(); @@ -956,13 +1029,16 @@ fn spawn_pipeline( // it must, or that crash waits for the master to restart. Claims // keep siblings from recovering the same session twice. if let Some(dir) = &worker_drain_dir { - run_recovery(&worker_shared, dir, generation, &launch_attempted); + if fresh_launch { + wait_for_launch_pass(&worker_shared); + } + run_recovery(&worker_shared, dir, generation, fresh_launch); } // Best effort, once: sessions a previous default location left // behind. No generation of ours lives there, so none is skipped. if let Some(dir) = &legacy_data_dir { queue::gc_orphans(dir); - run_recovery(&worker_shared, dir, u64::MAX, &BTreeSet::new()); + run_recovery(&worker_shared, dir, u64::MAX, false); } })); legacy_ready.store(true, Ordering::SeqCst); @@ -992,40 +1068,71 @@ fn spawn_pipeline( /// The crashed session's state is discarded ONLY after the report is durably /// enqueued β€” a failure leaves it on disk for a later attempt (no crash loss). /// -/// `skip` holds generations the launch-path pass already attempted: one it could -/// only hold in memory still has its evidence on disk, and doing it again would -/// report it twice. -fn run_recovery(shared: &Shared, data_dir: &Path, current_generation: u64, skip: &BTreeSet) { +/// With `skip_launch_pass`, generations the launch-path pass already claimed are +/// left alone. +fn run_recovery(shared: &Shared, data_dir: &Path, current_generation: u64, skip_launch_pass: bool) { for pending in recovery::find_pending(data_dir, current_generation) { - if !skip.contains(&pending.generation) { - recover_claimed(shared, data_dir, &pending); + if skip_launch_pass && shared.launch_attempted.lock().contains(&pending.generation) { + continue; } + recover_claimed(shared, data_dir, &pending, false); } // Reap parts/attempts left by cleanly-exited or already-recovered generations. recovery::gc_orphans(data_dir, current_generation); } -/// Claim one pending session and recover it. A session another process holds is -/// skipped. -fn recover_claimed(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSession) { +/// Claim one pending session and recover it. Returns whether this process won the +/// claim: a session another process holds is skipped. `launch_pass` applies the +/// launch path's dedup and queue cap to the recovered report. +fn recover_claimed( + shared: &Shared, + data_dir: &Path, + pending: &recovery::PendingSession, + launch_pass: bool, +) -> bool { // Several processes share this directory (a prefork master and its // workers, above all): exactly one recovers a given session. let Some(_claim) = recovery::claim_pending(data_dir, pending) else { - return; + return false; }; - // Count the attempt *before* processing: a corrupt session that panics - // the build/assemble path is a poison pill that would otherwise re-run β€” - // and abort the whole loop β€” on every launch. After the cap, abandon it. + if launch_pass { + shared.launch_attempted.lock().insert(pending.generation); + } + // Count the attempt *before* processing: a corrupt session that panics or + // faults in the build/assemble path is a poison pill that would otherwise + // re-run on every launch. After the cap, abandon it. let attempts = recovery::bump_attempts(data_dir, pending); if attempts > MAX_RECOVERY_ATTEMPTS { recovery::discard(data_dir, pending); - return; + return true; } // Isolate each pending so one corrupt session cannot block recovering the // rest (a panic here would otherwise unwind the entire loop). let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - recover_one(shared, data_dir, pending); + recover_one(shared, data_dir, pending, launch_pass); })); + true +} + +/// How long the worker holds its recovery back for the launch-path pass beyond +/// the pass's own budget. The pass runs host code (`before_send`) that can block; +/// the worker must not stall capture behind it for good. +const LAUNCH_PASS_GRACE: Duration = Duration::from_secs(5); + +/// Hold the worker's recovery until the launch-path pass has finished, so the two +/// do not interleave (claims would keep them exclusive, but the pass's contract is +/// that it handles the newest sessions first). A no-op when the pass is not armed. +/// Bounded: if the host layer never runs the pass, or the pass wedges, the worker +/// goes ahead. +fn wait_for_launch_pass(shared: &Shared) { + if !shared.launch_armed.load(Ordering::SeqCst) { + return; + } + let wait = Duration::from_millis(shared.launch_wait_ms.load(Ordering::SeqCst)); + let deadline = Instant::now() + wait + LAUNCH_PASS_GRACE; + while !shared.launch_done.load(Ordering::SeqCst) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(2)); + } } /// Directory entries the launch-path pass reads before giving up on finding more. @@ -1045,30 +1152,34 @@ const LAUNCH_SCAN_CAP: usize = 4096; /// the latest crash is the one most likely to be on the build now running, and /// what is left is not lost, since the worker's unbounded pass recovers the rest. /// -/// Fills `attempted` with the generations it tried, for the worker's pass to skip. -fn run_launch_recovery( - shared: &Shared, - data_dir: &Path, - current_generation: u64, - budget: Duration, - max_sessions: usize, - attempted: &mut BTreeSet, -) { +/// Records the generations it claimed in `launch_attempted`, for the worker's pass +/// to skip. +fn run_launch_recovery(shared: &Shared, plan: &LaunchPlan) { let started = Instant::now(); - for pending in - recovery::find_pending_bounded(data_dir, current_generation, LAUNCH_SCAN_CAP, max_sessions) - { - if !attempted.is_empty() && started.elapsed() >= budget { + let mut claimed = 0; + for pending in recovery::find_pending_bounded( + &plan.dir, + plan.generation, + LAUNCH_SCAN_CAP, + plan.max_sessions, + ) { + if claimed > 0 && started.elapsed() >= plan.budget { break; } - attempted.insert(pending.generation); - recover_claimed(shared, data_dir, &pending); + if recover_claimed(shared, &plan.dir, &pending, true) { + claimed += 1; + } } } /// Build, filter, assemble and enqueue one pending session's crash report, /// discarding its on-disk evidence only once it is safely queued. -fn recover_one(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSession) { +fn recover_one( + shared: &Shared, + data_dir: &Path, + pending: &recovery::PendingSession, + launch_pass: bool, +) { let span = export::report_span(&pending.parts_dir).ok().flatten(); let now = epoch_ms(); let (start, end) = span.unwrap_or((now, now)); @@ -1083,7 +1194,20 @@ fn recover_one(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSess recovery::discard(data_dir, pending); return; } - merge_scope(shared, &mut built.meta); + // A recovered crash carries no scope (email/labels/attributes) from THIS + // process: they describe the new run, not the one that crashed, and the + // launch-path and worker passes must agree on it. + if launch_pass { + match queue::admit_recovered(data_dir, &built.meta.signatures) { + queue::Admission::Admit => {} + // A crash loop reports the same crash on every start: the first one + // is already queued, and another copy adds nothing the server keeps. + queue::Admission::Duplicate | queue::Admission::Full => { + recovery::discard(data_dir, pending); + return; + } + } + } let window = TimeWindow { start, end }; let assembled = reporting::assemble_with_extras( diff --git a/crates/bugsee-core/tests/launch_recovery.rs b/crates/bugsee-core/tests/launch_recovery.rs index 7d25c5b..4f2844d 100644 --- a/crates/bugsee-core/tests/launch_recovery.rs +++ b/crates/bugsee-core/tests/launch_recovery.rs @@ -5,23 +5,27 @@ // Copyright Β© 2026 Bugsee. All rights reserved. // //! Early-crash recovery finishes on the launch path (issue #10): by the time -//! `launch` returns, the previous run's crash is a durable queue entry, so a host -//! that crashes again right away (a crash loop) cannot outrun its report. +//! `recover_on_launch_path` returns, the previous run's crash is a queue entry on +//! disk, so a host that crashes again right away (a crash loop) cannot outrun its +//! report. //! //! The crash loop is real: this test binary re-executes itself as children that //! launch the SDK and `abort()` immediately. use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant}; +use bugsee_core::capture::record; use bugsee_core::transport::{Transport, TransportError}; use bugsee_core::{queue, MockTransport, Recorder, RecorderConfig}; const ROLE: &str = "BUGSEE_LR_ROLE"; const DIR: &str = "BUGSEE_LR_DIR"; const OUT: &str = "BUGSEE_LR_OUT"; +const LAUNCH_THREAD: &str = "bugsee-launch-recovery"; /// A transport that never delivers anything, so the queue keeps its entries. struct Offline; @@ -37,6 +41,12 @@ impl Transport for Offline { } } +fn config(dir: &Path, budget: Duration) -> RecorderConfig { + let mut config = RecorderConfig::new(dir, "tok"); + config.launch_recovery_budget = budget; + config +} + /// Child entry point. A no-op in a normal test run. #[test] fn child_entry() { @@ -45,8 +55,10 @@ fn child_entry() { } let dir = PathBuf::from(std::env::var(DIR).unwrap()); let out = PathBuf::from(std::env::var(OUT).unwrap()); - let _rec = Recorder::launch(RecorderConfig::new(&dir, "tok"), Arc::new(Offline)).unwrap(); - // The instant `launch` returns: record what is durably queued, then die the + let rec = + Recorder::launch(config(&dir, Duration::from_millis(500)), Arc::new(Offline)).unwrap(); + rec.recover_on_launch_path(); + // The instant the launch path is done: record what is queued, then die the // way a crash does, with no destructor and no flush. std::fs::write(out, queue::list_pending(&dir).len().to_string()).unwrap(); std::process::abort(); @@ -78,83 +90,146 @@ fn seed_crashed(dir: &Path, generations: std::ops::RangeInclusive) { std::fs::write(dir.join("gen"), generations.end().to_string()).unwrap(); } +/// A `before_send` that records the name of the thread it runs on and sleeps. +fn recording_hook( + threads: &Arc>>, + sleep: Duration, +) -> bugsee_core::runtime::BeforeSend { + let seen = Arc::clone(threads); + Box::new(move |_meta| { + let t = std::thread::current(); + seen.lock() + .unwrap() + .push(t.name().unwrap_or("launch").to_string()); + std::thread::sleep(sleep); + true + }) +} + +fn on_launch_thread(threads: &Arc>>) -> usize { + threads + .lock() + .unwrap() + .iter() + .filter(|n| *n == LAUNCH_THREAD) + .count() +} + #[test] -fn the_previous_crash_is_queued_before_launch_returns() { +fn the_previous_crash_is_queued_before_the_launch_path_returns() { let tmp = TempDir::new(); let dir = tmp.0.join("data"); seed_crashed(&dir, 1..=1); - let rec = Recorder::launch(RecorderConfig::new(&dir, "tok"), Arc::new(Offline)).unwrap(); - // No flush, no waiting: it is on disk the moment launch returns. + let rec = + Recorder::launch(config(&dir, Duration::from_millis(500)), Arc::new(Offline)).unwrap(); + rec.recover_on_launch_path(); + // No flush, no waiting: it is on disk the moment the launch path returns. assert_eq!(queue::list_pending(&dir).len(), 1); drop(rec); } -/// Several pending crashes, a time cap, and newest-first: the hook observes which -/// thread recovers each. With a budget shorter than one session only the newest -/// is recovered on the launch path; the rest follow on the worker. +/// Time cap: a budget shorter than one session lets exactly the newest session +/// through on the launch path (it is always attempted); the rest follow on the +/// worker. And the launch path returns at the budget, not when the hook does. #[test] -fn the_launch_path_is_capped_by_time_and_the_rest_follows_in_the_background() { +fn a_short_budget_recovers_one_session_on_the_launch_path() { let tmp = TempDir::new(); let dir = tmp.0.join("data"); seed_crashed(&dir, 1..=5); let threads: Arc>> = Arc::default(); - let seen = Arc::clone(&threads); - let mut config = RecorderConfig::new(&dir, "tok"); - config.launch_recovery_budget = Duration::from_millis(150); - config.before_send = Some(Box::new(move |_meta| { - let t = std::thread::current(); - seen.lock() - .unwrap() - .push(t.name().unwrap_or("launch").to_string()); - std::thread::sleep(Duration::from_millis(100)); - true - })); + let mut config = config(&dir, Duration::from_millis(50)); + config.before_send = Some(recording_hook(&threads, Duration::from_millis(400))); let transport = Arc::new(MockTransport::default()); - let me = std::thread::current().name().map(String::from); + let started = Instant::now(); let rec = Recorder::launch(config, transport.clone()).unwrap(); - - let after_launch = threads.lock().unwrap().clone(); - // 100 ms per session against a 150 ms budget: two start, a third does not. - let on_launch_path = after_launch - .iter() - .filter(|n| Some(n.as_str()) != Some("bugsee-capture")) - .count(); - assert_eq!( - on_launch_path, 2, - "hook threads: {after_launch:?}, test thread {me:?}" + rec.recover_on_launch_path(); + assert!( + started.elapsed() < Duration::from_millis(300), + "the launch path waited for the hook: {:?}", + started.elapsed() ); - assert!(rec.flush(Duration::from_secs(10))); - assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 5); - let all = threads.lock().unwrap().clone(); + assert!(rec.flush(Duration::from_secs(20))); assert_eq!( - all.len(), - 5, - "each crash was processed exactly once: {all:?}" + on_launch_thread(&threads), + 1, + "{:?}", + threads.lock().unwrap() ); + assert_eq!(threads.lock().unwrap().len(), 5, "each session once"); + assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 5); } +/// Count cap: with time to spare, `launch_recovery_max_sessions` limits the pass. #[test] -fn a_zero_budget_leaves_recovery_to_the_worker() { +fn the_session_cap_limits_the_launch_path() { let tmp = TempDir::new(); let dir = tmp.0.join("data"); - seed_crashed(&dir, 1..=2); + seed_crashed(&dir, 1..=5); let threads: Arc>> = Arc::default(); - let seen = Arc::clone(&threads); - let mut config = RecorderConfig::new(&dir, "tok"); - config.launch_recovery_budget = Duration::ZERO; + let mut config = config(&dir, Duration::from_secs(10)); + config.launch_recovery_max_sessions = 2; + config.before_send = Some(recording_hook(&threads, Duration::ZERO)); + let transport = Arc::new(MockTransport::default()); + let rec = Recorder::launch(config, transport.clone()).unwrap(); + rec.recover_on_launch_path(); + // The pass was quick, so it is over: exactly the cap ran on it. + assert_eq!(on_launch_thread(&threads), 2); + + assert!(rec.flush(Duration::from_secs(20))); + assert_eq!(threads.lock().unwrap().len(), 5, "each session once"); + // The pass's second session repeats the first one's crash and is dropped. + assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 4); +} + +/// A hook that never returns must not hold `launch` past the budget. +#[test] +fn a_blocked_before_send_cannot_stall_the_launch_path() { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + seed_crashed(&dir, 1..=1); + + let release = Arc::new(AtomicBool::new(false)); + let gate = Arc::clone(&release); + let mut config = config(&dir, Duration::from_millis(100)); config.before_send = Some(Box::new(move |_meta| { - let t = std::thread::current(); - seen.lock() - .unwrap() - .push(t.name().unwrap_or("launch").to_string()); + while !gate.load(Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(5)); + } true })); let transport = Arc::new(MockTransport::default()); + let started = Instant::now(); + let rec = Recorder::launch(config, transport.clone()).unwrap(); + rec.recover_on_launch_path(); + assert!( + started.elapsed() < Duration::from_secs(2), + "launch waited on a blocked hook: {:?}", + started.elapsed() + ); + + // Unblocked, the pass finishes in the background and the report goes out. + release.store(true, Ordering::SeqCst); + assert!(rec.flush(Duration::from_secs(20))); + assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 1); +} + +/// Without a launch-path pass (the core default), the worker does it all. +#[test] +fn a_zero_budget_leaves_recovery_to_the_worker() { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + seed_crashed(&dir, 1..=2); + + let threads: Arc>> = Arc::default(); + let mut config = config(&dir, Duration::ZERO); + config.before_send = Some(recording_hook(&threads, Duration::ZERO)); + let transport = Arc::new(MockTransport::default()); let rec = Recorder::launch(config, transport.clone()).unwrap(); + rec.recover_on_launch_path(); assert!(rec.flush(Duration::from_secs(10))); assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 2); assert!(threads @@ -164,11 +239,12 @@ fn a_zero_budget_leaves_recovery_to_the_worker() { .all(|n| n == "bugsee-capture")); } -/// The product scenario: the host crashes right after every launch. Each launch -/// must already hold the previous crash in the queue, and a later healthy run -/// delivers every one of them exactly once. +/// The product scenario: the host crashes right after every launch. A crash loop +/// reports the same crash each time, so the first copy is queued and later copies +/// are not stacked behind it; and each launch already holds the previous crash in +/// the queue on return, with no async worker involved. #[test] -fn a_crash_loop_loses_no_crash_and_reports_each_exactly_once() { +fn a_crash_loop_queues_the_first_crash_before_each_relaunch_and_collapses_repeats() { const CRASHES: usize = 5; let tmp = TempDir::new(); let dir = tmp.0.join("data"); @@ -190,22 +266,86 @@ fn a_crash_loop_loses_no_crash_and_reports_each_exactly_once() { .unwrap_or_else(|_| panic!("child {i} never got past launch")) .parse() .unwrap(); - // Child i found the crashes of children 0..i already queued, with - // the previous child's among them: none waited for an async worker. - assert_eq!(queued, i, "child {i} saw {queued} queued crashes"); + // Child 0 has nothing to recover. Every later child found the crash + // before it already queued, and identical repeats are not queued again. + assert_eq!(queued, usize::from(i > 0), "child {i} saw {queued} queued"); } - // A healthy run: everything is delivered, once. + // Another launch, offline so nothing is delivered behind the test's back, + // recovers the last child's crash: a repeat of the queued one, so dropped. + let rec = + Recorder::launch(config(&dir, Duration::from_millis(500)), Arc::new(Offline)).unwrap(); + rec.recover_on_launch_path(); + assert_eq!(queue::list_pending(&dir).len(), 1); + drop(rec); + + // A healthy run delivers the queued crash, once. let transport = Arc::new(MockTransport::default()); - let rec = Recorder::launch(RecorderConfig::new(&dir, "tok"), transport.clone()).unwrap(); + let rec = + Recorder::launch(config(&dir, Duration::from_millis(500)), transport.clone()).unwrap(); + rec.recover_on_launch_path(); assert!(rec.flush(Duration::from_secs(20))); - assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), CRASHES); + assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 1); assert!(queue::list_pending(&dir).is_empty()); drop(rec); // And not again. let again = Arc::new(MockTransport::default()); - let rec = Recorder::launch(RecorderConfig::new(&dir, "tok"), again.clone()).unwrap(); + let rec = Recorder::launch(config(&dir, Duration::from_millis(500)), again.clone()).unwrap(); + rec.recover_on_launch_path(); assert!(rec.flush(Duration::from_secs(5))); assert!(again.uploaded_bundles.lock().unwrap().is_empty()); } + +/// Launch latency of the pass, for DESIGN.md: run with +/// `cargo test -p bugsee-core --release --test launch_recovery -- --ignored --nocapture`. +#[test] +#[ignore] +fn measure_launch_latency() { + fn seed_with_window(dir: &Path, generations: std::ops::RangeInclusive, bytes: usize) { + seed_crashed(dir, generations.clone()); + for g in generations { + let part = dir.join("parts").join(g.to_string()).join("0"); + std::fs::create_dir_all(&part).unwrap(); + let mut buf = Vec::new(); + let mut ts = 1_000i64; + while buf.len() < bytes { + let payload = format!( + r#"{{"timestamp":{ts},"level":1,"source":2,"message":"{}"}}"#, + "x".repeat(200) + ); + record::frame(ts, payload.as_bytes(), &mut buf); + ts += 1; + } + std::fs::write(part.join("log.part"), &buf).unwrap(); + } + } + for (label, sessions, bytes) in [ + ("empty directory", 0u64, 0usize), + ("1 session, 8 MiB window", 1, 8 << 20), + ("8 sessions, 8 MiB windows", 8, 8 << 20), + ] { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + std::fs::create_dir_all(&dir).unwrap(); + if sessions > 0 { + seed_with_window(&dir, 1..=sessions, bytes); + } + let mut config = config(&dir, Duration::from_secs(30)); + config.caps = bugsee_core::capture::WindowCaps { + max_window_ms: 60_000, + max_bytes: 8 << 20, + max_events: 100_000, + }; + let t0 = Instant::now(); + let rec = Recorder::launch(config, Arc::new(Offline)).unwrap(); + let launched = t0.elapsed(); + let t1 = Instant::now(); + rec.recover_on_launch_path(); + println!( + "{label}: Recorder::launch {launched:?}, recover_on_launch_path {:?}, queued {}", + t1.elapsed(), + queue::list_pending(&dir).len() + ); + } +} diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 1ecf9aa..648f445 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -178,6 +178,7 @@ impl Bugsee { #[cfg(feature = "panic")] let panic_info_path = recorder.panic_info_path(); + let recovery_handle = recorder.handle(); // Install the native crash handler pointing at this generation's marker. // Replace under a brief lock, then drop the OLD handler outside it. @@ -230,6 +231,13 @@ impl Bugsee { bugsee_panic::set_report_from_hook(options.report_panics_from_hook); } + // Last, once the native handler and the panic hook are in place: a fault + // inside the recovery pass is then recorded, with a stack, against THIS + // session (the previous crash's evidence stays on disk and is retried, + // attempt-counted). Run outside every lock: it can call the host's + // `before_send` on a helper thread, and waits at most the launch budget. + recovery_handle.recover_on_launch_path(); + Ok(LaunchGuard { _private: () }) } diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index 998569c..fa9b1de 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -78,6 +78,15 @@ impl LaunchOptions { /// Register a callback run on every report before delivery. Mutate the /// metadata (summary/severity/labels/attributes/…) in place; return `false` /// to drop the report. + /// + /// Threading: the callback runs on an SDK thread (the capture worker, or the + /// launch-recovery helper thread for a crash recovered during `launch`), and + /// that can be before `launch` has returned. Do not assume it runs on the + /// thread that launched, and do not make `launch` wait on a lock the callback + /// needs: `launch` itself never waits for the callback longer than + /// [`launch_recovery_budget`](Self::launch_recovery_budget), but a callback + /// that blocks forever stalls report processing. A recovered crash carries no + /// scope (email, labels, attributes) from the new process. pub fn before_send( mut self, f: impl Fn(&mut ReportMeta) -> bool + Send + Sync + 'static, @@ -214,18 +223,29 @@ impl LaunchOptions { self } - /// Time `launch` may spend queueing the previous run's crash reports before - /// it returns. Default 500 ms. + /// How long `launch` waits for the previous run's crash reports to be + /// queued. Default 500 ms + /// ([`DEFAULT_LAUNCH_RECOVERY_BUDGET`](bugsee_core::runtime::DEFAULT_LAUNCH_RECOVERY_BUDGET)). /// /// A host that crashes again right after launch (a crash loop under a - /// supervisor) still gets the previous crash reported, because it is made - /// durable on this path; only the upload is asynchronous. The newest crashed - /// session is always queued, and at most 8 are queued here, newest first; - /// the budget stops further ones from starting once it has elapsed, and - /// whatever is left is recovered on the background worker. It is checked - /// between sessions, so one very large session can exceed it (the cost of a - /// session grows with its capture window, `max_window`/`max_bytes`). `Duration::ZERO` skips the launch-path - /// pass and recovers everything in the background. + /// supervisor) still gets the previous crash reported, because it is put in + /// the on-disk queue on this path (it survives process death, not power + /// loss); only the upload is asynchronous. The work runs on a helper thread + /// and `launch` waits at most this long, so this is a hard upper bound on + /// what recovery adds to `launch`, whatever a `before_send` callback does; if + /// it runs out the pass finishes in the background. The newest crashed + /// session is attempted first, at most + /// [`DEFAULT_LAUNCH_RECOVERY_SESSIONS`](bugsee_core::runtime::DEFAULT_LAUNCH_RECOVERY_SESSIONS) + /// are taken, newest first, and no further session starts once the budget has + /// elapsed (the background worker recovers the rest, so nothing is lost). A + /// session's cost grows with its capture window (`max_window`/`max_bytes`). + /// Repeated identical crashes are queued once while the first is undelivered, + /// and recovered crashes stop being queued at + /// [`MAX_QUEUED_REPORTS`](bugsee_core::queue::MAX_QUEUED_REPORTS) reports or + /// [`MAX_QUEUED_BYTES`](bugsee_core::queue::MAX_QUEUED_BYTES) bytes. + /// `Duration::ZERO` disables the pass: everything is recovered in the + /// background. Recovered crashes are queued newest first, so they may upload + /// in that order; the server deduplicates by signature whatever the order. pub fn launch_recovery_budget(mut self, budget: Duration) -> Self { self.launch_recovery_budget = budget; self diff --git a/crates/bugsee/tests/launch_recovery.rs b/crates/bugsee/tests/launch_recovery.rs new file mode 100644 index 0000000..805ad56 --- /dev/null +++ b/crates/bugsee/tests/launch_recovery.rs @@ -0,0 +1,56 @@ +// +// launch_recovery.rs +// bugsee +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! Issue #10 through the facade: the previous run's crash is on disk in the queue +//! by the time `launch_with` returns, even if the transport never delivers. + +use std::path::PathBuf; +use std::sync::Arc; + +use bugsee::core::transport::{Transport, TransportError}; +use bugsee::{Bugsee, LaunchOptions}; + +struct Offline; +impl Transport for Offline { + fn register_session(&self, _: &str, _: &[u8]) -> Result { + Err(TransportError::Transient("offline".into())) + } + fn create_issue(&self, _: &str, _: Option<&str>, _: &[u8]) -> Result { + Err(TransportError::Transient("offline".into())) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), TransportError> { + Err(TransportError::Transient("offline".into())) + } +} + +#[test] +fn launch_with_queues_the_previous_crash_before_returning() { + let dir: PathBuf = + std::env::temp_dir().join(format!("bugsee-fl-{}", bugsee::core::util::random_hex(8))); + let sessions = dir.join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + // A crashed earlier session: marker naming no live owner, and its parts. + std::fs::write(sessions.join("1.alive"), "0").unwrap(); + std::fs::create_dir_all(dir.join("parts").join("1")).unwrap(); + std::fs::write(dir.join("gen"), "1").unwrap(); + + let guard = Bugsee::launch_with( + LaunchOptions::new("T") + .data_dir(&dir) + .with_transport(Arc::new(Offline)), + ) + .unwrap(); + let queued = std::fs::read_dir(dir.join("queue")) + .map(|d| { + d.flatten() + .filter(|e| e.path().to_string_lossy().ends_with(".bundle.zip")) + .count() + }) + .unwrap_or(0); + assert_eq!(queued, 1, "no flush, no waiting"); + drop(guard); + let _ = std::fs::remove_dir_all(&dir); +} From 1760cf2a9ced93a82960a1c7c3d8f9345af160bd Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:20:01 +0500 Subject: [PATCH 22/52] fix(native): harden in-flight crash flush (review feedback, #13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - One process-wide wake pipe/event and ONE helper thread, created once and re-created in a fork child; the flush target is swapped under a fork-aware lock (last registered wins). Handles are never closed, so no descriptor-reuse race and no SIGPIPE for C hosts; shared ticket counters are now coherent. - Dying flag raised by the crash handler before waking the helper: claims and recoveries start nothing in the window, so a report is never half-delivered (removes the 5 s UNOWNED_GRACE stall in fork_safety; test-helper workaround reverted). - Skip the wait when the crashing thread is the worker or helper. - Handler waits with poll (Unix) / ack event + QPC (Windows) instead of nanosleep / Sleep(1); pipe2(O_CLOEXEC) on Linux/Android; parent's fds closed in a fork child; unpublish-then-republish ordering. - Panic hook shares DEFAULT_BUDGET; docs cover host-caught panics, ordering, Windows filter order and last-registered-wins. - crash_inflight runs on Windows too (cfg any(unix, windows)); a hidden budget setter lets the burst test use a long budget deterministically, a small burst covers the real 50 ms budget, and the wedged test now requires the handler to wait the budget and no longer (all three fail without the feature). πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/claim.rs | 7 + crates/bugsee-core/src/crash_flush.rs | 518 ++++++++++++-------- crates/bugsee-core/src/runtime.rs | 6 + crates/bugsee-core/tests/crash_flush_e2e.rs | 156 +++++- crates/bugsee-native/src/lib.rs | 35 +- crates/bugsee-panic/src/lib.rs | 10 +- crates/bugsee/tests/crash_inflight.rs | 67 ++- crates/bugsee/tests/fork_safety.rs | 7 +- 9 files changed, 563 insertions(+), 245 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 9775b05..a62e087 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -274,7 +274,7 @@ That last case previously produced **no report whatsoever** β€” the SDK shut dow ## 12. Crash-time safety, best-effort flush & edge cases -**Best-effort in-flight flush (implemented, `bugsee_core::crash_flush`):** a producer only queues; the worker appends later, so entries still in the channel (or mid-append) when the process dies are lost. At crash the handler first secures the guaranteed artifact (marker + frames), then does one atomic ticket increment, one non-blocking `write(2)` (Windows: `SetEvent`) to a pre-created wake handle, and polls an atomic ack with `nanosleep` for at most 50 ms (`INFLIGHT_FLUSH_BUDGET`). A helper thread created at launch (never in the handler) is blocked on that handle; it posts a `Flush` message through the normal FIFO channel and, when the worker acks, publishes the ticket. FIFO order means an ack implies every entry queued before the crash is appended. The crashing thread never takes the queue lock, allocates or serializes, and a stuck/dead worker or poisoned allocator costs exactly the budget (the 5 s watchdog is not involved). The pid recorded at registration makes a fork child skip the flush until its revived recorder re-registers. Panic: the hook runs the same bounded flush for a main-thread panic no guard will catch (covers `panic = "abort"` builds with no native handler; with one, the SIGABRT handler flushes too); an unwinding panic is also drained by the launch guard's `Drop`. Native fault: flushed from every fatal exit of the signal handler / the Windows exception filter, after the marker is complete. **Tradeoffs:** zero hot-path cost and no new on-disk format (no spill file: a torn append is the already-tolerated torn part tail, pinned by a recovery test), but the flush needs a live worker and allocator, so a fault that corrupts the heap or hits the worker thread forfeits it, and what is drained is bounded by throughput (about 3.5k ~450-byte events in 50 ms on a debug build; the remainder is lost). A producer-side pre-serialized ring spilled by the handler would survive a dead worker but taxes every event; not built. +**Best-effort in-flight flush (implemented, `bugsee_core::crash_flush`):** a producer only queues; the worker appends later, so entries still in the channel (or mid-append) when the process dies are lost. At crash the handler first secures the guaranteed artifact (marker + frames), then does one atomic ticket increment, one non-blocking `write(2)` (Windows: `SetEvent`) to a process-wide wake handle (one pipe/event per process, never closed, so no descriptor-reuse race), and polls an atomic ack for at most 50 ms (`crash_flush::DEFAULT_BUDGET`, shared with the panic hook). It first raises a `dying` flag so the uploader and recovery start no new claims in the window (a kill mid-claim would strand it for the claim grace period). One helper thread per process (created at launch, re-created in a fork child, never in the handler) is blocked on that handle; it posts a `Flush` message through the normal FIFO channel and, when the worker acks, publishes the ticket. FIFO order means an ack implies every entry queued before the crash is appended. The crashing thread never takes the queue lock, allocates or serializes, and a stuck/dead worker or poisoned allocator costs exactly the budget (the 5 s watchdog is not involved). The pid recorded at registration makes a fork child skip the flush until its revived recorder re-registers. A fault on the worker or helper thread skips the wait; the last registered recorder is the flush target. Panic: the hook runs the same bounded flush for a main-thread panic no guard will catch (covers `panic = "abort"` builds with no native handler; with one, the SIGABRT handler flushes too); an unwinding panic is also drained by the launch guard's `Drop`. Native fault: flushed from every fatal exit of the signal handler / the Windows exception filter, after the marker is complete. **Tradeoffs:** zero hot-path cost and no new on-disk format (no spill file: a torn append is the already-tolerated torn part tail, pinned by a recovery test), but the flush needs a live worker and allocator, so a fault that corrupts the heap or hits the worker thread forfeits it, and what is drained is bounded by throughput (about 3.5k ~450-byte events in 50 ms on a debug build; the remainder is lost). A producer-side pre-serialized ring spilled by the handler would survive a dead worker but taxes every event; not built. **Recursion guard:** `NORMALβ†’PANICKINGβ†’HANDLING_FATALβ†’SECONDARY`; second fatal β†’ tiny marker + terminate, no re-entry. **Quarantine:** Class A (continue) / B (restart) / C (rebuild instance) / D (disable); restart budget (1 immediate, 3/10 min, then disable) β€” never infinite. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index 341f1e0..fdfe27b 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -40,6 +40,10 @@ impl Drop for Claim { /// Try to claim `path`. `None` if another live process (or another thread of this /// one) holds it, or if the claim file cannot be created at all. pub fn try_claim(path: &Path) -> Option { + // A dying process must not start work a kill would leave half-claimed. + if crate::crash_flush::is_dying() { + return None; + } match claim_state(path) { ClaimState::Held(claim) => Some(claim), ClaimState::Taken | ClaimState::Unavailable => None, @@ -53,6 +57,9 @@ pub fn try_claim(path: &Path) -> Option { /// may both do the work, and the backend deduplicates. `None` means another /// process holds the claim. pub fn try_claim_lenient(path: &Path) -> Option> { + if crate::crash_flush::is_dying() { + return None; + } match claim_state(path) { ClaimState::Held(claim) => Some(Some(claim)), ClaimState::Unavailable => Some(None), diff --git a/crates/bugsee-core/src/crash_flush.rs b/crates/bugsee-core/src/crash_flush.rs index a2a37f6..e639433 100644 --- a/crates/bugsee-core/src/crash_flush.rs +++ b/crates/bugsee-core/src/crash_flush.rs @@ -13,167 +13,271 @@ //! worker's channel and returns. The worker serializes and appends it to the //! part file a little later. Anything still in the channel (or in the worker's //! hands) when the process dies is gone, and those are exactly the last -//! breadcrumbs, log lines and network events before the fault, the ones a crash -//! report is read for. The window is normally microseconds, but it grows to -//! many milliseconds under a burst, or when the worker is descheduled. +//! breadcrumbs, log lines and network events before the fault. The window is +//! microseconds when idle, but grows to many milliseconds under a burst or when +//! the worker is descheduled. //! //! # What this does //! //! A fatal-crash handler cannot take the channel lock, allocate, or serialize //! (the faulting thread may hold the allocator or the lock). So it does not //! touch the queue at all. It performs one `write(2)` / `SetEvent` on a -//! pre-created wake handle and then **polls an atomic** for a bounded time: +//! process-wide wake handle created at first launch, then waits for an atomic +//! acknowledgement for a bounded time: //! -//! 1. a dedicated, ordinary thread (created at launch, long before any crash) -//! is blocked reading that handle; -//! 2. woken, it posts a `Flush` marker to the capture worker through the -//! normal channel. The channel is FIFO, so by the time the worker -//! acknowledges the marker every entry queued before the crash has been -//! serialized and appended to its part file; +//! 1. one helper thread per process (created at launch, long before any crash; +//! re-created in a `fork()` child) is blocked on that handle; +//! 2. woken, it calls the registered flush target, which posts a `Flush` marker +//! to the capture worker through the normal channel. The channel is FIFO, so +//! when the worker acknowledges the marker every entry queued before the +//! crash has been serialized and appended; //! 3. the helper publishes the acknowledgement in an atomic, which the waiting -//! crash handler observes and then proceeds to terminate. +//! handler observes before letting the process die. //! -//! The crash handler's side ([`flush_in_flight`]) uses only an atomic load and -//! `fetch_add`, `getpid`, one non-blocking `write`, `clock_gettime` and -//! `nanosleep` (Windows: `SetEvent`, `Sleep`, `GetTickCount64`) β€” all -//! async-signal-safe. There is no allocation and no lock, and the wait is capped -//! by the caller's budget, so a stuck or dead worker (or a corrupted heap that -//! blocks the helper in the allocator) costs the budget and no more. It is a -//! best effort: when the budget runs out we give up and the process dies -//! exactly as it would have without this module. +//! The handler's side ([`flush_for_crash`]) uses only atomics, `getpid`, one +//! non-blocking `write`, `clock_gettime` and `poll` (Windows: `SetEvent`, +//! `WaitForSingleObject`, `QueryPerformanceCounter`), all async-signal-safe: no +//! allocation, no lock. The wait is capped by the caller's budget, so a stuck or +//! dead worker (or a corrupted heap that blocks the helper in the allocator) +//! costs the budget and no more. //! -//! The expensive work (JSON, file I/O) runs on the worker, with the process -//! otherwise healthy; the crashing thread is only parked. Other threads keep -//! running while a *synchronous* fault is handled, which is what makes -//! this possible, and also why events emitted by other threads in the meantime -//! may also land. +//! The wake handles are **never closed** (one pipe or event pair per process, +//! not per recorder), so a handler racing a recorder's shutdown can never write +//! to a recycled descriptor, and the read end outlives every writer, so a C host +//! (through `bugsee-ffi`) never sees `SIGPIPE`. +//! +//! # The dying flag +//! +//! [`flush_for_crash`] first raises a process-wide flag. During the (up to +//! budget) window the process is still running other threads, and an uploader +//! that started a report now would be killed halfway through a claim. While the +//! flag is up the runtime starts no claims and no recoveries ([`is_dying`]). //! //! # Limits (deliberate, documented tradeoffs) //! //! * Costs nothing on the happy path (no extra copy or serialization on the //! producer) but depends on the worker and allocator being alive. A fault that //! corrupts the heap, or kills the worker thread itself, forfeits the flush. +//! A fault on the worker or the helper thread skips the wait entirely. //! * Entries are flushed to the OS page cache like every other append; this //! survives the process dying, not the machine. +//! * **Last registered wins.** The flush target is process-wide: a second +//! `Recorder` launched in the same process takes over, and dropping a recorder +//! that is not the current target does nothing. //! * A fork child has none of the parent's threads: the pid recorded at -//! registration lets the handler skip the flush there until the child's -//! revived recorder registers again. - -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +//! registration makes the handler skip the flush there until the child's +//! revived recorder registers again (which re-creates the pipe and helper). +//! * **Windows ordering:** the flush runs inside the unhandled-exception filter, +//! before the host's previously installed filter (if any) is called, so a host +//! reporter that runs afterwards sees the process as it was. +//! * A panic hook that calls [`flush_in_flight`] for a main-thread panic a host +//! goes on to catch outside our guards pays up to the budget once for it. + +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::JoinHandle; use std::time::Duration; +use crate::fork::ForkMutex; + +/// How long a dying process waits for the capture worker by default. +pub const DEFAULT_BUDGET: Duration = Duration::from_millis(50); + /// Flush requests issued by crash handlers (a ticket counter). static REQUESTED: AtomicU64 = AtomicU64::new(0); /// Highest ticket the helper has seen acknowledged by the worker. static ACKED: AtomicU64 = AtomicU64::new(0); +/// Raised by a fatal-crash handler: the process is about to die. +static DYING: AtomicBool = AtomicBool::new(false); + +/// Threads whose own crash must not wait for the flush (the worker and helper: +/// they cannot answer while handling a fault). +const PIPELINE_SLOTS: usize = 8; +static PIPELINE_THREADS: [AtomicUsize; PIPELINE_SLOTS] = + [const { AtomicUsize::new(0) }; PIPELINE_SLOTS]; + +type Target = Arc bool + Send + Sync>; + +struct State { + target: Option<(u64, Target)>, + next_id: u64, + /// Process that created the helper and pipe (`0`: none yet). A different + /// value means we are in a `fork()` child, where they do not exist. + helper_pid: u32, +} -/// Owns the helper thread and the wake handle for one capture pipeline. -/// Dropping it deregisters (if it is still the registered one) and stops the -/// thread. +static STATE: ForkMutex = ForkMutex::new(|| State { + target: None, + next_id: 1, + helper_pid: 0, +}); + +/// The registered flush target. Dropping it deregisters (if still current). pub struct CrashFlush { - stop: Arc, - thread: Option>, - handle: imp::Handle, + id: u64, } impl CrashFlush { - /// Start the helper thread and make this the process's registered flusher. + /// Make `flush` the process's crash-time flush target, starting the helper + /// thread and wake handle if this process has none yet. /// /// `flush` is called on the helper thread each time a crash handler asks, and /// must return `true` once everything queued before the call is on disk. It /// must bound its own waiting. - pub fn install(flush: impl Fn() -> bool + Send + 'static) -> std::io::Result { - let stop = Arc::new(AtomicBool::new(false)); - let (handle, waiter) = imp::create()?; - let thread_stop = Arc::clone(&stop); - let thread = std::thread::Builder::new() - .name("bugsee-crash-flush".into()) - .spawn(move || { - // `waiter` is closed by its own Drop when the loop ends. - while imp::wait(&waiter) { - if thread_stop.load(Ordering::Acquire) { - break; - } - let target = REQUESTED.load(Ordering::Acquire); - if target <= ACKED.load(Ordering::Acquire) { - continue; - } - let ok = std::panic::catch_unwind(std::panic::AssertUnwindSafe(&flush)) - .unwrap_or(false); - if ok { - ACKED.fetch_max(target, Ordering::AcqRel); - } - } - })?; - imp::register(&handle); - Ok(CrashFlush { - stop, - thread: Some(thread), - handle, - }) + pub fn install( + flush: impl Fn() -> bool + Send + Sync + 'static, + ) -> std::io::Result { + let mut st = STATE.lock(); + let pid = std::process::id(); + if st.helper_pid != pid { + imp::start_helper()?; + st.helper_pid = pid; + } + let id = st.next_id; + st.next_id += 1; + st.target = Some((id, Arc::new(flush))); + Ok(CrashFlush { id }) } } impl Drop for CrashFlush { fn drop(&mut self) { - imp::deregister(&self.handle); - self.stop.store(true, Ordering::Release); - imp::wake(&self.handle); - if let Some(thread) = self.thread.take() { - // The helper polls `stop` only between flushes; a flush is bounded by - // the closure's own timeout, so this join is too. - let _ = thread.join(); + let mut st = STATE.lock(); + if st.target.as_ref().is_some_and(|(id, _)| *id == self.id) { + st.target = None; + } + } +} + +/// The current flush target, for the helper thread. +fn current_target() -> Option { + STATE.lock().target.as_ref().map(|(_, t)| Arc::clone(t)) +} + +/// Marks the current thread as part of the capture pipeline for its lifetime. +pub struct PipelineThread(usize); + +/// Record that the calling thread is a pipeline thread (see [`PIPELINE_SLOTS`]). +pub fn register_pipeline_thread() -> PipelineThread { + let me = imp::thread_id(); + for slot in &PIPELINE_THREADS { + if slot + .compare_exchange(0, me, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + return PipelineThread(me); + } + } + PipelineThread(0) +} + +impl Drop for PipelineThread { + fn drop(&mut self) { + if self.0 == 0 { + return; + } + for slot in &PIPELINE_THREADS { + let _ = slot.compare_exchange(self.0, 0, Ordering::AcqRel, Ordering::Acquire); } - imp::close(&mut self.handle); } } -/// Ask the registered capture pipeline to drain everything queued so far, and -/// wait at most `budget` for it to finish. Returns whether it confirmed. +fn on_pipeline_thread() -> bool { + let me = imp::thread_id(); + PIPELINE_THREADS + .iter() + .any(|s| s.load(Ordering::Acquire) == me) +} + +/// Has a fatal-crash handler declared the process about to die? The runtime +/// starts no report claims or recoveries while this is set. +pub fn is_dying() -> bool { + DYING.load(Ordering::Acquire) +} + +/// Clear the dying flag. Tests only. +#[doc(hidden)] +pub fn reset_dying_for_test() { + DYING.store(false, Ordering::Release); +} + +/// For a fatal-crash handler: declare the process dying, then ask the registered +/// capture pipeline to drain everything queued so far, waiting at most `budget`. +/// Returns whether it confirmed. /// /// **Async-signal-safe**: callable from a signal handler or exception filter. -/// No allocation, no locks, bounded time. Returns `false` immediately when no -/// pipeline is registered or the registration belongs to another process (a -/// `fork()` child). +/// No allocation, no locks, bounded time. +pub fn flush_for_crash(budget: Duration) -> bool { + DYING.store(true, Ordering::Release); + flush_in_flight(budget) +} + +/// Like [`flush_for_crash`] without declaring the process dying: for a caller +/// that only *expects* to die (a main-thread panic hook). Returns immediately +/// when no pipeline is registered, the registration belongs to another process +/// (a `fork()` child), or the caller is itself a pipeline thread. pub fn flush_in_flight(budget: Duration) -> bool { + if on_pipeline_thread() { + return false; + } imp::request_and_wait(budget) } +/// Drain: bump the ack to `target` after a successful flush. +fn ack(target: u64) { + ACKED.fetch_max(target, Ordering::AcqRel); +} + +/// What the helper does each time it is woken. +fn serve_request() { + let target = REQUESTED.load(Ordering::Acquire); + if target <= ACKED.load(Ordering::Acquire) { + return; + } + let ok = match current_target() { + Some(flush) => { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| flush())).unwrap_or(false) + } + // Nothing registered: nothing is queued anywhere that we could drain. + None => true, + }; + if ok { + ack(target); + imp::signal_ack(); + } +} + #[cfg(unix)] mod imp { - use super::{ACKED, REQUESTED}; + use super::{serve_request, ACKED, PIPELINE_THREADS, REQUESTED}; use std::sync::atomic::{AtomicI32, Ordering}; use std::time::Duration; - /// Write end of the wake pipe the handler writes to (`-1` when none). + /// Write end of the wake pipe (`-1` when none). Never closed once set. static WAKE_FD: AtomicI32 = AtomicI32::new(-1); - /// Pid that created the registration (a fork child must not use it). + /// Read end, kept only so a fork child can close the parent's copy. + static READ_FD: AtomicI32 = AtomicI32::new(-1); + /// Pid that owns `WAKE_FD` (a fork child must not use it). static OWNER_PID: AtomicI32 = AtomicI32::new(0); - pub struct Handle { - write_fd: i32, - } - - /// The read end, closed when the helper thread drops it. - pub struct Waiter(i32); - - impl Drop for Waiter { - fn drop(&mut self) { - // SAFETY: closing a descriptor this value owns. - unsafe { libc::close(self.0) }; - } + pub fn thread_id() -> usize { + // SAFETY: pthread_self is async-signal-safe and has no preconditions. + unsafe { libc::pthread_self() as usize } } - pub fn create() -> std::io::Result<(Handle, Waiter)> { + fn make_pipe() -> std::io::Result<(i32, i32)> { let mut fds = [0i32; 2]; + #[cfg(any(target_os = "linux", target_os = "android"))] // SAFETY: `fds` is a valid two-element array. - if unsafe { libc::pipe(fds.as_mut_ptr()) } != 0 { + let rc = unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC) }; + #[cfg(not(any(target_os = "linux", target_os = "android")))] + // SAFETY: as above. + let rc = unsafe { libc::pipe(fds.as_mut_ptr()) }; + if rc != 0 { return Err(std::io::Error::last_os_error()); } // SAFETY: both descriptors were just created and are ours. unsafe { + #[cfg(not(any(target_os = "linux", target_os = "android")))] for fd in fds { libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC); } @@ -181,49 +285,68 @@ mod imp { let fl = libc::fcntl(fds[1], libc::F_GETFL); libc::fcntl(fds[1], libc::F_SETFL, fl | libc::O_NONBLOCK); } - Ok((Handle { write_fd: fds[1] }, Waiter(fds[0]))) - } - - /// Block until woken. `false` means the pipe is gone: stop. - pub fn wait(w: &Waiter) -> bool { - let mut buf = [0u8; 64]; - loop { - // SAFETY: reads into a valid buffer. - let n = unsafe { libc::read(w.0, buf.as_mut_ptr().cast(), buf.len()) }; - if n > 0 { - return true; - } - if n < 0 && std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted { - continue; - } - return false; - } + Ok((fds[0], fds[1])) } - pub fn register(h: &Handle) { + /// Create the pipe and the one helper thread for this process. In a fork + /// child this replaces the parent's (which are closed here, after the new + /// ones are published). + pub fn start_helper() -> std::io::Result<()> { + // Unpublish first, so a handler that runs mid-switch sees nothing. + let old_w = WAKE_FD.swap(-1, Ordering::AcqRel); + let old_r = READ_FD.swap(-1, Ordering::AcqRel); // SAFETY: getpid has no preconditions. OWNER_PID.store(unsafe { libc::getpid() }, Ordering::Release); - WAKE_FD.store(h.write_fd, Ordering::Release); - } - - pub fn deregister(h: &Handle) { - let _ = WAKE_FD.compare_exchange(h.write_fd, -1, Ordering::AcqRel, Ordering::Acquire); - } - - pub fn wake(h: &Handle) { - let b = 1u8; - // SAFETY: writes one byte from a valid buffer; non-blocking. - unsafe { libc::write(h.write_fd, (&b as *const u8).cast(), 1) }; - } + for slot in &PIPELINE_THREADS { + slot.store(0, Ordering::Release); + } + // Tickets of the parent's requests are meaningless here. + ACKED.store(REQUESTED.load(Ordering::Acquire), Ordering::Release); - pub fn close(h: &mut Handle) { - if h.write_fd >= 0 { - // SAFETY: closing the descriptor this handle owns, once. - unsafe { libc::close(h.write_fd) }; - h.write_fd = -1; + let (r, w) = make_pipe()?; + let spawned = std::thread::Builder::new() + .name("bugsee-crash-flush".into()) + .spawn(move || { + let _me = super::register_pipeline_thread(); + let mut buf = [0u8; 64]; + loop { + // SAFETY: reads into a valid buffer from a descriptor that is + // never closed while this thread runs. + let n = unsafe { libc::read(r, buf.as_mut_ptr().cast(), buf.len()) }; + if n > 0 { + serve_request(); + } else if n < 0 + && std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted + { + continue; + } else { + break; + } + } + }); + if let Err(e) = spawned { + // SAFETY: closing descriptors made above and shared with nobody. + unsafe { + libc::close(r); + libc::close(w); + } + return Err(e); } + READ_FD.store(r, Ordering::Release); + WAKE_FD.store(w, Ordering::Release); + // The parent's copies are of no use here. + for fd in [old_w, old_r] { + if fd >= 0 { + // SAFETY: inherited descriptors only this child's code could use, + // unpublished above. + unsafe { libc::close(fd) }; + } + } + Ok(()) } + pub fn signal_ack() {} + fn now_ns() -> u64 { let mut ts = libc::timespec { tv_sec: 0, @@ -252,12 +375,8 @@ mod imp { if now_ns() >= deadline { return false; } - let ts = libc::timespec { - tv_sec: 0, - tv_nsec: 200_000, - }; - // SAFETY: nanosleep is async-signal-safe. - unsafe { libc::nanosleep(&ts, core::ptr::null_mut()) }; + // SAFETY: poll with no descriptors is a plain async-signal-safe sleep. + unsafe { libc::poll(core::ptr::null_mut(), 0, 1) }; } true } @@ -265,7 +384,7 @@ mod imp { #[cfg(windows)] mod imp { - use super::{ACKED, REQUESTED}; + use super::{serve_request, ACKED, REQUESTED}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; @@ -275,70 +394,89 @@ mod imp { fn CreateEventW(attrs: *const u8, manual: i32, initial: i32, name: *const u16) -> Hnd; fn SetEvent(h: Hnd) -> i32; fn WaitForSingleObject(h: Hnd, ms: u32) -> u32; - fn CloseHandle(h: Hnd) -> i32; - fn Sleep(ms: u32); - fn GetTickCount64() -> u64; + fn GetCurrentThreadId() -> u32; + fn QueryPerformanceCounter(v: *mut i64) -> i32; + fn QueryPerformanceFrequency(v: *mut i64) -> i32; } - /// The auto-reset event the handler signals (`0` when none). + /// Auto-reset events, never closed: the handler signals `WAKE`, the helper + /// signals `ACK`. (`0` when none.) static WAKE: AtomicUsize = AtomicUsize::new(0); + static ACK: AtomicUsize = AtomicUsize::new(0); - pub struct Handle(Hnd); - pub struct Waiter(Hnd); + pub fn thread_id() -> usize { + // SAFETY: plain Win32 call. + unsafe { GetCurrentThreadId() as usize } + } - pub fn create() -> std::io::Result<(Handle, Waiter)> { - // SAFETY: plain Win32 call; auto-reset, initially unsignalled, unnamed. - let h = unsafe { CreateEventW(core::ptr::null(), 0, 0, core::ptr::null()) }; - if h == 0 { + pub fn start_helper() -> std::io::Result<()> { + // SAFETY: plain Win32 calls; auto-reset, initially unsignalled, unnamed. + let (wake, ack) = unsafe { + ( + CreateEventW(core::ptr::null(), 0, 0, core::ptr::null()), + CreateEventW(core::ptr::null(), 0, 0, core::ptr::null()), + ) + }; + if wake == 0 || ack == 0 { return Err(std::io::Error::last_os_error()); } - Ok((Handle(h), Waiter(h))) - } - - /// Block until signalled (`WAIT_OBJECT_0`). - pub fn wait(w: &Waiter) -> bool { - // SAFETY: waits on an event handle that outlives the thread (it is closed - // only after the thread is joined). - unsafe { WaitForSingleObject(w.0, u32::MAX) == 0 } - } - - pub fn register(h: &Handle) { - WAKE.store(h.0, Ordering::Release); - } - - pub fn deregister(h: &Handle) { - let _ = WAKE.compare_exchange(h.0, 0, Ordering::AcqRel, Ordering::Acquire); + std::thread::Builder::new() + .name("bugsee-crash-flush".into()) + .spawn(move || { + let _me = super::register_pipeline_thread(); + loop { + // SAFETY: waits on an event that is never closed. + if unsafe { WaitForSingleObject(wake, u32::MAX) } != 0 { + break; + } + serve_request(); + } + })?; + ACK.store(ack, Ordering::Release); + WAKE.store(wake, Ordering::Release); + Ok(()) } - pub fn wake(h: &Handle) { - // SAFETY: signals a live event. - unsafe { SetEvent(h.0) }; + pub fn signal_ack() { + let h = ACK.load(Ordering::Acquire); + if h != 0 { + // SAFETY: signals a live event. + unsafe { SetEvent(h) }; + } } - pub fn close(h: &mut Handle) { - if h.0 != 0 { - // SAFETY: closes the handle this value owns, once. - unsafe { CloseHandle(h.0) }; - h.0 = 0; + fn now_ticks() -> (i64, i64) { + let (mut t, mut f) = (0i64, 1i64); + // SAFETY: valid out-pointers; both calls cannot fail on supported Windows. + unsafe { + QueryPerformanceCounter(&mut t); + QueryPerformanceFrequency(&mut f); } + (t, f.max(1)) } pub fn request_and_wait(budget: Duration) -> bool { - let h = WAKE.load(Ordering::Acquire); - if h == 0 { + let wake = WAKE.load(Ordering::Acquire); + let ack = ACK.load(Ordering::Acquire); + if wake == 0 || ack == 0 { return false; } let ticket = REQUESTED.fetch_add(1, Ordering::AcqRel) + 1; - // SAFETY: SetEvent/Sleep/GetTickCount64 take no locks we could be holding. - unsafe { - SetEvent(h); - let deadline = GetTickCount64().saturating_add(budget.as_millis() as u64); - while ACKED.load(Ordering::Acquire) < ticket { - if GetTickCount64() >= deadline { - return false; - } - Sleep(1); + let (start, freq) = now_ticks(); + let budget_ticks = (budget.as_secs_f64() * freq as f64) as i64; + // SAFETY: SetEvent/WaitForSingleObject take no locks we could be holding. + unsafe { SetEvent(wake) }; + while ACKED.load(Ordering::Acquire) < ticket { + let (now, _) = now_ticks(); + let left = budget_ticks - (now - start); + if left <= 0 { + return false; } + // Wake on the helper's ack; the slice cap covers another waiter having + // consumed the (auto-reset) signal. + let ms = ((left as f64 / freq as f64 * 1000.0).ceil() as u32).clamp(1, 5); + // SAFETY: as above. + unsafe { WaitForSingleObject(ack, ms) }; } true } @@ -348,19 +486,13 @@ mod imp { mod imp { use std::time::Duration; - pub struct Handle; - pub struct Waiter; - - pub fn create() -> std::io::Result<(Handle, Waiter)> { - Err(std::io::Error::from(std::io::ErrorKind::Unsupported)) + pub fn thread_id() -> usize { + 0 } - pub fn wait(_: &Waiter) -> bool { - false + pub fn start_helper() -> std::io::Result<()> { + Err(std::io::Error::from(std::io::ErrorKind::Unsupported)) } - pub fn register(_: &Handle) {} - pub fn deregister(_: &Handle) {} - pub fn wake(_: &Handle) {} - pub fn close(_: &mut Handle) {} + pub fn signal_ack() {} pub fn request_and_wait(_: Duration) -> bool { false } diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 391d1f2..1f562c4 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -908,6 +908,8 @@ fn spawn_pipeline( let worker = std::thread::Builder::new() .name("bugsee-capture".into()) .spawn(move || { + // A crash on this thread cannot wait for itself to flush. + let _pipeline = crate::crash_flush::register_pipeline_thread(); // Queue any crashed prior session before capturing this one. // Contained: a panic here (e.g. a host before_send on a recovered // crash) must not abort the thread before worker_loop starts. @@ -1739,6 +1741,10 @@ fn drain_memory( if deadline.is_some_and(|d| Instant::now() >= d) { break; } + // A dying process starts no delivery it cannot finish (crash_flush). + if crate::crash_flush::is_dying() { + break; + } let now = epoch_ms(); let next = shared .mem_queue diff --git a/crates/bugsee-core/tests/crash_flush_e2e.rs b/crates/bugsee-core/tests/crash_flush_e2e.rs index c03abfc..5a5c6ae 100644 --- a/crates/bugsee-core/tests/crash_flush_e2e.rs +++ b/crates/bugsee-core/tests/crash_flush_e2e.rs @@ -135,48 +135,92 @@ fn a_stuck_worker_costs_only_the_budget() { drop(recorder); } +/// A flush target that only counts its calls. +fn counter() -> (Arc, impl Fn() -> bool + Send + Sync + 'static) { + let n = Arc::new(AtomicUsize::new(0)); + let c = Arc::clone(&n); + (n, move || { + c.fetch_add(1, Ordering::SeqCst); + true + }) +} + #[test] -fn dropping_the_recorder_deregisters() { +fn dropping_the_recorder_stops_flushing_into_it() { let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); let dir = TempDir::new(); + let (calls, f) = counter(); + let _sentinel = CrashFlush::install(f).unwrap(); let recorder = Recorder::launch( RecorderConfig::new(&dir.0, "TOKEN"), Arc::new(MockTransport::default()), ) .unwrap(); assert!(flush_in_flight(Duration::from_secs(2))); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "the recorder is the target" + ); drop(recorder); + // Nothing is registered now, so a request is answered at once rather than + // waited out, and the replaced sentinel is not called. let t = Instant::now(); - assert!(!flush_in_flight(Duration::from_secs(2))); + assert!(flush_in_flight(Duration::from_secs(2))); assert!(t.elapsed() < Duration::from_millis(200), "no wait at all"); + assert_eq!(calls.load(Ordering::SeqCst), 0); } -// The handshake itself, against stand-in flushers (no pipeline). - #[test] -fn unregistered_returns_false_immediately() { +fn memory_mode_does_not_take_over_the_target() { let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); - let t = Instant::now(); - assert!(!flush_in_flight(Duration::from_millis(500))); - assert!(t.elapsed() < Duration::from_millis(100)); + let dir = TempDir::new(); + let (calls, f) = counter(); + let _sentinel = CrashFlush::install(f).unwrap(); + let mut cfg = RecorderConfig::new(&dir.0, "TOKEN"); + cfg.memory_only = Some("test".into()); + let recorder = Recorder::launch(cfg, Arc::new(MockTransport::default())).unwrap(); + assert!(flush_in_flight(Duration::from_secs(2))); + assert_eq!(calls.load(Ordering::SeqCst), 1, "still the sentinel"); + drop(recorder); } +// The handshake itself, against stand-in flushers (no pipeline). + #[test] fn a_responsive_flusher_is_confirmed_promptly() { let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); - let calls = Arc::new(AtomicUsize::new(0)); - let c = Arc::clone(&calls); - let cf = CrashFlush::install(move || { - c.fetch_add(1, Ordering::SeqCst); - true - }) - .unwrap(); + let (calls, f) = counter(); + let cf = CrashFlush::install(f).unwrap(); let t = Instant::now(); assert!(flush_in_flight(Duration::from_millis(500))); assert!(t.elapsed() < Duration::from_millis(250)); assert_eq!(calls.load(Ordering::SeqCst), 1); drop(cf); - assert!(!flush_in_flight(Duration::from_millis(50)), "deregistered"); + assert!(flush_in_flight(Duration::from_millis(500))); + assert_eq!(calls.load(Ordering::SeqCst), 1, "deregistered: not called"); +} + +#[test] +fn the_last_registered_target_wins() { + let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let (a_calls, a) = counter(); + let (b_calls, b) = counter(); + let a = CrashFlush::install(a).unwrap(); + let b = CrashFlush::install(b).unwrap(); + assert!(flush_in_flight(Duration::from_millis(500))); + assert_eq!( + ( + a_calls.load(Ordering::SeqCst), + b_calls.load(Ordering::SeqCst) + ), + (0, 1) + ); + // Dropping the one that is not current must not deregister the current one. + drop(a); + assert!(flush_in_flight(Duration::from_millis(500))); + assert_eq!(b_calls.load(Ordering::SeqCst), 2); + drop(b); } #[test] @@ -198,6 +242,8 @@ fn a_stuck_flusher_costs_the_budget_and_no_more() { assert!(took < Duration::from_millis(400), "{took:?}"); release.store(true, Ordering::SeqCst); drop(cf); + // Let the helper finish the stuck call before the next test. + std::thread::sleep(Duration::from_millis(50)); } #[test] @@ -224,12 +270,78 @@ fn a_panicking_flusher_does_not_kill_the_helper() { } #[test] -fn memory_mode_has_nothing_to_flush() { +fn a_pipeline_thread_does_not_wait_for_itself() { + let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let (calls, f) = counter(); + let cf = CrashFlush::install(f).unwrap(); + let t = Instant::now(); + let r = std::thread::spawn(|| { + let _p = bugsee_core::crash_flush::register_pipeline_thread(); + flush_in_flight(Duration::from_secs(2)) + }) + .join() + .unwrap(); + assert!(!r, "skipped"); + assert!(t.elapsed() < Duration::from_millis(200)); + assert_eq!(calls.load(Ordering::SeqCst), 0); + drop(cf); +} + +#[test] +fn a_crash_flush_raises_the_dying_flag_and_blocks_new_claims() { let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); let dir = TempDir::new(); - let mut cfg = RecorderConfig::new(&dir.0, "TOKEN"); - cfg.memory_only = Some("test".into()); - let recorder = Recorder::launch(cfg, Arc::new(MockTransport::default())).unwrap(); - assert!(!flush_in_flight(Duration::from_millis(50))); - drop(recorder); + let claim = dir.0.join("x.claim"); + bugsee_core::crash_flush::reset_dying_for_test(); + assert!(bugsee_core::claim::try_claim(&claim).is_some()); + let (_calls, f) = counter(); + let cf = CrashFlush::install(f).unwrap(); + // The plain flush (a panic hook's) does not declare the process dying. + flush_in_flight(Duration::from_millis(100)); + assert!(!bugsee_core::crash_flush::is_dying()); + bugsee_core::crash_flush::flush_for_crash(Duration::from_millis(100)); + assert!(bugsee_core::crash_flush::is_dying()); + assert!( + bugsee_core::claim::try_claim(&claim).is_none(), + "a dying process starts no claim a kill would leave half-made" + ); + assert!(bugsee_core::claim::try_claim_lenient(&claim).is_none()); + bugsee_core::crash_flush::reset_dying_for_test(); + assert!(bugsee_core::claim::try_claim(&claim).is_some()); + drop(cf); +} + +/// A `fork()` child has none of the parent's threads: it must not wait on the +/// parent's helper, and must get its own once it registers. +#[cfg(unix)] +#[test] +fn a_fork_child_gets_its_own_helper() { + let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let (_calls, f) = counter(); + let cf = CrashFlush::install(f).unwrap(); + assert!(flush_in_flight(Duration::from_millis(500))); + // SAFETY: the child only calls the code under test and `_exit`s. + let pid = unsafe { libc::fork() }; + if pid == 0 { + let before = flush_in_flight(Duration::from_millis(500)); + let (_c, g) = counter(); + let mine = CrashFlush::install(g).unwrap(); + let after = flush_in_flight(Duration::from_millis(500)); + std::mem::forget(mine); + let code = if !before && after { 0 } else { 1 }; + // SAFETY: leaving a forked child. + unsafe { libc::_exit(code) }; + } + let mut status = 0; + // SAFETY: waiting for our own child. + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!( + libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, + "child status {status}" + ); + assert!( + flush_in_flight(Duration::from_millis(500)), + "parent unaffected" + ); + drop(cf); } diff --git a/crates/bugsee-native/src/lib.rs b/crates/bugsee-native/src/lib.rs index 56815e7..46e9ead 100644 --- a/crates/bugsee-native/src/lib.rs +++ b/crates/bugsee-native/src/lib.rs @@ -195,20 +195,37 @@ mod imp { const HANDLER_BUDGET_SECS: u32 = 5; /// How long a dying process waits for the capture worker to write out the - /// entries still queued when it faulted (see `bugsee_core::crash_flush`). - /// Small against [`HANDLER_BUDGET_SECS`] on purpose: it is a courtesy, never - /// something the process may be held for, and it runs only AFTER the marker is - /// complete, so giving up costs the last few events and nothing else. + /// entries still queued when it faulted (see `bugsee_core::crash_flush`), in + /// milliseconds. Small against [`HANDLER_BUDGET_SECS`] on purpose: it is a + /// courtesy, never something the process may be held for, and it runs only + /// AFTER the marker is complete, so giving up costs the last few events and + /// nothing else. #[cfg(any(unix, windows))] - const INFLIGHT_FLUSH_BUDGET: std::time::Duration = std::time::Duration::from_millis(50); + static INFLIGHT_FLUSH_BUDGET_MS: core::sync::atomic::AtomicU64 = + core::sync::atomic::AtomicU64::new( + bugsee_core::crash_flush::DEFAULT_BUDGET.as_millis() as u64 + ); - /// Best-effort drain of in-flight capture entries. Async-signal-safe (one - /// atomic, one non-blocking write, a bounded sleep loop); the heavy lifting - /// happens on the capture worker. Call only once the marker is on disk. + /// Override the in-flight flush budget. Tests only: they use a long one to + /// prove the mechanism independent of how fast the machine drains. + #[cfg(any(unix, windows))] + #[doc(hidden)] + pub fn set_inflight_flush_budget_for_test(budget: std::time::Duration) { + INFLIGHT_FLUSH_BUDGET_MS.store( + budget.as_millis() as u64, + core::sync::atomic::Ordering::SeqCst, + ); + } + + /// Best-effort drain of in-flight capture entries. Async-signal-safe (atomics, + /// one non-blocking write, a bounded poll loop); the heavy lifting happens on + /// the capture worker. Call only once the marker is on disk. Also raises the + /// dying flag, so the runtime starts no new claims in the window. #[cfg(any(unix, windows))] #[inline(never)] fn flush_inflight_events() { - let _ = bugsee_core::crash_flush::flush_in_flight(INFLIGHT_FLUSH_BUDGET); + let ms = INFLIGHT_FLUSH_BUDGET_MS.load(core::sync::atomic::Ordering::Relaxed); + let _ = bugsee_core::crash_flush::flush_for_crash(std::time::Duration::from_millis(ms)); } /// A `SIGALRM`-based deadline for the crash handler. Only async-signal-safe diff --git a/crates/bugsee-panic/src/lib.rs b/crates/bugsee-panic/src/lib.rs index c4ca1d1..728c277 100644 --- a/crates/bugsee-panic/src/lib.rs +++ b/crates/bugsee-panic/src/lib.rs @@ -253,8 +253,14 @@ pub fn install(reporter: Arc) { }); }); if likely_fatal { - let _ = - bugsee_core::crash_flush::flush_in_flight(std::time::Duration::from_millis(50)); + // Runs AFTER the inline report above (when `report_panics_from_hook` + // is on), so that report's processing is ahead of the flush marker + // in the worker's queue and is part of what is waited for. A panic + // the host goes on to catch outside our guards pays up to the + // budget once. The same budget the native handler uses. + let _ = bugsee_core::crash_flush::flush_in_flight( + bugsee_core::crash_flush::DEFAULT_BUDGET, + ); } // Preserve prior hook behavior (default abort message, other SDKs). previous(info); diff --git a/crates/bugsee/tests/crash_inflight.rs b/crates/bugsee/tests/crash_inflight.rs index 78a8c8d..18c206f 100644 --- a/crates/bugsee/tests/crash_inflight.rs +++ b/crates/bugsee/tests/crash_inflight.rs @@ -10,17 +10,22 @@ //! little later. These tests die for real, in a child process, immediately after //! emitting a burst of events, and read what the next launch recovers: //! -//! * `burst_then_segv` β€” every event emitted before the fault, including the -//! ones still queued when it hit, is in the recovered report. +//! * `burst_then_segv` β€” with a long flush budget (so the result does not depend +//! on how fast this machine drains), every event emitted before the fault, +//! including the ones still queued when it hit, is in the recovered report. +//! Without the flush only a few dozen of the 1000 survive. +//! * `small_burst_real_budget` β€” the same under the real 50 ms budget, with a +//! burst small enough to drain inside it. //! * `wedged_worker_then_segv` β€” the worker is stuck in a host callback, so the -//! flush cannot succeed. The process must still die promptly (the handler is -//! bounded; the 5 s watchdog must not be what ends it) and the crash is still -//! reported, with the events that had already reached disk. +//! flush cannot succeed. The process must wait the (test-shortened) budget and +//! no longer β€” proving the wait happens and is bounded; the 5 s watchdog must +//! not be what ends it β€” and the crash is still reported, with the events that +//! had already reached disk. //! //! `harness = false` (see `Cargo.toml`): the child must fault from the real main //! thread. -#![cfg(all(feature = "native", unix))] +#![cfg(all(feature = "native", any(unix, windows)))] use std::io::{BufRead, BufReader, Cursor, Read, Write}; use std::process::{Command, Stdio}; @@ -38,6 +43,13 @@ const DIR_ENV: &str = "BUGSEE_INFLIGHT_DIR"; /// before the worker (which serializes and writes) has drained, yet the worker /// can drain within the 50 ms handler budget. const BURST: usize = 1000; +/// The burst for the real-budget case: drains well inside 50 ms even on a slow +/// core, yet is far more than the ~20-40 the no-flush baseline recovers. +const SMALL_BURST: usize = 250; +/// Budget used by the cases that prove the mechanism rather than the speed. +const LONG_BUDGET: Duration = Duration::from_secs(5); +/// Budget used by the wedged case: long enough to measure, short enough to run. +const WEDGE_BUDGET: Duration = Duration::from_millis(300); #[inline(never)] fn fault_null() { @@ -45,9 +57,9 @@ fn fault_null() { unsafe { std::ptr::write_volatile(std::hint::black_box(std::ptr::null_mut::()), 1) } } -fn emit_burst(prefix: &str) { +fn emit_burst(prefix: &str, count: usize) { let pad = "x".repeat(400); - for i in 0..BURST { + for i in 0..count { let mut params = Map::new(); params.insert("pad".into(), Value::String(pad.clone())); Bugsee::event_with(format!("{prefix}-{i}"), params); @@ -77,7 +89,11 @@ fn run_child(kind: &str, dir: &str) -> ! { std::thread::sleep(Duration::from_millis(300)); match kind { - "burst_then_segv" => emit_burst("burst"), + "burst_then_segv" => { + bugsee_native::set_inflight_flush_budget_for_test(LONG_BUDGET); + emit_burst("burst", BURST); + } + "small_burst_real_budget" => emit_burst("small", SMALL_BURST), "wedged_worker_then_segv" => { // These reach disk before the worker is wedged... for i in 0..20 { @@ -88,7 +104,8 @@ fn run_child(kind: &str, dir: &str) -> ! { // is stuck in the queue. Bugsee::capture_message(LogLevel::Error, "wedge the worker"); std::thread::sleep(Duration::from_millis(300)); - emit_burst("stuck"); + bugsee_native::set_inflight_flush_budget_for_test(WEDGE_BUDGET); + emit_burst("stuck", 100); } other => panic!("unknown kind {other}"), } @@ -209,10 +226,35 @@ fn burst_then_segv() -> Result<(), String> { }) } +fn small_burst_real_budget() -> Result<(), String> { + with_dir("small", |dir| { + crash_child("small_burst_real_budget", dir)?; + let bundles = recover(dir); + if bundles.len() != 1 { + return Err(format!("expected one report, got {}", bundles.len())); + } + let names = event_names(&bundles[0]); + let have = names.iter().filter(|n| n.starts_with("small-")).count(); + let last = format!("small-{}", SMALL_BURST - 1); + if !names.contains(&last) || have != SMALL_BURST { + return Err(format!( + "events lost under the real budget ({have}/{SMALL_BURST}, last present: {})", + names.contains(&last) + )); + } + Ok(()) + }) +} + fn wedged_worker_then_segv() -> Result<(), String> { with_dir("wedged", |dir| { let took = crash_child("wedged_worker_then_segv", dir)?; - if took > Duration::from_secs(2) { + // Waited the budget (without the feature the process dies at once) and no + // longer (the 5 s watchdog must not be what ended it). + if took < WEDGE_BUDGET - Duration::from_millis(50) { + return Err(format!("died in {took:?}: the flush did not wait")); + } + if took > Duration::from_secs(1) { return Err(format!("the dying process hung for {took:?}")); } let bundles = recover(dir); @@ -239,8 +281,9 @@ fn main() { run_child(&kind, &std::env::var(DIR_ENV).expect("dir")); } let filter = std::env::args().skip(1).find(|a| !a.starts_with('-')); - let cases: [(&str, Case); 2] = [ + let cases: [(&str, Case); 3] = [ ("burst_then_segv", burst_then_segv), + ("small_burst_real_budget", small_burst_real_budget), ("wedged_worker_then_segv", wedged_worker_then_segv), ]; let mut failed = Vec::new(); diff --git a/crates/bugsee/tests/fork_safety.rs b/crates/bugsee/tests/fork_safety.rs index e72ef5a..a27b0f5 100644 --- a/crates/bugsee/tests/fork_safety.rs +++ b/crates/bugsee/tests/fork_safety.rs @@ -64,12 +64,7 @@ impl Transport for FileTransport { fn upload_bundle(&self, _: &str, zip: &[u8]) -> Result<(), TransportError> { let n = self.n.fetch_add(1, Ordering::SeqCst); let name = format!("{}-{n}.zip", std::process::id()); - // Write-then-rename: a process killed mid-upload (a crashing worker now - // lingers briefly to flush in-flight events, so one can die with an upload - // under way) must not leave a half-written `.zip` for the test to read. - let tmp = self.dir.join(format!(".{name}.part")); - std::fs::write(&tmp, zip) - .and_then(|()| std::fs::rename(&tmp, self.dir.join(name))) + std::fs::write(self.dir.join(name), zip) .map_err(|e| TransportError::Transient(e.to_string())) } } From ee3f8080c5a36501c498f27840d10a629409e954 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:20:37 +0500 Subject: [PATCH 23/52] fix(core): review round 3 on lock claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A leftover .lock that cannot be opened (0444, read-only volume) or whose lock call errors probes as Unknown, not Held, and the probe opens read-only, so it can no longer hide a crash forever. - Lock-less sessions (filesystem rejects locks) keep a .beat heartbeat file, touched every 2 s by the worker; a beat younger than 60 s means alive whatever pid the marker names, so a peer in another pid namespace is never mistaken for a crash. Without a beat the pid is probed as before. - claim_pending re-runs the liveness decision under the recovery claim, so a session that finished locking since the scan is not torn down. - Windows maps ERROR_INVALID_FUNCTION/NOT_SUPPORTED/CALL_NOT_IMPLEMENTED to Unsupported, so the documented degrade exists there; the dead-code allow is gone. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/claim.rs | 25 ++++-- crates/bugsee-core/src/recovery.rs | 140 ++++++++++++++++++++++------- crates/bugsee-core/src/runtime.rs | 26 +++++- crates/bugsee-core/src/session.rs | 20 +++++ 5 files changed, 170 insertions(+), 43 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 56449d1..8ad8f9c 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -207,7 +207,7 @@ Crash recovery needs the data directory to **outlive the process**: on a contain - **Costs of per-process locks, handled in `claim.rs`.** They do not exclude threads, so a registry of held paths does (keyed by pid and spin-locked with the holder's pid stamped in, so a `fork` child that inherits it mid-update breaks the lock instead of deadlocking). POSIX drops a process's locks on a file when *any* descriptor for it closes, so a lock file is never opened twice by one process: the registry is checked before each open, and a claim closes its descriptor before leaving the registry. - **Ordering at launch.** `Session::begin` creates the marker (`O_EXCL`), takes `.lock`, then writes the pid. Until the pid is written the marker is empty and fresh, which peers skip even if the lock reads free. A lock that cannot be held with kernel enforcement (contended past a short retry, or the filesystem rejects locks) is *removed*: an existing but unheld lock file reads as "that session died", so peers fall back to the marker's pid instead. On unix a probe asks `F_GETLK` and never takes the lock, so it cannot steal one from a launching peer. - **Unclean shutdown.** If the worker or uploader thread is detached at shutdown it may still be writing the session, so the session's lock is kept until the process exits (the kernel then drops it) rather than released on drop. -- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown so recovery uses the marker's pid. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. +- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown (as it does for a leftover lock file that cannot be opened, e.g. on a read-only volume, so a crash is never hidden by it). A session that could not take a lock keeps a `.beat` heartbeat file, touched every 2 s by the worker: a beat younger than 60 s (generous, since NFS caches attributes) means alive whatever pid the marker names, which is the only signal that works across pid namespaces without locks; with no beat the marker's pid is probed as before. The price is that a crash on such a filesystem is recovered only after its heartbeat has been quiet for a minute, so a faster restart defers it to a later launch. Under the recovery claim the liveness decision is re-run, so a session that finished launching since the scan is never torn down. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. - **Paths.** The in-process registry canonicalises the lock file's directory, so `/tmp` vs `/private/tmp` or relative vs absolute paths are one claim. A `fork` child clears the registry (`claim::reset_after_fork`, from the fork hook) before claiming anything. - **Stale scans.** After winning a session claim the marker is re-checked, since a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s: a peer's in-flight `enqueue` looks like an orphan, and reaping it lost the report. Lock files of finished items and dead sessions are swept when free. - **Compatibility.** The `.alive` marker is still a bare pid and the lock files are new names, so older builds sharing the directory keep working. They do not take part in the locking, so a mixed-version directory (a rolling upgrade) gives no exactly-once guarantee between old and new processes, and a new process treats an old build's session by its pid as before. Run one SDK version per directory for the guarantee. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index 39aea2f..5c2c730 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -116,7 +116,9 @@ pub(crate) enum Probe { } /// Is `path` currently claimed? A missing file is free. A file that cannot be -/// inspected is reported held: the conservative answer never disturbs a peer. +/// inspected is [`Probe::Unknown`], never assumed held: a crashed session's +/// leftover lock file must not hide the crash on a volume that has become +/// read-only. /// /// On unix this asks (`F_GETLK`) without taking the lock, so a probe can never /// steal the lock from a creator that has made the file but not yet locked it. @@ -128,20 +130,24 @@ pub(crate) fn probe(path: &Path) -> Probe { if !registry().insert(&key) { return Probe::Held; // held by a thread of this process, or being probed } - let result = match OpenOptions::new().read(true).write(true).open(path) { + // Read-only: a querying lock call needs no write access, and a leftover lock + // file on a read-only volume (or with 0444 mode) must still be probeable. + let result = match OpenOptions::new().read(true).open(path) { Ok(file) => { let r = match test_is_locked(&file) { Ok(Peek::Free) => Probe::Free, Ok(Peek::Held) => Probe::Held, - Ok(Peek::Unsupported) => Probe::Unknown, - Err(_) => Probe::Held, + // Cannot tell (no lock support, or the call failed): let the + // caller use another liveness signal rather than assume a live + // holder and hide a crash forever. + Ok(Peek::Unsupported) | Err(_) => Probe::Unknown, }; drop(file); // close before leaving the registry r } // Vanished since the existence check: nothing holds it. Err(_) if !path.exists() => Probe::Free, - Err(_) => Probe::Held, + Err(_) => Probe::Unknown, }; registry().remove(&key); result @@ -240,8 +246,6 @@ fn key_of(path: &Path) -> PathBuf { enum Lock { Got, Contended, - /// Only unix can tell "locks unsupported" apart from other errors. - #[cfg_attr(not(unix), allow(dead_code))] Unsupported, } @@ -446,6 +450,10 @@ mod os { const LOCKFILE_FAIL_IMMEDIATELY: u32 = 1; const LOCKFILE_EXCLUSIVE_LOCK: u32 = 2; const ERROR_LOCK_VIOLATION: i32 = 33; + // The filesystem has no byte-range locking (some network shares, FAT shims). + const ERROR_INVALID_FUNCTION: i32 = 1; + const ERROR_NOT_SUPPORTED: i32 = 50; + const ERROR_CALL_NOT_IMPLEMENTED: i32 = 120; #[repr(C)] struct Overlapped { @@ -493,6 +501,9 @@ mod os { let err = io::Error::last_os_error(); match err.raw_os_error() { Some(ERROR_LOCK_VIOLATION) => Ok(Lock::Contended), + Some(ERROR_INVALID_FUNCTION | ERROR_NOT_SUPPORTED | ERROR_CALL_NOT_IMPLEMENTED) => { + Ok(Lock::Unsupported) + } _ => Err(err), } } diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index ad23596..eeae9ea 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -80,30 +80,7 @@ pub fn find_pending(data_dir: &Path, current_generation: u64) -> Vec= current_generation { continue; } - // A session whose `.lock` exists is arbitrated by that OS lock: held - // means a live process (in any pid namespace), free means it died. The - // marker's pid is not consulted, since it is only meaningful inside the - // writer's own pid namespace and can be reused. - let lock = crate::session::Session::lock_path(data_dir, generation); - if lock.exists() { - match crate::claim::probe(&lock) { - crate::claim::Probe::Held => continue, - crate::claim::Probe::Free => { - // A launching session locks BEFORE it writes its pid, and a - // marker still empty and fresh may be one that is between - // creating the lock file and locking it. - if marker_is_empty(&marker) && marker_is_fresh(&marker) { - continue; - } - } - // The filesystem rejects locks: fall back to the pid. - crate::claim::Probe::Unknown => { - if !legacy_marker_is_dead(&marker, self_pid) { - continue; - } - } - } - } else if !legacy_marker_is_dead(&marker, self_pid) { + if !session_is_crashed(data_dir, generation, &marker, self_pid) { continue; } let parts_dir = data_dir.join("parts").join(generation.to_string()); @@ -125,10 +102,60 @@ fn exists_opt(p: PathBuf) -> Option { p.exists().then_some(p) } -/// Whether a marker with no lock file beside it (written by an older SDK build, -/// or by a session that could not lock) is a crashed session rather than a live -/// peer, judged from its recorded pid. Only meaningful within one pid namespace. +/// Whether the session `generation`, whose marker survives, has ended abnormally. +/// +/// A session whose `.lock` exists is arbitrated by that OS lock: held means a +/// live process (in any pid namespace), free means it died. The marker's pid is +/// not consulted, since it is only meaningful inside the writer's own pid +/// namespace and can be reused. Otherwise (no lock file, or the filesystem cannot +/// say) the marker is judged by [`legacy_marker_is_dead`]. +fn session_is_crashed(data_dir: &Path, generation: u64, marker: &Path, self_pid: u32) -> bool { + let lock = crate::session::Session::lock_path(data_dir, generation); + if lock.exists() { + match crate::claim::probe(&lock) { + crate::claim::Probe::Held => return false, + // A launching session locks BEFORE it writes its pid, and a marker + // still empty and fresh may be one that is between creating the lock + // file and locking it. + crate::claim::Probe::Free => { + return !(marker_is_empty(marker) && marker_is_fresh(marker)); + } + crate::claim::Probe::Unknown => {} + } + } + legacy_marker_is_dead(marker, self_pid) +} + +/// How recently a lock-less session's heartbeat file must have been touched for +/// the session to count as alive. Generous because network filesystems cache file +/// attributes for up to about a minute. +const HEARTBEAT_STALE: std::time::Duration = std::time::Duration::from_secs(60); + +/// The heartbeat file beside a marker. Only a session that could not take a lock +/// (the filesystem rejects them) keeps one: it is touched every couple of seconds. +pub(crate) fn heartbeat_path(marker: &Path) -> PathBuf { + marker.with_extension("beat") +} + +/// Whether a marker with no usable lock beside it (written by an older SDK build, +/// or by a session on a filesystem without locks) is a crashed session rather +/// than a live peer. +/// +/// A fresh heartbeat file means alive, whatever pid the marker names: that is the +/// only signal that works across pid namespaces when there are no locks. Without +/// one, the recorded pid is probed, which is only meaningful within one pid +/// namespace. A crash is therefore recovered only once the heartbeat has gone +/// quiet for [`HEARTBEAT_STALE`] (a restart faster than that defers recovery to a +/// later launch). fn legacy_marker_is_dead(marker: &Path, self_pid: u32) -> bool { + if std::fs::metadata(heartbeat_path(marker)) + .and_then(|m| m.modified()) + .ok() + .and_then(|t| t.elapsed().ok()) + .is_some_and(|age| age < HEARTBEAT_STALE) + { + return false; + } let owner = std::fs::read_to_string(marker).unwrap_or_default(); let owner = owner.trim(); if owner.is_empty() { @@ -325,6 +352,7 @@ fn read_minidump(pending: &PendingSession) -> Vec { pub fn discard(data_dir: &Path, pending: &PendingSession) { let sessions = data_dir.join("sessions"); let _ = std::fs::remove_file(sessions.join(format!("{}.alive", pending.generation))); + let _ = std::fs::remove_file(sessions.join(format!("{}.beat", pending.generation))); let _ = std::fs::remove_file(sessions.join(format!("{}.attempts", pending.generation))); let _ = std::fs::remove_dir_all(&pending.parts_dir); } @@ -373,11 +401,14 @@ pub fn claim_pending( // (its claim released, the marker gone). Recovering it now would deliver a // phantom second report for it. The marker is removed first by `discard`, so // its absence, re-checked while holding the claim, is conclusive. - data_dir - .join("sessions") - .join(format!("{}.alive", pending.generation)) - .exists() - .then_some(claim) + // + // It also re-evaluates liveness under the claim: between the scan and now a + // launching session may have finished taking its lock (the scan saw it free, + // then the pid appeared), and a live session must not be torn down. + let marker = crate::session::Session::marker_path(data_dir, pending.generation); + (marker.exists() + && session_is_crashed(data_dir, pending.generation, &marker, std::process::id())) + .then_some(claim) } /// Reap leftover on-disk state from generations that no longer have a liveness @@ -404,7 +435,10 @@ pub fn gc_orphans(data_dir: &Path, current_generation: u64) { if let Ok(read) = std::fs::read_dir(&sessions) { for entry in read.flatten() { let name = entry.file_name().to_string_lossy().into_owned(); - if let Some(gen_str) = name.strip_suffix(".attempts") { + if let Some(gen_str) = name + .strip_suffix(".attempts") + .or_else(|| name.strip_suffix(".beat")) + { if let Ok(generation) = gen_str.parse::() { if generation < current_generation && !alive(generation) { let _ = std::fs::remove_file(entry.path()); @@ -826,6 +860,46 @@ mod tests { assert_eq!(found, vec![1]); } + #[cfg(unix)] + #[test] + fn an_unwritable_leftover_lock_file_does_not_hide_a_crash() { + use std::os::unix::fs::PermissionsExt; + let dir = tmp(); + marker(&dir, 1, "4294967290"); + let lock = Session::lock_path(&dir, 1); + std::fs::write(&lock, "").unwrap(); + std::fs::set_permissions(&lock, std::fs::Permissions::from_mode(0o444)).unwrap(); + assert_eq!(gens(&dir), vec![1]); + } + + #[test] + fn a_fresh_heartbeat_keeps_a_lockless_session_alive_whatever_its_pid() { + // No lock file (the filesystem has none): the marker's pid, even our own + // (a peer in another pid namespace), must not condemn a beating session. + let dir = tmp(); + let m = marker(&dir, 1, &std::process::id().to_string()); + let beat = heartbeat_path(&m); + std::fs::write(&beat, "").unwrap(); + assert!(gens(&dir).is_empty(), "heartbeat is fresh: alive"); + let f = std::fs::OpenOptions::new().write(true).open(&beat).unwrap(); + f.set_modified(std::time::SystemTime::now() - HEARTBEAT_STALE * 2) + .unwrap(); + drop(f); + assert_eq!(gens(&dir), vec![1], "quiet heartbeat: crashed"); + } + + #[test] + fn claim_pending_gives_up_if_the_session_took_its_lock_since_the_scan() { + let dir = tmp(); + marker(&dir, 1, "4294967290"); + std::fs::write(Session::lock_path(&dir, 1), "").unwrap(); + let pending = find_pending(&dir, u64::MAX); + assert_eq!(pending.len(), 1); + // The launching session finishes locking after the scan. + let _live = try_claim(&Session::lock_path(&dir, 1)).unwrap(); + assert!(claim_pending(&dir, &pending[0]).is_none()); + } + #[test] fn old_scheme_claim_files_of_dead_sessions_are_swept() { let dir = tmp(); diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 3a41e66..c97288d 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1078,8 +1078,11 @@ fn worker_loop( // sampled (F17). Instead we recv with a timeout bounded by the next rotation // deadline and run the tick whenever that deadline passes, message or not. let mut next_rotate = Instant::now() + rotate_interval; + let mut next_heartbeat = Instant::now() + HEARTBEAT_EVERY; loop { - let timeout = next_rotate.saturating_duration_since(Instant::now()); + let timeout = next_rotate + .saturating_duration_since(Instant::now()) + .min(HEARTBEAT_EVERY); match rx.recv_timeout(timeout) { Ok(Msg::Capture(entry)) => { shared.queued.fetch_sub(1, Ordering::Relaxed); @@ -1159,14 +1162,33 @@ fn worker_loop( // elapsed β€” via a message OR a timeout β€” so steady traffic can't starve it. if Instant::now() >= next_rotate { rotate_tick(store.as_mut(), &shared, &mut sampler); + next_rotate = Instant::now() + rotate_interval; + } + // Independent of the rotation interval (which a host may set large). + if Instant::now() >= next_heartbeat { if let Some(marker) = &marker { ensure_marker(&shared, marker); + touch_heartbeat(marker); } - next_rotate = Instant::now() + rotate_interval; + next_heartbeat = Instant::now() + HEARTBEAT_EVERY; } } } +/// How often the worker checks the liveness marker and, for a session that could +/// not take a file lock, touches its heartbeat file. +const HEARTBEAT_EVERY: Duration = Duration::from_secs(2); + +/// Touch this session's heartbeat file, if it keeps one (see +/// `recovery::heartbeat_path`). Never creates it: a session with a kernel lock has +/// none, and one that has ended must not leave one behind. +fn touch_heartbeat(marker: &Path) { + let beat = recovery::heartbeat_path(marker); + if let Ok(file) = std::fs::OpenOptions::new().write(true).open(&beat) { + let _ = file.set_modified(std::time::SystemTime::now()); + } +} + /// Make sure this session's liveness marker still exists. If the data directory /// was removed and recreated under us (`create_dir_all` heals the tree but not /// the marker), the next launch would not see this session as live, and its diff --git a/crates/bugsee-core/src/session.rs b/crates/bugsee-core/src/session.rs index 6a8218e..b92515a 100644 --- a/crates/bugsee-core/src/session.rs +++ b/crates/bugsee-core/src/session.rs @@ -56,6 +56,16 @@ impl Session { // Lock BEFORE writing the pid: the marker is empty (and fresh, // so peers skip it) until then. let lock = Session::take_lock(data_dir, candidate); + if lock.is_none() { + // No kernel lock to tell peers we are alive: a heartbeat + // file, touched by the worker, is the substitute. + let _ = std::fs::write( + crate::recovery::heartbeat_path(&Session::marker_path( + data_dir, candidate, + )), + b"", + ); + } // The marker stays a bare pid, readable by older builds. let _ = write!(f, "{}", std::process::id()); break (candidate, lock); @@ -133,6 +143,10 @@ impl Session { pub fn end(&self) { let _ = std::fs::remove_file(Session::marker_path(&self.data_dir, self.generation)); let _ = std::fs::remove_file(Session::lock_path(&self.data_dir, self.generation)); + let _ = std::fs::remove_file(crate::recovery::heartbeat_path(&Session::marker_path( + &self.data_dir, + self.generation, + ))); } } @@ -217,6 +231,12 @@ mod tests { crate::claim::FORCE_UNSUPPORTED.with(|f| f.set(false)); assert!(session._lock.is_none()); assert!(!Session::lock_path(&dir, session.generation()).exists()); + // Instead it keeps a heartbeat file, removed on a clean end. + let beat = + crate::recovery::heartbeat_path(&Session::marker_path(&dir, session.generation())); + assert!(beat.exists()); + session.end(); + assert!(!beat.exists()); } #[test] From 38f73e62712d6171fc5bea91ade6fcc34b0c9671 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:27:22 +0500 Subject: [PATCH 24/52] feat(health): local session health records for release health (#12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records how each session ends (ok/errored/crashed/abnormal) behind an off-by-default option. Recovered crashes settle their session exactly once. Nothing is sent: the backend has no session-health ingest endpoint. Refs #12 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 4 + PROGRESS.md | 2 +- crates/bugsee-core/src/health.rs | 381 ++++++++++++++++++++++++++ crates/bugsee-core/src/lib.rs | 1 + crates/bugsee-core/src/recovery.rs | 58 ++++ crates/bugsee-core/src/runtime.rs | 64 +++++ crates/bugsee/Cargo.toml | 6 + crates/bugsee/src/api.rs | 21 ++ crates/bugsee/src/options.rs | 16 ++ crates/bugsee/tests/release_health.rs | 280 +++++++++++++++++++ 10 files changed, 832 insertions(+), 1 deletion(-) create mode 100644 crates/bugsee-core/src/health.rs create mode 100644 crates/bugsee/tests/release_health.rs diff --git a/DESIGN.md b/DESIGN.md index 249cc71..7dd121c 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -234,6 +234,10 @@ Tests: `bugsee-core/tests/launch_recovery.rs` (a real abort-after-launch crash l **Snapshot (no seal, no copy):** pin `[start,end]` (`end`=snapshot instant, `start`=`endβˆ’window`) + **hard-link current parts including the active one** into the report dir (`PartStore::snapshot_into`, after a `flush`). At export, read linked parts and include only entries with `start ≀ timestamp ≀ end` (monotonic order β†’ early-stop at the tail; the self-terminating `RecordIter` cleanly ignores any torn trailing write on the active part). Bounds become `manifest.time.start/end`. Cost: 2 timestamps + O(#parts) `link` syscalls. Fallback to copy on FAT/exFAT/cross-volume (capability-probed once). Reflink (APFS `clonefile` / Linux `FICLONE`) a noted future tier. +**Release health: sessions and crash-free rates (#12), SDK half only.** What the backend does today (`appserver` `components/shared/statistics`): it counts one session per session-create (`POST /v2/sessions`) and one crash per crash issue, and computes crash-free sessions as `(sessions - crashes) / sessions`, per day and per release (app `version`/`build`). There is **no endpoint** that accepts a session end, status, duration or error count from an SDK, no `ok/errored/crashed/abnormal` model, and no spec (`specs/`, `report-bundle-structure/`) mentions one. The SDK therefore does not send anything. `LaunchOptions::release_health(true)` (off by default) keeps a durable local record per session, `health/.live` while running and `health/.json` when finished, readable with `Bugsee::finished_sessions()` / `session_health()` and cleared with `acknowledge_session`. Rules: a session is one SDK launch (desktop app, server and CLI alike; no idle timeout); a handled error, including a panic on a non-main thread, makes it `errored`, never `crashed`; a clean stop (guard drop, not while unwinding) finishes it `ok`/`errored`; a session that died is finished by the next launch's recovery as `crashed` (native signal, minidump or uncaught-panic evidence) or `abnormal` (none), inside `recovery::discard`, under the recover claim and whatever became of its report, so each dead session is counted once across relaunches, and the finished record is written (rename) before the marker goes so a retry is idempotent. Memory mode (#9) tracks nothing (a crash there is invisible, so every session would read healthy). A fork child is a session of its own; the parent's is not touched. The crash handler and panic hook write nothing new (the crash path is unchanged and allocation-free); the status is inferred at the next launch. Error counts are rewritten for the first 8 errors and then at powers of two, so a session that died keeps an approximate count and a dead session's end time is the last file activity in its capture parts. + +*Blocked on the backend:* (1) an ingest endpoint or a field on session-create/close for `{session_id, started_at, ended_at, duration, status, errors, release, environment}`, authenticated like the other calls, idempotent on `session_id`; (2) how crashed-vs-abnormal feeds `crashes_free_sessions` (today only crash *issues* count, and a session whose crash report was deduplicated or blacklisted still counts as crash-free server-side, which the SDK records as crashed); (3) user-level rates need a stable anonymous user id (none exists in the Rust SDK); (4) sampling for servers with many short sessions, to be decided with the endpoint; (5) a `Transport::report_session` method and offline queueing of finished records (the durable queue and `.recover.lock` claims are ready to carry them) once the shape exists. + ## 9. Failure pipeline **Boundary guards (`bugsee-panic`):** every exported FFI fn / SDK thread root / task root runs inside `catch_unwind` + a TLS scope guard (`sdk_scope_depth`, `foreign_callback_depth`, `operation_id`). A panic never unwinds across `extern "C"`. Caught panic β†’ quarantine (Class A–D) β†’ non-fatal event (`type:"error"`, `handled:true`). diff --git a/PROGRESS.md b/PROGRESS.md index 6f6bbeb..dc147ef 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -515,7 +515,7 @@ Rust and are worth evaluating before building one. - Phase 2: subsystem quarantine (auto-disable a repeatedly-faulting integration). - Phase 3: best-effort in-flight flush at crash; full out-of-process minidump (desktop) β€” see the backend note below, which is the same machinery. -- Phase 4: sessions / release-health. +- Phase 4: sessions / release-health: SDK half done (#12: local session records, off by default; nothing is sent, the backend has no session-health ingest, see DESIGN.md Β§8). - Phase 5: general PII scrubber, rate limits. - Phase 6: iOS/Android target builds; Swift/Kotlin language wrappers. - CI: ~~cross-platform matrix~~ **done**; loom/miri/fuzz jobs still open β€” fuzzing is diff --git a/crates/bugsee-core/src/health.rs b/crates/bugsee-core/src/health.rs new file mode 100644 index 0000000..c42166e --- /dev/null +++ b/crates/bugsee-core/src/health.rs @@ -0,0 +1,381 @@ +// +// health.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// + +//! Session health: a durable record of how each run of the application ended +//! (issue #12), the SDK half of release health / crash-free sessions. +//! +//! **There is no backend half yet.** The Bugsee backend derives crash-free +//! sessions itself: it counts one session per `POST /v2/sessions` and one crash +//! per crash issue (`appserver` `shared/statistics`: `sessions`, +//! `issues.crashes`; crash-free = `(sessions - crashes) / sessions`), and it has +//! no endpoint that accepts a session's end, status, duration or error count. +//! Nothing here is sent anywhere. The records are kept locally, off by default +//! ([`HealthTracker`] exists only when the host asked for release health), and +//! are exposed through [`finished`] / [`acknowledge`] so a host can use them +//! today and a transport can ship them once a contract exists. DESIGN.md +//! (release health) lists what the backend still has to define. +//! +//! # Lifecycle +//! +//! One **session** is one SDK launch (one generation), whatever the process +//! does: a desktop app, a long-running server and a short-lived CLI all start it +//! at launch and end it at a clean stop. There is no idle timeout; a server's +//! session is the process's life. +//! +//! ```text +//! launch health/.live written (status ok, 0 errors) +//! handled error health/.live error count rewritten (rate limited) +//! clean stop health/.json written (ok | errored), then .live removed +//! crash / kill .live stays; the NEXT launch's recovery finalizes it as +//! crashed (crash evidence) or abnormal (none), then removes it +//! ``` +//! +//! A non-main-thread panic is an **error**, not a crash: the process survives, +//! and counting it as a crash would corrupt crash-free rates. +//! +//! # Exactly once +//! +//! A finished record is named by its generation and written by whoever holds the +//! session (the live process at a clean stop, or the process holding +//! `sessions/.recover.lock` at recovery), via a temp file and a rename. A +//! finalizer that finds the record already there only removes the `.live` file, +//! so a retry after a crash between the two steps cannot flip a status or count +//! a session twice. Generations are never reused in a data directory. +//! +//! None of this runs on the crash path: the signal handler and panic hook write +//! nothing here. The crash is inferred at the next launch from the evidence the +//! handlers already leave. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::model::environment::Environment; + +const HEALTH_DIR: &str = "health"; +/// Newest finished records kept; older ones are pruned when a session finishes. +pub const MAX_FINISHED: usize = 256; +/// Error counts are rewritten to disk for the first few errors and then at each +/// power of two, so an error storm costs a handful of small writes, not one each. +const EAGER_WRITES: u64 = 8; + +/// How a session ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SessionStatus { + /// Ended cleanly with no handled errors. + Ok, + /// Ended cleanly but reported at least one handled error (including a + /// panic on a non-main thread). + Errored, + /// The process died of a fatal signal or an uncaught panic. + Crashed, + /// The process vanished without evidence of a crash (SIGKILL, OOM kill, + /// power loss). + Abnormal, +} + +/// The durable record of one session. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SessionRecord { + /// Schema version of this record. + pub version: u32, + /// Random id, stable across the session's records. + pub session_id: String, + /// The generation in the data directory (monotonic, never reused). + pub generation: u64, + /// Epoch ms. + pub started_at: i64, + /// Epoch ms; for a crashed or abnormal session, the last activity seen. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub ended_at: Option, + pub status: SessionStatus, + /// Handled errors counted in the session. Approximate for a session that + /// died: counts after the last rewrite are lost. + pub errors: u64, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub duration_ms: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub package_id: Option, + /// The release: the application version and build. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub app_version: Option, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub app_build: Option, + pub platform: String, + pub sdk_version: String, +} + +fn dir(data_dir: &Path) -> PathBuf { + data_dir.join(HEALTH_DIR) +} + +fn live_path(data_dir: &Path, generation: u64) -> PathBuf { + dir(data_dir).join(format!("{generation}.live")) +} + +fn finished_path(data_dir: &Path, generation: u64) -> PathBuf { + dir(data_dir).join(format!("{generation}.json")) +} + +/// Write `record` atomically (temp file, then rename). +fn write_record(path: &Path, record: &SessionRecord) -> std::io::Result<()> { + let bytes = serde_json::to_vec(record).map_err(std::io::Error::other)?; + let tmp = path.with_extension("tmp"); + std::fs::write(&tmp, bytes)?; + std::fs::rename(&tmp, path).inspect_err(|_| { + let _ = std::fs::remove_file(&tmp); + }) +} + +fn read_record(path: &Path) -> Option { + serde_json::from_slice(&std::fs::read(path).ok()?).ok() +} + +/// The live session's record, owned by the running process. +pub struct HealthTracker { + data_dir: PathBuf, + record: SessionRecord, +} + +impl HealthTracker { + /// Start tracking the session `generation` in `data_dir`. `None` when the + /// record cannot be written (the session then simply is not tracked). + pub fn start(data_dir: &Path, generation: u64, env: &Environment) -> Option { + std::fs::create_dir_all(dir(data_dir)).ok()?; + let record = SessionRecord { + version: 1, + session_id: crate::util::random_hex(8), + generation, + started_at: crate::util::epoch_ms(), + ended_at: None, + status: SessionStatus::Ok, + errors: 0, + duration_ms: None, + package_id: env.app.package_id.clone(), + app_version: env.app.version.clone(), + app_build: env.app.build.clone(), + platform: env.platform.os_type.clone(), + sdk_version: env.sdk.version.clone(), + }; + write_record(&live_path(data_dir, generation), &record).ok()?; + Some(HealthTracker { + data_dir: data_dir.to_path_buf(), + record, + }) + } + + /// Count one handled error. + pub fn note_error(&mut self) { + self.record.errors = self.record.errors.saturating_add(1); + self.record.status = SessionStatus::Errored; + let n = self.record.errors; + if n <= EAGER_WRITES || n.is_power_of_two() { + let _ = write_record( + &live_path(&self.data_dir, self.record.generation), + &self.record, + ); + } + } + + /// The record as it stands. + pub fn record(&self) -> &SessionRecord { + &self.record + } + + /// End the session cleanly: publish the finished record, then drop the live + /// one. + pub fn finish_clean(&self) { + let now = crate::util::epoch_ms(); + let mut record = self.record.clone(); + record.ended_at = Some(now); + record.duration_ms = Some((now - record.started_at).max(0)); + publish(&self.data_dir, record); + } +} + +/// Finish a session that died, from the process that recovered it. Marks it +/// `status` (`Crashed` or `Abnormal`) unless a finished record already exists, +/// which is left alone. Idempotent; a missing live record (release health was +/// off for that run) does nothing. +pub fn finalize_dead( + data_dir: &Path, + generation: u64, + status: SessionStatus, + last_activity: Option, +) { + let live = live_path(data_dir, generation); + let Some(mut record) = read_record(&live) else { + // Unreadable or absent: nothing to report, but do not leave junk. + let _ = std::fs::remove_file(&live); + let _ = std::fs::remove_file(live.with_extension("tmp")); + return; + }; + record.status = status; + let end = last_activity + .unwrap_or(record.started_at) + .max(record.started_at); + record.ended_at = Some(end); + record.duration_ms = Some(end - record.started_at); + publish(data_dir, record); +} + +/// Write the finished record unless one exists, then remove the live one. +fn publish(data_dir: &Path, record: SessionRecord) { + let done = finished_path(data_dir, record.generation); + if !done.exists() { + let _ = write_record(&done, &record); + } + // Only drop the live record once the finished one is durable: a failure + // leaves it for the next recovery, which finds the finished one and stops. + if done.exists() { + let _ = std::fs::remove_file(live_path(data_dir, record.generation)); + } + prune(data_dir); +} + +fn prune(data_dir: &Path) { + let mut gens = finished_generations(data_dir); + if gens.len() > MAX_FINISHED { + gens.sort_unstable(); + for g in &gens[..gens.len() - MAX_FINISHED] { + let _ = std::fs::remove_file(finished_path(data_dir, *g)); + } + } +} + +fn finished_generations(data_dir: &Path) -> Vec { + let Ok(read) = std::fs::read_dir(dir(data_dir)) else { + return Vec::new(); + }; + read.flatten() + .filter_map(|e| { + let name = e.file_name().to_string_lossy().into_owned(); + name.strip_suffix(".json")?.parse().ok() + }) + .collect() +} + +/// Finished sessions in `data_dir`, oldest first, that have not been +/// [`acknowledge`]d. +pub fn finished(data_dir: &Path) -> Vec { + let mut gens = finished_generations(data_dir); + gens.sort_unstable(); + gens.into_iter() + .filter_map(|g| read_record(&finished_path(data_dir, g))) + .collect() +} + +/// Forget a finished session once the host (or a future transport) has taken it. +pub fn acknowledge(data_dir: &Path, generation: u64) { + let _ = std::fs::remove_file(finished_path(data_dir, generation)); +} + +/// The newest modification time (epoch ms) under `dir`, to `depth` levels down. +/// A dead session's last-activity stamp: the capture parts are written as it +/// runs. +pub(crate) fn newest_mtime(dir: &Path, depth: u32) -> Option { + let mut newest = None::; + for entry in std::fs::read_dir(dir).ok()?.flatten() { + let Ok(meta) = entry.metadata() else { continue }; + let mut t = meta + .modified() + .ok() + .and_then(|m| m.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_millis() as i64); + if meta.is_dir() && depth > 0 { + t = t.max(newest_mtime(&entry.path(), depth - 1)); + } + newest = newest.max(t); + } + newest +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp() -> PathBuf { + let p = std::env::temp_dir().join(format!("bugsee-health-{}", crate::util::random_hex(8))); + std::fs::create_dir_all(&p).unwrap(); + p + } + + fn env() -> Environment { + Environment::detect_with( + "9.9.9", + &Default::default(), + &crate::model::environment::AppIdentity { + package_id: Some("app".into()), + version: Some("1.2.3".into()), + build: Some("45".into()), + }, + ) + } + + #[test] + fn a_clean_session_is_ok_and_carries_the_release() { + let d = tmp(); + let t = HealthTracker::start(&d, 1, &env()).unwrap(); + assert!(live_path(&d, 1).exists()); + t.finish_clean(); + assert!(!live_path(&d, 1).exists()); + let all = finished(&d); + assert_eq!(all.len(), 1); + assert_eq!(all[0].status, SessionStatus::Ok); + assert_eq!(all[0].errors, 0); + assert_eq!(all[0].app_version.as_deref(), Some("1.2.3")); + assert_eq!(all[0].app_build.as_deref(), Some("45")); + assert!(all[0].duration_ms.unwrap() >= 0); + } + + #[test] + fn handled_errors_make_the_session_errored_not_crashed() { + let d = tmp(); + let mut t = HealthTracker::start(&d, 1, &env()).unwrap(); + t.note_error(); + t.note_error(); + t.finish_clean(); + let r = &finished(&d)[0]; + assert_eq!((r.status, r.errors), (SessionStatus::Errored, 2)); + } + + #[test] + fn a_dead_session_is_finalized_once_whoever_asks() { + let d = tmp(); + let _t = HealthTracker::start(&d, 3, &env()).unwrap(); + finalize_dead(&d, 3, SessionStatus::Crashed, None); + // A second finalizer (a retry after a crash between the two steps, with + // the live record put back) must not flip or duplicate the status. + let _t2 = HealthTracker::start(&d, 3, &env()); + finalize_dead(&d, 3, SessionStatus::Abnormal, None); + let all = finished(&d); + assert_eq!(all.len(), 1); + assert_eq!(all[0].status, SessionStatus::Crashed); + assert!(!live_path(&d, 3).exists()); + } + + #[test] + fn finalizing_a_session_that_was_not_tracked_does_nothing() { + let d = tmp(); + finalize_dead(&d, 5, SessionStatus::Crashed, None); + assert!(finished(&d).is_empty()); + } + + #[test] + fn acknowledged_and_old_records_are_dropped() { + let d = tmp(); + for g in 1..=(MAX_FINISHED as u64 + 3) { + HealthTracker::start(&d, g, &env()).unwrap().finish_clean(); + } + let all = finished(&d); + assert_eq!(all.len(), MAX_FINISHED); + assert_eq!(all[0].generation, 4); + acknowledge(&d, 4); + assert_eq!(finished(&d).len(), MAX_FINISHED - 1); + } +} diff --git a/crates/bugsee-core/src/lib.rs b/crates/bugsee-core/src/lib.rs index e187ece..899705b 100644 --- a/crates/bugsee-core/src/lib.rs +++ b/crates/bugsee-core/src/lib.rs @@ -21,6 +21,7 @@ pub mod capture; pub mod claim; pub mod errors; pub mod fork; +pub mod health; pub mod model; pub mod panic_info; pub(crate) mod process; diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 76f7b1a..577887d 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -355,12 +355,35 @@ fn read_minidump(pending: &PendingSession) -> Vec { /// GC'able via [`gc_orphans`]) rather than leaving a marker that would re-recover /// a phantom empty report or duplicate the already-delivered one. pub fn discard(data_dir: &Path, pending: &PendingSession) { + // The session's health record is settled here, whatever became of its report + // (delivered, deduplicated, dropped by `before_send`): the session crashed + // either way. Callers hold the recovery claim, and `discard` runs only once + // the evidence is spent, so this is the one place a dead session is counted; + // the finished record is written before the marker goes (see `health`). + crate::health::finalize_dead( + data_dir, + pending.generation, + dead_status(pending), + crate::health::newest_mtime(&pending.parts_dir, 3), + ); let sessions = data_dir.join("sessions"); let _ = std::fs::remove_file(sessions.join(format!("{}.alive", pending.generation))); let _ = std::fs::remove_file(sessions.join(format!("{}.attempts", pending.generation))); let _ = std::fs::remove_dir_all(&pending.parts_dir); } +/// How a dead session ended: `Crashed` when the crash handlers left evidence (a +/// native signal, a minidump or an uncaught panic's snapshot, the same evidence +/// [`build_report`] reports as a crash), `Abnormal` when the process just +/// vanished (SIGKILL, OOM kill, power loss). +fn dead_status(pending: &PendingSession) -> crate::health::SessionStatus { + if pending.minidump.is_some() || pending.crash_info.is_some() || pending.panic_info.is_some() { + crate::health::SessionStatus::Crashed + } else { + crate::health::SessionStatus::Abnormal + } +} + /// The recovery-attempt-counter sidecar path for a generation. fn attempts_path(data_dir: &Path, generation: u64) -> PathBuf { data_dir @@ -811,6 +834,41 @@ mod tests { .collect() } + #[test] + fn discarding_a_dead_session_settles_its_health_by_the_evidence_exactly_once() { + use crate::health::{finished, HealthTracker, SessionStatus}; + let dir = tmp(); + let env = crate::model::environment::Environment::detect_with( + "0", + &Default::default(), + &Default::default(), + ); + for (generation, panic_info) in [(1u64, true), (2, false)] { + marker(&dir, generation, "0"); + HealthTracker::start(&dir, generation, &env).unwrap(); + let parts = dir.join("parts").join(generation.to_string()); + std::fs::create_dir_all(&parts).unwrap(); + let pending = PendingSession { + generation, + parts_dir: parts, + minidump: None, + crash_info: None, + panic_info: panic_info.then(|| dir.join("panic.info")), + }; + discard(&dir, &pending); + // A repeat (the same session recovered again) changes nothing. + discard(&dir, &pending); + } + let got: Vec<_> = finished(&dir) + .iter() + .map(|r| (r.generation, r.status)) + .collect(); + assert_eq!( + got, + vec![(1, SessionStatus::Crashed), (2, SessionStatus::Abnormal)] + ); + } + #[test] fn the_bounded_scan_is_newest_first_and_capped() { let dir = tmp(); diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index c25eb0b..c8c1657 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -197,6 +197,9 @@ pub struct RecorderConfig { /// `legacy_data_dir`: trusted trees that could not take new files (a full or /// read-only default) even though a later default was writable. pub extra_data_dirs: Vec, + /// Record how this session ends (issue #12, see [`crate::health`]). Off by + /// default: nothing consumes it server-side yet. + pub release_health: bool, } impl RecorderConfig { @@ -222,6 +225,7 @@ impl RecorderConfig { launch_recovery_budget: Duration::ZERO, launch_recovery_max_sessions: DEFAULT_LAUNCH_RECOVERY_SESSIONS, extra_data_dirs: Vec::new(), + release_health: false, } } } @@ -290,6 +294,10 @@ struct Shared { launch_armed: AtomicBool, launch_done: AtomicBool, launch_wait_ms: AtomicU64, + /// This session's health record (issue #12); `None` unless release health is + /// on and the session is on disk (memory mode has no recovery, so a crash + /// would never be seen and every session would read as healthy). + health: ForkLock>, } /// What the launch-path recovery pass needs, fixed at launch. @@ -312,8 +320,10 @@ impl ForkLockable for Shared { self.delivered_stuck.fork_lock(); self.launch_plan.fork_lock(); self.launch_attempted.fork_lock(); + self.health.fork_lock(); } fn fork_unlock(&self) { + self.health.fork_unlock(); self.launch_attempted.fork_unlock(); self.launch_plan.fork_unlock(); self.delivered_stuck.fork_unlock(); @@ -421,6 +431,7 @@ impl Recorder { launch_armed: AtomicBool::new(false), launch_done: AtomicBool::new(false), launch_wait_ms: AtomicU64::new(0), + health: ForkLock::new(None), }); crate::fork::install_fork_hook(); @@ -453,6 +464,15 @@ impl Recorder { Some((session, store)) => (Some(session), Some(store)), None => (None, None), }; + if config.release_health { + if let Some(session) = &session { + *shared.health.lock() = crate::health::HealthTracker::start( + &data_dir, + session.generation(), + &shared.env, + ); + } + } // A directory that cannot take NEW files may still hold reports and crashes // a previous run persisted (that is what a full disk looks like), and // recovering them only reads and deletes. So an existing directory is @@ -570,6 +590,21 @@ impl Recorder { Some((session, store)) => (Some(session), Some(store)), None => (None, None), }; + // The child is a session of its own: the parent's record stays the + // parent's (never finished or counted from here), and a child that fell + // back to memory tracks nothing. + { + let mut health = self.shared.health.lock(); + let enabled = health.is_some(); + *health = match (&session, enabled) { + (Some(s), true) => crate::health::HealthTracker::start( + &self.data_dir, + s.generation(), + &self.shared.env, + ), + _ => None, + }; + } let pipeline = spawn_pipeline( &self.shared, store.is_some().then_some(self.data_dir.as_path()), @@ -688,6 +723,21 @@ impl Recorder { self.handle().flush(timeout) } + /// The running session's health record, when release health is on. + pub fn session_health(&self) -> Option { + self.shared + .health + .lock() + .as_ref() + .map(|h| h.record().clone()) + } + + /// Finished sessions recorded in this data directory (clean ends and + /// recovered crashes), oldest first, until [`crate::health::acknowledge`]d. + pub fn finished_sessions(&self) -> Vec { + crate::health::finished(&self.data_dir) + } + /// Data directory root for this recorder. pub fn data_dir(&self) -> &Path { &self.data_dir @@ -782,6 +832,11 @@ impl Drop for Recorder { // cannot turn a handled panic into a phantom crash. if worker_acked && uploader_acked && !std::thread::panicking() { if let Some(session) = &self.session { + // The finished health record first, then the marker: a death in + // between leaves a marker that recovery settles idempotently. + if let Some(health) = self.shared.health.lock().take() { + health.finish_clean(); + } session.end(); } } else if !(worker_acked && uploader_acked) { @@ -936,6 +991,15 @@ fn report_into( window_end: i64, crash_json: Option>, ) { + // A handled error counts against the session's health whatever becomes of + // the report (sampled out, dropped, rate limited): the session had an error. + // A panic on a non-main thread arrives here as an `Error` too, never as a + // crash: the process survives it. + if meta.issue_type == IssueType::Error { + if let Some(health) = shared.health.lock().as_mut() { + health.note_error(); + } + } // Every enqueued report increments the counter (the worker decrements on // drain), but only NON-crash reports are subject to the drop-newest cap. let prev = shared.report_queued.fetch_add(1, Ordering::Relaxed); diff --git a/crates/bugsee/Cargo.toml b/crates/bugsee/Cargo.toml index 3051090..c1723d1 100644 --- a/crates/bugsee/Cargo.toml +++ b/crates/bugsee/Cargo.toml @@ -43,5 +43,11 @@ name = "crash_e2e_matrix" harness = false required-features = ["native"] +# Same reason: the session-health matrix kills real child processes. +[[test]] +name = "release_health" +harness = false +required-features = ["native"] + [target.'cfg(unix)'.dev-dependencies] libc = "0.2" diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index 3804499..b268802 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -142,6 +142,7 @@ impl Bugsee { config.before_breadcrumb = options.before_breadcrumb; config.sample_rate = options.sample_rate; config.launch_recovery_budget = options.launch_recovery_budget; + config.release_health = options.release_health; config.app_identity = bugsee_core::model::environment::AppIdentity { package_id: options.app_package_id.clone(), version: options.app_version.clone(), @@ -267,6 +268,26 @@ impl Bugsee { }) } + /// The running session's health record, or `None` when the SDK is not + /// launched, release health is off ([`LaunchOptions::release_health`]) or + /// the SDK is in memory mode. + pub fn session_health() -> Option { + Self::with_recorder(|r| r.session_health()).flatten() + } + + /// Sessions of this data directory that have ended, oldest first: clean ends + /// (`ok` / `errored`) and earlier runs this launch recovered (`crashed` / + /// `abnormal`), each exactly once. Take one with + /// [`Bugsee::acknowledge_session`]; the newest 256 are kept. + pub fn finished_sessions() -> Vec { + Self::with_recorder(|r| r.finished_sessions()).unwrap_or_default() + } + + /// Forget a finished session (by its `generation`) once it has been used. + pub fn acknowledge_session(generation: u64) { + Self::with_recorder(|r| bugsee_core::health::acknowledge(r.data_dir(), generation)); + } + /// Stop the SDK and flush pending work (dropping the recorder joins the worker). pub fn stop() { check_fork(); diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index e2c014d..99b1480 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -40,6 +40,7 @@ pub struct LaunchOptions { pub(crate) before_breadcrumb: Option, pub(crate) sample_rate: f64, pub(crate) launch_recovery_budget: Duration, + pub(crate) release_health: bool, } impl LaunchOptions { @@ -65,6 +66,7 @@ impl LaunchOptions { before_breadcrumb: None, sample_rate: 1.0, launch_recovery_budget: bugsee_core::runtime::DEFAULT_LAUNCH_RECOVERY_BUDGET, + release_health: false, } } @@ -251,6 +253,20 @@ impl LaunchOptions { self } + /// Record how each session ends (clean, errored, crashed, abnormal exit) in + /// the data directory, for [`Bugsee::finished_sessions`](crate::Bugsee::finished_sessions). + /// Off by default. + /// + /// **Local only.** The backend has no endpoint for session health yet (it + /// derives crash-free rates from session creation and crash issues), so + /// nothing is sent. Requires a usable data directory: in memory mode a crash + /// could never be seen, so no session is tracked. See DESIGN.md, release + /// health. + pub fn release_health(mut self, enabled: bool) -> Self { + self.release_health = enabled; + self + } + pub fn endpoint(mut self, endpoint: impl Into) -> Self { self.endpoint = Some(endpoint.into()); self diff --git a/crates/bugsee/tests/release_health.rs b/crates/bugsee/tests/release_health.rs new file mode 100644 index 0000000..b2469f4 --- /dev/null +++ b/crates/bugsee/tests/release_health.rs @@ -0,0 +1,280 @@ +// +// release_health.rs +// bugsee +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! Issue #12 end to end: how a session ended, recorded locally and settled +//! exactly once, across real process deaths. +//! +//! A child process launches the SDK with release health on and ends in the +//! requested way; the parent inspects what is on disk, relaunches (which +//! recovers the dead session) and relaunches again (which must not count it a +//! second time). +//! +//! `harness = false`, like the crash matrix: the child dies from the real main +//! thread (an uncaught panic must unwind out of `main`; a panic on any other +//! thread is by definition an error, not a crash). + +#![cfg(all(feature = "native", any(unix, windows)))] + +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use bugsee::core::health::{SessionRecord, SessionStatus}; +use bugsee::core::MockTransport; +use bugsee::{Bugsee, LaunchOptions}; + +const KIND_ENV: &str = "BUGSEE_HEALTH_E2E_KIND"; +const DIR_ENV: &str = "BUGSEE_HEALTH_E2E_DIR"; + +#[derive(Debug)] +struct Boom; +impl std::fmt::Display for Boom { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("boom") + } +} +impl std::error::Error for Boom {} + +fn launch(dir: &str, health: bool) -> bugsee::LaunchGuard { + Bugsee::launch_with( + LaunchOptions::new("APP_TOKEN") + .data_dir(dir) + .app_version("4.5.6") + .app_build("789") + .release_health(health) + .with_transport(Arc::new(MockTransport::default())), + ) + .expect("launch") +} + +// --------------------------------------------------------------------------- +// The child. +// --------------------------------------------------------------------------- + +#[inline(never)] +fn fault_null() { + // SAFETY: deliberately faulting. + unsafe { std::ptr::write_volatile(std::hint::black_box(std::ptr::null_mut::()), 1) } +} + +fn run_child(kind: &str, dir: &str) -> ! { + let guard = launch(dir, kind != "health_off"); + std::thread::sleep(Duration::from_millis(300)); + match kind { + "clean" | "health_off" => {} + "errored" => { + Bugsee::capture_error(&Boom); + Bugsee::capture_error(&Boom); + } + // The process survives a panic on another thread: an error, not a crash. + "thread_panic" => { + let _ = std::thread::Builder::new() + .name("worker".into()) + .spawn(|| panic!("worker-thread-panic")) + .unwrap() + .join(); + std::thread::sleep(Duration::from_millis(300)); + } + "segv" => fault_null(), + "unwinding_panic" => { + let _keep = &guard; + panic!("health-e2e-deliberate-panic"); + } + "exit_without_shutdown" => std::process::exit(3), + other => panic!("unknown kind {other}"), + } + drop(guard); + std::process::exit(0); +} + +// --------------------------------------------------------------------------- +// The parent. +// --------------------------------------------------------------------------- + +struct Case { + kind: &'static str, + /// Whether the child is expected to die rather than exit 0. + dies: bool, + status: SessionStatus, + min_errors: u64, +} + +fn cases() -> Vec { + vec![ + Case { + kind: "clean", + dies: false, + status: SessionStatus::Ok, + min_errors: 0, + }, + Case { + kind: "errored", + dies: false, + status: SessionStatus::Errored, + min_errors: 2, + }, + Case { + kind: "thread_panic", + dies: false, + status: SessionStatus::Errored, + min_errors: 1, + }, + Case { + kind: "segv", + dies: true, + status: SessionStatus::Crashed, + min_errors: 0, + }, + Case { + kind: "unwinding_panic", + dies: true, + status: SessionStatus::Crashed, + min_errors: 0, + }, + Case { + kind: "exit_without_shutdown", + dies: true, + status: SessionStatus::Abnormal, + min_errors: 0, + }, + ] +} + +/// Relaunch over `dir` (recovering whatever died), and return every finished +/// session, then stop cleanly. +fn relaunch(dir: &Path) -> Vec { + let guard = launch(dir.to_str().unwrap(), true); + std::thread::sleep(Duration::from_millis(300)); + Bugsee::flush(Duration::from_secs(10)); + let all = Bugsee::finished_sessions(); + drop(guard); + all +} + +fn spawn_child(kind: &str, dir: &Path) -> std::process::Output { + std::process::Command::new(std::env::current_exe().unwrap()) + .env(KIND_ENV, kind) + .env(DIR_ENV, dir) + .output() + .expect("spawn child") +} + +fn run_case(case: &Case) -> Result<(), String> { + let dir = std::env::temp_dir().join(format!( + "bugsee-health-e2e-{}-{}", + case.kind, + bugsee::core::util::random_hex(6) + )); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + let result = (|| { + let out = spawn_child(case.kind, &dir); + if out.status.success() == case.dies { + return Err(format!( + "child exit {:?} (wanted dies={}); stderr={}", + out.status, + case.dies, + String::from_utf8_lossy(&out.stderr) + )); + } + // Whatever happened, the child was generation 1. + let first = relaunch(&dir); + let mine: Vec<_> = first.iter().filter(|r| r.generation == 1).collect(); + if mine.len() != 1 { + return Err(format!("want one record for the child, got {first:?}")); + } + let r = mine[0]; + if r.status != case.status { + return Err(format!( + "status {:?}, want {:?}: {r:?}", + r.status, case.status + )); + } + if r.errors < case.min_errors { + return Err(format!( + "errors {}, want >= {}: {r:?}", + r.errors, case.min_errors + )); + } + if r.app_version.as_deref() != Some("4.5.6") || r.app_build.as_deref() != Some("789") { + return Err(format!("release lost: {r:?}")); + } + if r.duration_ms.is_none() || r.ended_at.is_none() { + return Err(format!("no end or duration: {r:?}")); + } + + // A later launch must not count the dead session again, nor change it. + let second = relaunch(&dir); + let again: Vec<_> = second.iter().filter(|x| x.generation == 1).collect(); + if again.len() != 1 || *again[0] != *r { + return Err(format!("session counted or changed twice: {second:?}")); + } + // The relaunches themselves ended cleanly. + let others = second.iter().filter(|x| x.generation != 1); + for o in others { + if o.status != SessionStatus::Ok { + return Err(format!("a clean relaunch is not ok: {o:?}")); + } + } + Ok(()) + })(); + let _ = std::fs::remove_dir_all(&dir); + result +} + +/// Off by default: a run without release health leaves no record, and a later +/// run that has it on does not invent one for it. +fn run_off() -> Result<(), String> { + let dir = std::env::temp_dir().join(format!( + "bugsee-health-e2e-off-{}", + bugsee::core::util::random_hex(6) + )); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + let result = (|| { + let out = spawn_child("health_off", &dir); + if !out.status.success() { + return Err(format!("child failed: {:?}", out.status)); + } + if dir.join("health").exists() { + return Err("release health off still wrote records".into()); + } + let all = relaunch(&dir); + if all.iter().any(|r| r.generation == 1) { + return Err(format!("a record appeared for the untracked run: {all:?}")); + } + Ok(()) + })(); + let _ = std::fs::remove_dir_all(&dir); + result +} + +fn main() { + if let Ok(kind) = std::env::var(KIND_ENV) { + let dir = std::env::var(DIR_ENV).expect("data dir"); + run_child(&kind, &dir); + } + let filter = std::env::args().skip(1).find(|a| !a.starts_with('-')); + let mut failed = Vec::new(); + let mut run = |name: &str, f: &dyn Fn() -> Result<(), String>| { + if filter.as_deref().is_some_and(|x| !name.contains(x)) { + return; + } + match f() { + Ok(()) => println!("test release_health::{name} ... ok"), + Err(e) => { + println!("test release_health::{name} ... FAILED\n {e}"); + failed.push(name.to_string()); + } + } + }; + for case in cases() { + run(case.kind, &|| run_case(&case)); + } + run("off_by_default", &run_off); + if !failed.is_empty() { + println!("\nrelease health failures: {failed:?}"); + std::process::exit(1); + } +} From c4f81d6874cdba947cb6b0a1e7f29bb5dea628ee Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:33:18 +0500 Subject: [PATCH 25/52] fix(core): make the lock-less heartbeat robust to clock skew and slow launches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A heartbeat whose mtime is in the future (file server clock ahead) counts as fresh, like marker_is_fresh; only a demonstrably quiet beat condemns a session. The staleness window is 3 minutes, above NFS attribute-cache lifetimes. - Launch-time recovery (before the worker loop starts) now keeps the beat going. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/recovery.rs | 27 +++++++++++++++++++++------ crates/bugsee-core/src/runtime.rs | 21 ++++++++++++++++++--- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 8ad8f9c..2154dd9 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -207,7 +207,7 @@ Crash recovery needs the data directory to **outlive the process**: on a contain - **Costs of per-process locks, handled in `claim.rs`.** They do not exclude threads, so a registry of held paths does (keyed by pid and spin-locked with the holder's pid stamped in, so a `fork` child that inherits it mid-update breaks the lock instead of deadlocking). POSIX drops a process's locks on a file when *any* descriptor for it closes, so a lock file is never opened twice by one process: the registry is checked before each open, and a claim closes its descriptor before leaving the registry. - **Ordering at launch.** `Session::begin` creates the marker (`O_EXCL`), takes `.lock`, then writes the pid. Until the pid is written the marker is empty and fresh, which peers skip even if the lock reads free. A lock that cannot be held with kernel enforcement (contended past a short retry, or the filesystem rejects locks) is *removed*: an existing but unheld lock file reads as "that session died", so peers fall back to the marker's pid instead. On unix a probe asks `F_GETLK` and never takes the lock, so it cannot steal one from a launching peer. - **Unclean shutdown.** If the worker or uploader thread is detached at shutdown it may still be writing the session, so the session's lock is kept until the process exits (the kernel then drops it) rather than released on drop. -- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown (as it does for a leftover lock file that cannot be opened, e.g. on a read-only volume, so a crash is never hidden by it). A session that could not take a lock keeps a `.beat` heartbeat file, touched every 2 s by the worker: a beat younger than 60 s (generous, since NFS caches attributes) means alive whatever pid the marker names, which is the only signal that works across pid namespaces without locks; with no beat the marker's pid is probed as before. The price is that a crash on such a filesystem is recovered only after its heartbeat has been quiet for a minute, so a faster restart defers it to a later launch. Under the recovery claim the liveness decision is re-run, so a session that finished launching since the scan is never torn down. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. +- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown (as it does for a leftover lock file that cannot be opened, e.g. on a read-only volume, so a crash is never hidden by it). A session that could not take a lock keeps a `.beat` heartbeat file, touched every 2 s by the worker (and during launch-time recovery): a beat younger than 3 minutes, or dated in the future (generous: NFS caches attributes for up to a minute and its mtime is the server's clock) means alive whatever pid the marker names, which is the only signal that works across pid namespaces without locks; with no beat the marker's pid is probed as before. The price is that a crash on such a filesystem is recovered only after its heartbeat has been quiet for three minutes, so a faster restart defers it to a later launch. Under the recovery claim the liveness decision is re-run, so a session that finished launching since the scan is never torn down. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. - **Paths.** The in-process registry canonicalises the lock file's directory, so `/tmp` vs `/private/tmp` or relative vs absolute paths are one claim. A `fork` child clears the registry (`claim::reset_after_fork`, from the fork hook) before claiming anything. - **Stale scans.** After winning a session claim the marker is re-checked, since a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s: a peer's in-flight `enqueue` looks like an orphan, and reaping it lost the report. Lock files of finished items and dead sessions are swept when free. - **Compatibility.** The `.alive` marker is still a bare pid and the lock files are new names, so older builds sharing the directory keep working. They do not take part in the locking, so a mixed-version directory (a rolling upgrade) gives no exactly-once guarantee between old and new processes, and a new process treats an old build's session by its pid as before. Run one SDK version per directory for the guarantee. diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index eeae9ea..526717b 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -128,8 +128,9 @@ fn session_is_crashed(data_dir: &Path, generation: u64, marker: &Path, self_pid: /// How recently a lock-less session's heartbeat file must have been touched for /// the session to count as alive. Generous because network filesystems cache file -/// attributes for up to about a minute. -const HEARTBEAT_STALE: std::time::Duration = std::time::Duration::from_secs(60); +/// attributes (Linux NFS `acregmax` defaults to 60 s), so a live beat can look much +/// older than it is. +const HEARTBEAT_STALE: std::time::Duration = std::time::Duration::from_secs(180); /// The heartbeat file beside a marker. Only a session that could not take a lock /// (the filesystem rejects them) keeps one: it is touched every couple of seconds. @@ -145,14 +146,15 @@ pub(crate) fn heartbeat_path(marker: &Path) -> PathBuf { /// only signal that works across pid namespaces when there are no locks. Without /// one, the recorded pid is probed, which is only meaningful within one pid /// namespace. A crash is therefore recovered only once the heartbeat has gone -/// quiet for [`HEARTBEAT_STALE`] (a restart faster than that defers recovery to a +/// quiet for [`HEARTBEAT_STALE`] (three minutes; a restart faster than that defers recovery to a /// later launch). fn legacy_marker_is_dead(marker: &Path, self_pid: u32) -> bool { + // A beat whose age cannot be established (a modification time in the future: + // the file server's clock runs ahead of ours) counts as fresh, like + // `marker_is_fresh`: only a beat demonstrably quiet condemns a session. if std::fs::metadata(heartbeat_path(marker)) .and_then(|m| m.modified()) - .ok() - .and_then(|t| t.elapsed().ok()) - .is_some_and(|age| age < HEARTBEAT_STALE) + .is_ok_and(|t| t.elapsed().map(|age| age < HEARTBEAT_STALE).unwrap_or(true)) { return false; } @@ -888,6 +890,19 @@ mod tests { assert_eq!(gens(&dir), vec![1], "quiet heartbeat: crashed"); } + #[test] + fn a_future_dated_heartbeat_counts_as_alive() { + let dir = tmp(); + let m = marker(&dir, 1, "4294967290"); + let beat = heartbeat_path(&m); + std::fs::write(&beat, "").unwrap(); + let f = std::fs::OpenOptions::new().write(true).open(&beat).unwrap(); + f.set_modified(std::time::SystemTime::now() + std::time::Duration::from_secs(3600)) + .unwrap(); + drop(f); + assert!(gens(&dir).is_empty()); + } + #[test] fn claim_pending_gives_up_if_the_session_took_its_lock_since_the_scan() { let dir = tmp(); diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index c97288d..aef7dab 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -900,6 +900,9 @@ fn spawn_pipeline( let worker_data_dir = data_dir.map(Path::to_path_buf); let worker_drain_dir = drain_dir.map(Path::to_path_buf); let marker = data_dir.map(|d| Session::marker_path(d, generation)); + // Recovery can run long (a host `before_send`, many sessions); it keeps this + // session's heartbeat going meanwhile, as the worker loop is not yet running. + let beat_marker = marker.clone(); let worker_upload_tx = upload_tx.clone(); let worker = std::thread::Builder::new() @@ -930,13 +933,13 @@ fn spawn_pipeline( // it must, or that crash waits for the master to restart. Claims // keep siblings from recovering the same session twice. if let Some(dir) = &worker_drain_dir { - run_recovery(&worker_shared, dir, generation); + run_recovery(&worker_shared, dir, generation, beat_marker.as_deref()); } // Best effort, once: sessions a previous default location left // behind. No generation of ours lives there, so none is skipped. for dir in &legacy_data_dirs { queue::gc_orphans(dir); - run_recovery(&worker_shared, dir, u64::MAX); + run_recovery(&worker_shared, dir, u64::MAX, beat_marker.as_deref()); } })); legacy_ready.store(true, Ordering::SeqCst); @@ -965,8 +968,20 @@ fn spawn_pipeline( /// Recover any prior generation that ended abnormally, queueing it for delivery. /// The crashed session's state is discarded ONLY after the report is durably /// enqueued β€” a failure leaves it on disk for a later attempt (no crash loss). -fn run_recovery(shared: &Shared, data_dir: &Path, current_generation: u64) { +fn run_recovery( + shared: &Shared, + data_dir: &Path, + current_generation: u64, + heartbeat_of: Option<&Path>, +) { + let beat = || { + if let Some(marker) = heartbeat_of { + touch_heartbeat(marker); + } + }; + beat(); for pending in recovery::find_pending(data_dir, current_generation) { + beat(); // Several processes share this directory (a prefork master and its // workers, above all): exactly one recovers a given session. let Some(_claim) = recovery::claim_pending(data_dir, &pending) else { From 6dfc9c834b2d4f5542f9e6ce560fdf3201c5aae5 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:34:35 +0500 Subject: [PATCH 26/52] fix(core): keep unique crashes the launch pass cannot queue; scan newest-first without a cap; keep attempts progressing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Admission::Full no longer discards the session; it is left to the worker - find_pending_bounded reads all marker names, sorts, then probes (no capped readdir), and includes sessions with prior attempts so a fault during recovery still reaches MAX_RECOVERY_ATTEMPTS πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 4 +- crates/bugsee-core/src/queue.rs | 4 +- crates/bugsee-core/src/recovery.rs | 71 +++++++++++++-------- crates/bugsee-core/src/runtime.rs | 25 ++++---- crates/bugsee-core/tests/launch_recovery.rs | 34 ++++++++++ crates/bugsee/src/options.rs | 6 +- 6 files changed, 97 insertions(+), 47 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 249cc71..a7cfb35 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -222,8 +222,8 @@ Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.r - **Threading contract.** The pass runs on a helper thread (`bugsee-launch-recovery`) and `launch` waits at most `launch_recovery_budget` (500 ms default, `LaunchOptions::launch_recovery_budget`; zero disables the pass) by polling a flag (never parking: a parker used before a `fork` is inherited by the child, and on macOS the child then traps in libdispatch when it parks). That is a hard upper bound whatever a `before_send` callback does: on timeout `launch` returns and the pass finishes in the background, so a host that holds its own lock while calling `launch` cannot deadlock against a callback that takes it. The callback can run on an SDK thread before `launch` returns; this is documented on `LaunchOptions::before_send`. - **Ordering with the crash handlers.** The pass is the last thing `launch_with` does, after the native handler and the panic hook are installed and outside every lock, so a fault inside it is recorded with a stack against THIS session (its marker and `crash.info` path are already this generation's) and reported by the next launch. The previous session's evidence is untouched until its report is queued, and `bump_attempts` runs before processing, so a session that crashes the assemble path is abandoned after `MAX_RECOVERY_ATTEMPTS` launches instead of looping. On a re-launch within one process the native handler is replaced with one for the new generation before the pass runs; a fault in the pass is therefore the new generation's. - **Worker coordination.** With the pass armed, the worker holds its own recovery back until the pass is done (bounded by budget plus 5 s, so a wedged callback cannot stall capture for good), then skips generations the pass claimed (`launch_attempted`, recorded only after winning the #14 claim). Claims keep the two exclusive in any interleaving. A host using `Recorder::launch` directly with a non-zero budget must call `recover_on_launch_path` or the worker waits that grace out; the core default budget is zero, i.e. no pass. -- **Bounds.** At most 8 sessions (`launch_recovery_max_sessions`), no new session after the budget (checked between sessions, so one huge session can overrun it, and the newest is always attempted), a scan of at most 4096 session markers (and four times as many directory entries; `.lock` and `.attempts` files do not use the cap). **Newest first**: the latest crash is the one most likely to belong to the build now running, and a crash loop makes only recent evidence reachable. What the pass leaves is recovered by the worker's unbounded pass, so nothing is lost; recovered reports may therefore upload newest first, which is harmless because the server deduplicates by signature. -- **Crash-loop escalation.** A fast loop never lets the uploader finish, so the server's `12004` verdict and the local blacklist are never reached. In the pass, a recovered crash whose signature set equals one already queued is dropped (the first copy is kept; once it has been delivered a recurrence is queued again), and recovered crashes stop being queued when the queue holds 64 reports or 64 MiB (`queue::admit_recovered`; the newest is dropped). Not done: a repeat counter on the kept report. The worker pass does not dedup or cap. +- **Bounds.** At most 8 sessions (`launch_recovery_max_sessions`), no new session after the budget (checked between sessions, so one huge session can overrun it, and the newest is always attempted), and at most 16 times that many candidates probed. Every session marker NAME is read (`readdir` order is not generation order, so a capped scan could miss the newest), sorted, and only the newest are probed; up to as many older sessions that already have an `.attempts` sidecar are included too, so a session that faults during recovery itself, whose failure leaves a newer session behind each time, still reaches `MAX_RECOVERY_ATTEMPTS` and is abandoned. **Newest first**: the latest crash is the one most likely to belong to the build now running, and a crash loop makes only recent evidence reachable. What the pass leaves is recovered by the worker's unbounded pass, so nothing is lost; recovered reports may therefore upload newest first, which is harmless because the server deduplicates by signature. +- **Crash-loop escalation.** A fast loop never lets the uploader finish, so the server's `12004` verdict and the local blacklist are never reached. In the pass, a recovered crash whose signature set equals one already queued is dropped (the first copy is kept; once it has been delivered a recurrence is queued again), and the launch pass stops queuing recovered crashes when the queue holds 64 reports or 64 MiB, counting every queued report (`queue::admit_recovered`). A unique crash the cap refuses is NOT discarded: its evidence stays and it is left to the worker's pass, which does not cap (the cap bounds launch-path work, not total retention). Not done: a repeat counter on the kept report. The worker pass does not dedup or cap. - **Scope.** A recovered crash carries no scope (email, labels, attributes) from the new process, on either path: the scope describes the new run, and merging it made the two paths disagree. Persisting scope per session is possible, not done. - **Not covered.** Sessions of the legacy data directory are recovered by the worker only. A fork child (prefork worker boot loop) recovers on its worker, not on a launch path, so a prefork worker that dies before its worker thread finishes recovery leaves the evidence for a later launch (nothing is lost, it is reported late); the master's pass covers only the master's own launch. In memory mode with a recovery directory (#9), a crash that cannot be queued on disk is delivered from memory and its evidence stays, so a later launch reports it again (the backend deduplicates). - **Measured** (`launch_recovery.rs::measure_launch_latency`, Apple silicon, synthetic log-only windows at the default 8 MiB / 60 s caps, a pass that waits for completion): empty directory about 0.1 to 0.3 ms; one crashed session about 7 ms release, 21 ms debug; eight identical sessions about 17 ms release, 43 ms debug. The eight-session figure undercounts assembly, since seven of them are dropped as duplicates before being assembled; scrubbing (#11) will add per-session cost. `Recorder::launch` itself is about 0.5 to 0.8 ms of that. diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index a5ccf91..b01b7a9 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -125,8 +125,8 @@ pub enum Admission { /// local blacklist is ever reached. Instead the first copy stays queued and /// later copies with the identical, non-empty signature set are dropped; once it /// has been delivered a recurrence is queued again. The queue is also capped -/// (entries and bytes): beyond it the newest recovered crash is dropped rather -/// than grow the directory without bound. Reports without signatures are never +/// (entries and bytes): beyond it the caller keeps the crash's evidence for the +/// background pass rather than queue more on the launch path. Reports without signatures are never /// treated as duplicates. pub fn admit_recovered(data_dir: &Path, signatures: &[String]) -> Admission { let wanted: BTreeSet<&String> = signatures.iter().collect(); diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 76f7b1a..2ec80cd 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -61,20 +61,21 @@ pub struct RecoveredReport { /// because it launched earlier) has a live marker that is *not* a crash. Its /// marker is left in place for a future launch to recover once the peer is gone. pub fn find_pending(data_dir: &Path, current_generation: u64) -> Vec { - find_pending_bounded(data_dir, current_generation, usize::MAX, usize::MAX) + find_pending_bounded(data_dir, current_generation, usize::MAX) } /// Like [`find_pending`], for the launch path: newest generation first, and -/// bounded. At most `scan_cap` directory entries are read and at most `limit` -/// sessions returned, so a directory a crash loop has filled does not make launch -/// scan, probe and stat without end. Names are sorted before any file is touched, -/// so only the newest candidates cost a probe. A scan cut off by `scan_cap` may -/// miss the newest session; the worker's unbounded pass picks up whatever the -/// bounded one left. +/// bounded in what it probes. Every session marker NAME is read (cheap, and the +/// newest cannot be known without reading them all: `readdir` order is not +/// generation order), sorted, and only then are files touched: the newest `limit` +/// pending sessions, plus up to `limit` older ones that already have an +/// `.attempts` sidecar. The latter keep a session that faults during recovery +/// itself moving toward `MAX_RECOVERY_ATTEMPTS`, instead of being starved by the +/// newer sessions each failed recovery leaves behind. At most `limit * 16` +/// candidates are probed. pub fn find_pending_bounded( data_dir: &Path, current_generation: u64, - scan_cap: usize, limit: usize, ) -> Vec { let mut pending = Vec::new(); @@ -85,12 +86,7 @@ pub fn find_pending_bounded( }; let self_pid = std::process::id(); let mut candidates: Vec = Vec::new(); - // The cap counts session markers only: a session also leaves `.lock` and - // `.attempts` files that must not use it up. - // Entries read are bounded too (a session leaves about three files), so a - // directory full of unrelated files cannot make the scan long either. - let mut seen = 0usize; - for entry in read.flatten().take(scan_cap.saturating_mul(4)) { + for entry in read.flatten() { let name = entry.file_name().to_string_lossy().into_owned(); let Some(gen_str) = name.strip_suffix(".alive") else { continue; @@ -98,19 +94,20 @@ pub fn find_pending_bounded( let Ok(generation) = gen_str.parse::() else { continue; }; - if seen >= scan_cap { - break; - } - seen += 1; if generation < current_generation { candidates.push(generation); } } candidates.sort_unstable_by(|a, b| b.cmp(a)); - for generation in candidates { - if pending.len() >= limit { + let (mut fresh, mut retries) = (0usize, 0usize); + for generation in candidates.into_iter().take(limit.saturating_mul(16)) { + if fresh >= limit && retries >= limit { break; } + let retrying = attempts_path(data_dir, generation).exists(); + if (retrying && retries >= limit) || (!retrying && fresh >= limit) { + continue; + } let marker = sessions.join(format!("{generation}.alive")); // A session whose `.lock` exists is arbitrated by that OS lock: held // means a live process (in any pid namespace), free means it died. The @@ -142,6 +139,11 @@ pub fn find_pending_bounded( let minidump = exists_opt(parts_dir.join(MINIDUMP_NAME)); let crash_info = exists_opt(parts_dir.join(CRASH_INFO_NAME)); let panic_info = exists_opt(parts_dir.join(PANIC_INFO_NAME)); + if retrying { + retries += 1; + } else { + fresh += 1; + } pending.push(PendingSession { generation, parts_dir, @@ -817,19 +819,32 @@ mod tests { for g in [3u64, 9, 1, 7, 5] { marker(&dir, g, "0"); } - let got = |current, scan, limit| -> Vec { - find_pending_bounded(&dir, current, scan, limit) + let got = |current, limit| -> Vec { + find_pending_bounded(&dir, current, limit) .iter() .map(|p| p.generation) .collect() }; - assert_eq!(got(u64::MAX, 100, 100), vec![9, 7, 5, 3, 1]); - assert_eq!(got(u64::MAX, 100, 2), vec![9, 7]); + assert_eq!(got(u64::MAX, 100), vec![9, 7, 5, 3, 1]); + assert_eq!(got(u64::MAX, 2), vec![9, 7]); // Only older generations than the caller's own are candidates. - assert_eq!(got(7, 100, 100), vec![5, 3, 1]); - // A scan cut short reads fewer entries (which ones is directory order). - assert!(got(u64::MAX, 2, 100).len() <= 2); - assert!(got(u64::MAX, 0, 100).is_empty()); + assert_eq!(got(7, 100), vec![5, 3, 1]); + assert!(got(u64::MAX, 0).is_empty()); + } + + #[test] + fn a_session_with_failed_attempts_is_not_starved_by_newer_ones() { + let dir = tmp(); + for g in 1..=6u64 { + marker(&dir, g, "0"); + } + // Session 2 was tried before (and the process died doing it). + std::fs::write(attempts_path(&dir, 2), "1").unwrap(); + let got: Vec = find_pending_bounded(&dir, u64::MAX, 2) + .iter() + .map(|p| p.generation) + .collect(); + assert_eq!(got, vec![6, 5, 2]); } #[test] diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index c25eb0b..0654121 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1152,18 +1152,15 @@ fn wait_for_launch_pass(shared: &Shared) { } } -/// Directory entries the launch-path pass reads before giving up on finding more. -const LAUNCH_SCAN_CAP: usize = 4096; - /// The launch-path recovery pass (issue #10): turn the newest crashed sessions /// into durable queue entries BEFORE `launch` returns, so a host that dies again /// within milliseconds (a crash loop under a supervisor) cannot outrun the report /// of the crash before it. Only the queueing is synchronous; upload stays on the /// uploader thread. /// -/// Bounded three ways: at most `max_sessions` are recovered, the scan reads at -/// most [`LAUNCH_SCAN_CAP`] directory entries, and no new session is started once -/// `budget` has elapsed. The budget is checked between sessions, so one session +/// Bounded: at most `max_sessions` (plus as many already-attempted ones, so a +/// session that faults in recovery still reaches `MAX_RECOVERY_ATTEMPTS`) are +/// recovered, and no new session is started once `budget` has elapsed. The budget is checked between sessions, so one session /// can overrun it; the newest session is always attempted whatever the budget, /// since it is the one a crash loop is about to make unreachable. Newest first: /// the latest crash is the one most likely to be on the build now running, and @@ -1174,12 +1171,7 @@ const LAUNCH_SCAN_CAP: usize = 4096; fn run_launch_recovery(shared: &Shared, plan: &LaunchPlan) { let started = Instant::now(); let mut claimed = 0; - for pending in recovery::find_pending_bounded( - &plan.dir, - plan.generation, - LAUNCH_SCAN_CAP, - plan.max_sessions, - ) { + for pending in recovery::find_pending_bounded(&plan.dir, plan.generation, plan.max_sessions) { if claimed > 0 && started.elapsed() >= plan.budget { break; } @@ -1219,10 +1211,17 @@ fn recover_one( queue::Admission::Admit => {} // A crash loop reports the same crash on every start: the first one // is already queued, and another copy adds nothing the server keeps. - queue::Admission::Duplicate | queue::Admission::Full => { + queue::Admission::Duplicate => { recovery::discard(data_dir, pending); return; } + // A unique crash the queue has no room for on the launch path: + // keep its evidence and leave it to the worker's pass, which does not + // cap, rather than destroy it. + queue::Admission::Full => { + shared.launch_attempted.lock().remove(&pending.generation); + return; + } } } diff --git a/crates/bugsee-core/tests/launch_recovery.rs b/crates/bugsee-core/tests/launch_recovery.rs index 4f2844d..96b887d 100644 --- a/crates/bugsee-core/tests/launch_recovery.rs +++ b/crates/bugsee-core/tests/launch_recovery.rs @@ -185,6 +185,40 @@ fn the_session_cap_limits_the_launch_path() { assert_eq!(transport.uploaded_bundles.lock().unwrap().len(), 4); } +/// A unique crash the launch pass has no queue room for is not destroyed: the +/// worker's (uncapped) pass queues it. +#[test] +fn a_full_queue_defers_a_unique_crash_to_the_worker_instead_of_discarding_it() { + let tmp = TempDir::new(); + let dir = tmp.0.join("data"); + seed_crashed(&dir, 1..=1); + for i in 0..queue::MAX_QUEUED_REPORTS { + queue::enqueue( + &dir, + &bugsee_core::AssembledReport { + bundle_name: format!("{}.bundle.zip", bugsee_core::util::random_hex(10)), + zip: b"z".to_vec(), + request_json: format!("{{\"signatures\":[\"other-{i}\"]}}").into_bytes(), + }, + ) + .unwrap(); + } + + let rec = + Recorder::launch(config(&dir, Duration::from_millis(500)), Arc::new(Offline)).unwrap(); + rec.recover_on_launch_path(); + // Not queued by the launch pass, and its evidence is still on disk. + assert_eq!(queue::list_pending(&dir).len(), queue::MAX_QUEUED_REPORTS); + assert!(dir.join("sessions").join("1.alive").exists()); + + let deadline = Instant::now() + Duration::from_secs(10); + while queue::list_pending(&dir).len() <= queue::MAX_QUEUED_REPORTS { + assert!(Instant::now() < deadline, "the worker never recovered it"); + std::thread::sleep(Duration::from_millis(20)); + } + drop(rec); +} + /// A hook that never returns must not hold `launch` past the budget. #[test] fn a_blocked_before_send_cannot_stall_the_launch_path() { diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index e2c014d..bbc6dca 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -240,9 +240,11 @@ impl LaunchOptions { /// elapsed (the background worker recovers the rest, so nothing is lost). A /// session's cost grows with its capture window (`max_window`/`max_bytes`). /// Repeated identical crashes are queued once while the first is undelivered, - /// and recovered crashes stop being queued at + /// and at /// [`MAX_QUEUED_REPORTS`](bugsee_core::queue::MAX_QUEUED_REPORTS) reports or - /// [`MAX_QUEUED_BYTES`](bugsee_core::queue::MAX_QUEUED_BYTES) bytes. + /// [`MAX_QUEUED_BYTES`](bugsee_core::queue::MAX_QUEUED_BYTES) bytes queued + /// the launch pass stops adding more (a crash it leaves is kept on disk and + /// recovered in the background). /// `Duration::ZERO` disables the pass: everything is recovered in the /// background. Recovered crashes are queued newest first, so they may upload /// in that order; the server deduplicates by signature whatever the order. From 460e093df3a8e7a2bcfe56e31e61ba51cb8ef197 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:44:36 +0500 Subject: [PATCH 27/52] fix(health): address review of release health (#12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Register the backend session eagerly at launch when release_health is on, finish records on any non-panicking stop, ignore stale panic snapshots, memory-mode and empty-dir guards, ack tombstones, generation floor, no file I/O under the facade lock, gc, monotonic durations, unstable/hidden API, docs moved to section 10. Refs #12 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 28 ++- PROGRESS.md | 2 +- crates/bugsee-core/src/health.rs | 249 +++++++++++++++++---- crates/bugsee-core/src/lib.rs | 1 + crates/bugsee-core/src/recovery.rs | 91 +++++++- crates/bugsee-core/src/runtime.rs | 110 +++++++-- crates/bugsee-core/src/session.rs | 3 + crates/bugsee-core/tests/release_health.rs | 187 ++++++++++++++++ crates/bugsee-panic/src/lib.rs | 10 + crates/bugsee/src/api.rs | 31 +-- crates/bugsee/src/options.rs | 18 +- crates/bugsee/tests/release_health.rs | 20 +- 12 files changed, 666 insertions(+), 84 deletions(-) create mode 100644 crates/bugsee-core/tests/release_health.rs diff --git a/DESIGN.md b/DESIGN.md index 7dd121c..cea6cbf 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -95,6 +95,7 @@ let _guard = Bugsee::launch_with(LaunchOptions::new("APP_TOKEN") .sample_rate(1.0).native_crash_capture(true).system_telemetry(true) .before_send(|r: &mut ReportMeta| { r.severity = Severity::High; true }) .before_breadcrumb(|c| Some(c)))?; +// Experimental, off by default: `release_health(bool)` (see Β§10, Release health). // Real builder set (see `crates/bugsee/src/options.rs`): data_dir, max_window, // max_bytes, max_events, endpoint, with_transport, before_send, // before_breadcrumb, sample_rate, native_crash_capture, system_telemetry, @@ -234,10 +235,6 @@ Tests: `bugsee-core/tests/launch_recovery.rs` (a real abort-after-launch crash l **Snapshot (no seal, no copy):** pin `[start,end]` (`end`=snapshot instant, `start`=`endβˆ’window`) + **hard-link current parts including the active one** into the report dir (`PartStore::snapshot_into`, after a `flush`). At export, read linked parts and include only entries with `start ≀ timestamp ≀ end` (monotonic order β†’ early-stop at the tail; the self-terminating `RecordIter` cleanly ignores any torn trailing write on the active part). Bounds become `manifest.time.start/end`. Cost: 2 timestamps + O(#parts) `link` syscalls. Fallback to copy on FAT/exFAT/cross-volume (capability-probed once). Reflink (APFS `clonefile` / Linux `FICLONE`) a noted future tier. -**Release health: sessions and crash-free rates (#12), SDK half only.** What the backend does today (`appserver` `components/shared/statistics`): it counts one session per session-create (`POST /v2/sessions`) and one crash per crash issue, and computes crash-free sessions as `(sessions - crashes) / sessions`, per day and per release (app `version`/`build`). There is **no endpoint** that accepts a session end, status, duration or error count from an SDK, no `ok/errored/crashed/abnormal` model, and no spec (`specs/`, `report-bundle-structure/`) mentions one. The SDK therefore does not send anything. `LaunchOptions::release_health(true)` (off by default) keeps a durable local record per session, `health/.live` while running and `health/.json` when finished, readable with `Bugsee::finished_sessions()` / `session_health()` and cleared with `acknowledge_session`. Rules: a session is one SDK launch (desktop app, server and CLI alike; no idle timeout); a handled error, including a panic on a non-main thread, makes it `errored`, never `crashed`; a clean stop (guard drop, not while unwinding) finishes it `ok`/`errored`; a session that died is finished by the next launch's recovery as `crashed` (native signal, minidump or uncaught-panic evidence) or `abnormal` (none), inside `recovery::discard`, under the recover claim and whatever became of its report, so each dead session is counted once across relaunches, and the finished record is written (rename) before the marker goes so a retry is idempotent. Memory mode (#9) tracks nothing (a crash there is invisible, so every session would read healthy). A fork child is a session of its own; the parent's is not touched. The crash handler and panic hook write nothing new (the crash path is unchanged and allocation-free); the status is inferred at the next launch. Error counts are rewritten for the first 8 errors and then at powers of two, so a session that died keeps an approximate count and a dead session's end time is the last file activity in its capture parts. - -*Blocked on the backend:* (1) an ingest endpoint or a field on session-create/close for `{session_id, started_at, ended_at, duration, status, errors, release, environment}`, authenticated like the other calls, idempotent on `session_id`; (2) how crashed-vs-abnormal feeds `crashes_free_sessions` (today only crash *issues* count, and a session whose crash report was deduplicated or blacklisted still counts as crash-free server-side, which the SDK records as crashed); (3) user-level rates need a stable anonymous user id (none exists in the Rust SDK); (4) sampling for servers with many short sessions, to be decided with the endpoint; (5) a `Transport::report_session` method and offline queueing of finished records (the durable queue and `.recover.lock` claims are ready to carry them) once the shape exists. - ## 9. Failure pipeline **Boundary guards (`bugsee-panic`):** every exported FFI fn / SDK thread root / task root runs inside `catch_unwind` + a TLS scope guard (`sdk_scope_depth`, `foreign_callback_depth`, `operation_id`). A panic never unwinds across `extern "C"`. Caught panic β†’ quarantine (Class A–D) β†’ non-fatal event (`type:"error"`, `handled:true`). @@ -293,6 +290,27 @@ That last case previously produced **no report whatsoever** β€” the SDK shut dow **TLS:** optional cert-pinning to `*.bugsee.com` for API calls (not presigned PUT). +### Release health: sessions and crash-free rates (#12) β€” experimental + +**What the backend does today** (`appserver` `components/shared/statistics/index.js`, `formatter.js`): a session is counted when it is created (`POST /v2/sessions`, `statsInc.sessions`), a crash when a crash **report** is received (`logIssue`, `issues.crashes`), and `crashes_free_sessions = (sessions - crashes) / sessions`, per day and per release (app `version`/`build`). There is **no endpoint** that accepts a session end, status, duration or error count, no `ok/errored/crashed/abnormal` model, and no spec (`specs/`, `report-bundle-structure/`) mentions one. + +**The design gap that matters most.** The Rust transport registers its session lazily, inside `deliver`, so only a process that uploads something ever creates a backend session. A healthy run that reports nothing is never counted while its crash report is, and the rate collapses toward 0%. With `LaunchOptions::release_health(true)` (experimental, **off by default**) the uploader thread therefore registers the session **eagerly, once per launch**, right after launch: best effort, never on the caller's thread, never an error for launch, and offline-tolerant (a failure is dropped, retried at most every 30 s, and the first delivery registers lazily if it is still unregistered). The token is the one `deliver` uses, so there is exactly one `POST /v2/sessions` per online launch. The session of a run that died is registered by that run itself; its crash report arrives with the next process, which counts it as a crash on its own day and release (the backend has no link between the two), as for any recovered crash. A fork child inherits the parent's token and registers nothing of its own. + +**Local record (no backend half).** The same option keeps a durable record per session in `health/`: `.live` while running and `.json` when finished (`ok`, `errored`, `crashed`, `abnormal`; duration, error count, release). Nothing is sent. It is unstable by design: the types live in a `doc(hidden)` module, are `#[non_exhaustive]`, and the facade offers only `doc(hidden)` accessors (`Bugsee::__finished_sessions` and friends); there is no stable API for a contract that does not exist. + +Rules. +- A session is one SDK launch (desktop app, server and CLI alike; no idle timeout). +- A handled error, including a panic on a non-main thread, makes it `errored`, never `crashed`. Counting happens with an atomic on the reporting path (no file I/O under the facade's lock); the worker's rotation tick persists it. +- A stop that is not a death by panic finishes the record (`ok`/`errored`) whatever the worker and uploader acked. A detached thread leaves the liveness marker, so the next launch recovers the session too and finds the finished record (idempotent). +- A session that died is finished by the next launch's recovery as `crashed` (native crash info, a minidump, or a panic snapshot that is **fresh**: the session did not keep capturing more than 10 s after it) or `abnormal` (none). This happens in `recovery::discard`, under the recover claim and whatever became of the report (delivered, deduplicated, dropped by `before_send`). A worker-thread panic deletes its snapshot after the inline report, and `build_report` applies the same freshness rule, so a later SIGKILL is not misread as that panic's crash. +- Exactly once: the finished record is written by rename before the marker goes and rewritten if unreadable; `acknowledge` leaves a `.acked` tombstone that a late finalizer honours; gc removes tombstones and orphan `.live` files once their marker is gone. Records raise the generation floor, so a number is not reused. +- Memory mode (#9) tracks nothing and lists nothing (a crash there is invisible, so every session would read healthy); an empty data directory is refused. A fork child is a session of its own and never finishes the parent's. +- Durations of clean sessions come from a monotonic clock; a dead session's end is the later of its newest capture-part mtime and its live record's last write. + +Documented limits. Records of legacy and extra recovery directories are not surfaced; the newest 256 finished records are kept and older ones are dropped even if unacknowledged; right after launch a recovered earlier-run crash may not be listed yet (recovery past the launch-path budget is asynchronous); a server that is ended by SIGTERM, `exec` or `_exit` without dropping its guard is `abnormal`, which inflates the crashed rate, so drop the guard from the termination handler (the shutdown contract, #8, `install_termination_flush`); error counts of a session that died are approximate (up to one tick behind). The crash handler and panic hook add no work of their own for this; the one new hook-side action is the worker-thread snapshot deletion above, and the inline worker-thread report (already there) now also bumps the error counter. + +**Blocked on the backend.** (1) An ingest endpoint, or fields on session-create/close, for `{session_id, started_at, ended_at, duration, status, errors, release, environment}`, authenticated like the other calls and idempotent on `session_id`; until then crashed and abnormal sessions can only be counted through crash reports. (2) How `crashed`/`abnormal` should feed `crashes_free_sessions`: today a session whose crash report was deduplicated (`12003`) or blacklisted (`12004`) counts as crash-free server-side while the SDK records it `crashed`. (3) Crash-free **users**: the backend's user counts key on `environment.hardware.device_id` (`session.service.js`), which the Rust environment does not send; a stable anonymous id is needed first. (4) Sampling for servers with many short sessions (each launch is one `POST /v2/sessions`), to be decided with the endpoint. (5) A `Transport::report_session` method and offline queueing of finished records, which the durable queue and recover-lock claims can carry once the shape exists. + ## 11. Concurrency & threading | Role | Count | Owns | Does | @@ -403,6 +421,8 @@ worker jobs (alongside `ios`/`javascript`), not Android's `mappings/` + ProGuard `mapping.*` path β€” the worker resolves a Rust minidump against that unified store, so the two halves must agree or every Rust minidump stays unsymbolicated. +**Still open, release health (#12):** the backend has no session end/status/duration ingest and counts a session only at session-create; see Β§10, *Release health*, for the contract the Rust SDK would need and why crash-free users also needs `hardware.device_id`. + **Still open (not required for ingestion):** the `viewer` dashboard has no Rust-specific rendering yet (reports display through the generic managed path), no Rust entry exists in the app-create wizard, and there is still no cross-repo diff --git a/PROGRESS.md b/PROGRESS.md index dc147ef..a8bb83b 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -515,7 +515,7 @@ Rust and are worth evaluating before building one. - Phase 2: subsystem quarantine (auto-disable a repeatedly-faulting integration). - Phase 3: best-effort in-flight flush at crash; full out-of-process minidump (desktop) β€” see the backend note below, which is the same machinery. -- Phase 4: sessions / release-health: SDK half done (#12: local session records, off by default; nothing is sent, the backend has no session-health ingest, see DESIGN.md Β§8). +- Phase 4: sessions / release-health: SDK half done, experimental and off by default (#12: eager session registration plus local session records; the backend has no session-health ingest, see DESIGN.md Β§10). - Phase 5: general PII scrubber, rate limits. - Phase 6: iOS/Android target builds; Swift/Kotlin language wrappers. - CI: ~~cross-platform matrix~~ **done**; loom/miri/fuzz jobs still open β€” fuzzing is diff --git a/crates/bugsee-core/src/health.rs b/crates/bugsee-core/src/health.rs index c42166e..09d8a8c 100644 --- a/crates/bugsee-core/src/health.rs +++ b/crates/bugsee-core/src/health.rs @@ -8,16 +8,17 @@ //! Session health: a durable record of how each run of the application ended //! (issue #12), the SDK half of release health / crash-free sessions. //! -//! **There is no backend half yet.** The Bugsee backend derives crash-free -//! sessions itself: it counts one session per `POST /v2/sessions` and one crash -//! per crash issue (`appserver` `shared/statistics`: `sessions`, -//! `issues.crashes`; crash-free = `(sessions - crashes) / sessions`), and it has -//! no endpoint that accepts a session's end, status, duration or error count. -//! Nothing here is sent anywhere. The records are kept locally, off by default -//! ([`HealthTracker`] exists only when the host asked for release health), and -//! are exposed through [`finished`] / [`acknowledge`] so a host can use them -//! today and a transport can ship them once a contract exists. DESIGN.md -//! (release health) lists what the backend still has to define. +//! **Experimental and unstable; there is no backend contract for it.** The +//! backend derives crash-free sessions itself: it counts one session per +//! `POST /v2/sessions` and one crash per crash report received (`appserver` +//! `shared/statistics`: `sessions`, `issues.crashes`; crash-free = +//! `(sessions - crashes) / sessions`), and it has no endpoint that accepts a +//! session's end, status, duration or error count. These records are therefore +//! kept locally and never sent; what release health does send is the session +//! registration itself, made eagerly at launch (see `runtime`), because a +//! session the backend never hears of cannot be counted. Off by default. +//! DESIGN.md (release health) lists what the backend still has to define. This +//! module is `doc(hidden)` and may change without notice. //! //! # Lifecycle //! @@ -42,15 +43,28 @@ //! A finished record is named by its generation and written by whoever holds the //! session (the live process at a clean stop, or the process holding //! `sessions/.recover.lock` at recovery), via a temp file and a rename. A -//! finalizer that finds the record already there only removes the `.live` file, -//! so a retry after a crash between the two steps cannot flip a status or count -//! a session twice. Generations are never reused in a data directory. +//! finalizer that finds the record already there (or its `.acked` +//! tombstone, left by [`acknowledge`]) only removes the `.live` file, so a retry +//! after a crash between the two steps cannot flip a status or count a session +//! twice. Generations are not reused: the generation floor also reads this +//! directory, so even a wiped `gen` hint and marker tree cannot hand out a +//! generation that has a record. +//! +//! Limits: records of the legacy and extra recovery directories are not +//! surfaced (only the primary data directory's are); [`MAX_FINISHED`] prunes the +//! oldest records even if never acknowledged; an unacknowledged record of a +//! session recovered right at launch may not be listed until that recovery has +//! run (it is asynchronous past the launch-path budget); and a server that is +//! stopped by SIGTERM, `exec`s or calls `_exit` without dropping its guard ends +//! its session `abnormal`, not `ok`: drop the guard from the termination +//! handler (see the shutdown contract, #8). //! //! None of this runs on the crash path: the signal handler and panic hook write //! nothing here. The crash is inferred at the next launch from the evidence the //! handlers already leave. use std::path::{Path, PathBuf}; +use std::time::Instant; use serde::{Deserialize, Serialize}; @@ -59,13 +73,11 @@ use crate::model::environment::Environment; const HEALTH_DIR: &str = "health"; /// Newest finished records kept; older ones are pruned when a session finishes. pub const MAX_FINISHED: usize = 256; -/// Error counts are rewritten to disk for the first few errors and then at each -/// power of two, so an error storm costs a handful of small writes, not one each. -const EAGER_WRITES: u64 = 8; /// How a session ended. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] +#[non_exhaustive] pub enum SessionStatus { /// Ended cleanly with no handled errors. Ok, @@ -81,6 +93,7 @@ pub enum SessionStatus { /// The durable record of one session. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[non_exhaustive] pub struct SessionRecord { /// Schema version of this record. pub version: u32, @@ -122,10 +135,22 @@ fn finished_path(data_dir: &Path, generation: u64) -> PathBuf { dir(data_dir).join(format!("{generation}.json")) } +fn acked_path(data_dir: &Path, generation: u64) -> PathBuf { + dir(data_dir).join(format!("{generation}.acked")) +} + +/// The temp file for `path`: its own name plus `.tmp`, so `.live` and +/// `.json` never share one. +fn tmp_path(path: &Path) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(".tmp"); + PathBuf::from(name) +} + /// Write `record` atomically (temp file, then rename). fn write_record(path: &Path, record: &SessionRecord) -> std::io::Result<()> { let bytes = serde_json::to_vec(record).map_err(std::io::Error::other)?; - let tmp = path.with_extension("tmp"); + let tmp = tmp_path(path); std::fs::write(&tmp, bytes)?; std::fs::rename(&tmp, path).inspect_err(|_| { let _ = std::fs::remove_file(&tmp); @@ -140,12 +165,18 @@ fn read_record(path: &Path) -> Option { pub struct HealthTracker { data_dir: PathBuf, record: SessionRecord, + /// Monotonic start, so a clean session's duration survives clock changes. + started: Instant, } impl HealthTracker { /// Start tracking the session `generation` in `data_dir`. `None` when the /// record cannot be written (the session then simply is not tracked). pub fn start(data_dir: &Path, generation: u64, env: &Environment) -> Option { + // An empty path would resolve against the working directory. + if data_dir.as_os_str().is_empty() { + return None; + } std::fs::create_dir_all(dir(data_dir)).ok()?; let record = SessionRecord { version: 1, @@ -166,20 +197,31 @@ impl HealthTracker { Some(HealthTracker { data_dir: data_dir.to_path_buf(), record, + started: Instant::now(), }) } - /// Count one handled error. - pub fn note_error(&mut self) { - self.record.errors = self.record.errors.saturating_add(1); - self.record.status = SessionStatus::Errored; - let n = self.record.errors; - if n <= EAGER_WRITES || n.is_power_of_two() { - let _ = write_record( - &live_path(&self.data_dir, self.record.generation), - &self.record, - ); + /// Bring the record up to `errors` handled errors (the process-wide counter, + /// which the reporting path bumps without touching the disk) and persist it + /// if that changed anything. Called from the worker's tick, never from a + /// caller's thread. + pub fn sync_errors(&mut self, errors: u64) { + if errors == self.record.errors { + return; + } + self.record.errors = errors; + if errors > 0 { + self.record.status = SessionStatus::Errored; } + let _ = write_record( + &live_path(&self.data_dir, self.record.generation), + &self.record, + ); + } + + /// The directory this session's records live in. + pub fn data_dir(&self) -> &Path { + &self.data_dir } /// The record as it stands. @@ -189,11 +231,17 @@ impl HealthTracker { /// End the session cleanly: publish the finished record, then drop the live /// one. - pub fn finish_clean(&self) { - let now = crate::util::epoch_ms(); + pub fn finish_clean(&self, errors: u64) { + let duration = self.started.elapsed().as_millis() as i64; let mut record = self.record.clone(); - record.ended_at = Some(now); - record.duration_ms = Some((now - record.started_at).max(0)); + record.errors = errors; + record.status = if errors > 0 { + SessionStatus::Errored + } else { + SessionStatus::Ok + }; + record.ended_at = Some(record.started_at + duration); + record.duration_ms = Some(duration); publish(&self.data_dir, record); } } @@ -212,13 +260,21 @@ pub fn finalize_dead( let Some(mut record) = read_record(&live) else { // Unreadable or absent: nothing to report, but do not leave junk. let _ = std::fs::remove_file(&live); - let _ = std::fs::remove_file(live.with_extension("tmp")); + let _ = std::fs::remove_file(tmp_path(&live)); return; }; record.status = status; + // The last sign of life: the newest capture part, or the live record's own + // last write (an error count), whichever is later. + let live_written = std::fs::metadata(&live) + .and_then(|m| m.modified()) + .ok() + .and_then(|m| m.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_millis() as i64); let end = last_activity - .unwrap_or(record.started_at) - .max(record.started_at); + .into_iter() + .chain(live_written) + .fold(record.started_at, i64::max); record.ended_at = Some(end); record.duration_ms = Some(end - record.started_at); publish(data_dir, record); @@ -227,12 +283,15 @@ pub fn finalize_dead( /// Write the finished record unless one exists, then remove the live one. fn publish(data_dir: &Path, record: SessionRecord) { let done = finished_path(data_dir, record.generation); - if !done.exists() { + // Settled already: the host took it (tombstone) or an intact record exists. + // An unreadable one (zero-length after a power loss) is rewritten. + let settled = acked_path(data_dir, record.generation).exists() || read_record(&done).is_some(); + if !settled { let _ = write_record(&done, &record); } // Only drop the live record once the finished one is durable: a failure // leaves it for the next recovery, which finds the finished one and stops. - if done.exists() { + if settled || read_record(&done).is_some() { let _ = std::fs::remove_file(live_path(data_dir, record.generation)); } prune(data_dir); @@ -263,6 +322,9 @@ fn finished_generations(data_dir: &Path) -> Vec { /// Finished sessions in `data_dir`, oldest first, that have not been /// [`acknowledge`]d. pub fn finished(data_dir: &Path) -> Vec { + if data_dir.as_os_str().is_empty() { + return Vec::new(); + } let mut gens = finished_generations(data_dir); gens.sort_unstable(); gens.into_iter() @@ -271,10 +333,57 @@ pub fn finished(data_dir: &Path) -> Vec { } /// Forget a finished session once the host (or a future transport) has taken it. +/// Leaves a `.acked` tombstone, so a finalizer that still finds the +/// session's marker (a recovery racing the acknowledgement) cannot write the +/// record again and have it counted twice. [`gc`] drops the tombstone once the +/// marker is gone. pub fn acknowledge(data_dir: &Path, generation: u64) { + if data_dir.as_os_str().is_empty() { + return; + } + let _ = std::fs::write(acked_path(data_dir, generation), b""); let _ = std::fs::remove_file(finished_path(data_dir, generation)); } +/// Sweep leftovers of generations below `current` that no longer have a liveness +/// marker (`alive` says whether one does): tombstones, whose job is done, and +/// `.live` records no recovery will ever finish. Called from +/// [`crate::recovery::gc_orphans`]. +pub(crate) fn gc(data_dir: &Path, current: u64, alive: impl Fn(u64) -> bool) { + let Ok(read) = std::fs::read_dir(dir(data_dir)) else { + return; + }; + for entry in read.flatten() { + let name = entry.file_name().to_string_lossy().into_owned(); + let Some((stem, ext)) = name.split_once('.') else { + continue; + }; + let Ok(generation) = stem.parse::() else { + continue; + }; + if generation >= current || alive(generation) { + continue; + } + if matches!(ext, "acked" | "live" | "live.tmp" | "json.tmp") { + let _ = std::fs::remove_file(entry.path()); + } + } +} + +/// The highest generation any record here names, for the generation floor. +pub(crate) fn max_generation(data_dir: &Path) -> u64 { + std::fs::read_dir(dir(data_dir)) + .into_iter() + .flatten() + .flatten() + .filter_map(|e| { + let name = e.file_name().to_string_lossy().into_owned(); + name.split('.').next()?.parse::().ok() + }) + .max() + .unwrap_or(0) +} + /// The newest modification time (epoch ms) under `dir`, to `depth` levels down. /// A dead session's last-activity stamp: the capture parts are written as it /// runs. @@ -282,6 +391,14 @@ pub(crate) fn newest_mtime(dir: &Path, depth: u32) -> Option { let mut newest = None::; for entry in std::fs::read_dir(dir).ok()?.flatten() { let Ok(meta) = entry.metadata() else { continue }; + // Crash evidence is written at the moment of death, not by capture. + let name = entry.file_name(); + if matches!( + name.to_str(), + Some("panic.info" | "crash.info" | "crash.modules") + ) { + continue; + } let mut t = meta .modified() .ok() @@ -322,7 +439,7 @@ mod tests { let d = tmp(); let t = HealthTracker::start(&d, 1, &env()).unwrap(); assert!(live_path(&d, 1).exists()); - t.finish_clean(); + t.finish_clean(0); assert!(!live_path(&d, 1).exists()); let all = finished(&d); assert_eq!(all.len(), 1); @@ -337,9 +454,8 @@ mod tests { fn handled_errors_make_the_session_errored_not_crashed() { let d = tmp(); let mut t = HealthTracker::start(&d, 1, &env()).unwrap(); - t.note_error(); - t.note_error(); - t.finish_clean(); + t.sync_errors(2); + t.finish_clean(2); let r = &finished(&d)[0]; assert_eq!((r.status, r.errors), (SessionStatus::Errored, 2)); } @@ -370,7 +486,7 @@ mod tests { fn acknowledged_and_old_records_are_dropped() { let d = tmp(); for g in 1..=(MAX_FINISHED as u64 + 3) { - HealthTracker::start(&d, g, &env()).unwrap().finish_clean(); + HealthTracker::start(&d, g, &env()).unwrap().finish_clean(0); } let all = finished(&d); assert_eq!(all.len(), MAX_FINISHED); @@ -378,4 +494,55 @@ mod tests { acknowledge(&d, 4); assert_eq!(finished(&d).len(), MAX_FINISHED - 1); } + + #[test] + fn an_acknowledged_session_is_not_counted_again_by_a_late_finalizer() { + let d = tmp(); + let t = HealthTracker::start(&d, 2, &env()).unwrap(); + t.finish_clean(0); + acknowledge(&d, 2); + assert!(finished(&d).is_empty()); + // The marker survived (a recovery racing the acknowledgement): the + // finalizer finds the tombstone and writes nothing. + let _again = HealthTracker::start(&d, 2, &env()); + finalize_dead(&d, 2, SessionStatus::Crashed, None); + assert!(finished(&d).is_empty()); + assert!(!live_path(&d, 2).exists()); + // Once the marker is gone the tombstone is swept. + gc(&d, 10, |_| false); + assert!(!acked_path(&d, 2).exists()); + } + + #[test] + fn gc_keeps_what_a_live_marker_or_current_generation_still_needs() { + let d = tmp(); + HealthTracker::start(&d, 1, &env()).unwrap(); + HealthTracker::start(&d, 2, &env()).unwrap(); + HealthTracker::start(&d, 3, &env()).unwrap(); + // 1: marker present (a peer); 2: no marker, below current (junk); 3: current. + gc(&d, 3, |g| g == 1); + assert!(live_path(&d, 1).exists()); + assert!(!live_path(&d, 2).exists()); + assert!(live_path(&d, 3).exists()); + } + + #[test] + fn a_zero_length_finished_record_is_rewritten() { + let d = tmp(); + std::fs::create_dir_all(dir(&d)).unwrap(); + std::fs::write(finished_path(&d, 4), b"").unwrap(); + HealthTracker::start(&d, 4, &env()).unwrap().finish_clean(0); + assert_eq!(finished(&d).len(), 1); + } + + #[test] + fn records_raise_the_generation_floor_and_temp_files_do_not_collide() { + let d = tmp(); + HealthTracker::start(&d, 7, &env()).unwrap(); + assert_eq!(max_generation(&d), 7); + assert_ne!(tmp_path(&live_path(&d, 7)), tmp_path(&finished_path(&d, 7))); + // A wiped hint and marker tree still cannot reuse generation 7. + let session = crate::session::Session::begin(&d).unwrap(); + assert_eq!(session.generation(), 8); + } } diff --git a/crates/bugsee-core/src/lib.rs b/crates/bugsee-core/src/lib.rs index 899705b..ad85ee4 100644 --- a/crates/bugsee-core/src/lib.rs +++ b/crates/bugsee-core/src/lib.rs @@ -21,6 +21,7 @@ pub mod capture; pub mod claim; pub mod errors; pub mod fork; +#[doc(hidden)] pub mod health; pub mod model; pub mod panic_info; diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 577887d..c644934 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -240,7 +240,13 @@ pub fn build_report(pending: &PendingSession, timestamp: i64) -> RecoveredReport // shutdown when `Recorder::drop` sees `thread::panicking()`, i.e. // the process was unwinding out of `main` because of this panic. // So we report the panic itself rather than a generic abnormal exit. - (false, Some(info)) => build_fatal_panic(info, timestamp), + // + // ...unless the session kept capturing long after the snapshot, in + // which case the panic was survived and something else ended it. + (false, Some(info)) if panic_is_recent(pending, &info) => { + build_fatal_panic(info, timestamp) + } + (false, Some(_)) => build_abnormal_exit(timestamp), (false, None) => build_abnormal_exit(timestamp), } } @@ -377,13 +383,36 @@ pub fn discard(data_dir: &Path, pending: &PendingSession) { /// [`build_report`] reports as a crash), `Abnormal` when the process just /// vanished (SIGKILL, OOM kill, power loss). fn dead_status(pending: &PendingSession) -> crate::health::SessionStatus { - if pending.minidump.is_some() || pending.crash_info.is_some() || pending.panic_info.is_some() { + let lone_panic = pending + .panic_info + .as_ref() + .and_then(|p| PanicInfo::read_from(p)) + .is_some_and(|info| panic_is_recent(pending, &info)); + if pending.minidump.is_some() || pending.crash_info.is_some() || lone_panic { crate::health::SessionStatus::Crashed } else { crate::health::SessionStatus::Abnormal } } +/// How long after a panic snapshot the session may keep writing and still be +/// the one the panic killed: unwinding out of `main` and the recorder's final +/// flush take well under this. +const PANIC_TAIL_MS: i64 = 10_000; + +/// Whether a panic snapshot with no native crash beside it can be what ended +/// this session. A snapshot is written for every panic and deleted only when a +/// boundary of ours catches it (or the hook reports a worker-thread panic), so +/// one left by a panic the app survived, caught by a foreign `catch_unwind`, is +/// still there when the process is later SIGKILLed. A session that kept +/// capturing long after the snapshot was not killed by it. +fn panic_is_recent(pending: &PendingSession, info: &PanicInfo) -> bool { + match crate::health::newest_mtime(&pending.parts_dir, 3) { + Some(last) => info.timestamp == 0 || last - info.timestamp <= PANIC_TAIL_MS, + None => true, + } +} + /// The recovery-attempt-counter sidecar path for a generation. fn attempts_path(data_dir: &Path, generation: u64) -> PathBuf { data_dir @@ -456,6 +485,7 @@ pub fn gc_orphans(data_dir: &Path, current_generation: u64) { } } } + crate::health::gc(data_dir, current_generation, alive); if let Ok(read) = std::fs::read_dir(&sessions) { for entry in read.flatten() { let name = entry.file_name().to_string_lossy().into_owned(); @@ -855,6 +885,18 @@ mod tests { crash_info: None, panic_info: panic_info.then(|| dir.join("panic.info")), }; + if panic_info { + crate::panic_info::PanicInfo { + reason: "fatal".into(), + file: None, + line: 0, + column: 0, + timestamp: crate::util::epoch_ms(), + frames: Vec::new(), + } + .write_to(&dir.join("panic.info")) + .unwrap(); + } discard(&dir, &pending); // A repeat (the same session recovered again) changes nothing. discard(&dir, &pending); @@ -869,6 +911,51 @@ mod tests { ); } + #[test] + fn a_stale_panic_snapshot_does_not_turn_a_later_kill_into_a_crash() { + use crate::health::SessionStatus; + let dir = tmp(); + let parts = dir.join("parts").join("1"); + std::fs::create_dir_all(&parts).unwrap(); + // A worker-thread panic long ago (its snapshot was never deleted), then + // the session kept capturing until it was killed. + let snapshot = dir.join("panic.info"); + PanicInfo { + reason: "old".into(), + file: None, + line: 0, + column: 0, + timestamp: 1_000, + frames: Vec::new(), + } + .write_to(&snapshot) + .unwrap(); + std::fs::write(parts.join("part"), b"x").unwrap(); + let pending = PendingSession { + generation: 1, + parts_dir: parts.clone(), + minidump: None, + crash_info: None, + panic_info: Some(snapshot.clone()), + }; + assert_eq!(dead_status(&pending), SessionStatus::Abnormal); + let report = build_report(&pending, 5_000); + assert!(String::from_utf8_lossy(&report.crash_json).contains("AppExit")); + + // A fresh snapshot (the panic that killed it) is still a crash. + PanicInfo { + reason: "fatal".into(), + file: None, + line: 0, + column: 0, + timestamp: crate::util::epoch_ms(), + frames: Vec::new(), + } + .write_to(&snapshot) + .unwrap(); + assert_eq!(dead_status(&pending), SessionStatus::Crashed); + } + #[test] fn the_bounded_scan_is_newest_first_and_capped() { let dir = tmp(); diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index c8c1657..73ddfe8 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -298,6 +298,13 @@ struct Shared { /// on and the session is on disk (memory mode has no recovery, so a crash /// would never be seen and every session would read as healthy). health: ForkLock>, + /// Handled errors this session has seen. Counted with an atomic so the + /// reporting path (a caller's thread, under the facade's lock) never does + /// file I/O; the worker's tick persists it into `health`. + health_errors: AtomicU64, + /// Register the backend session at launch instead of with the first report + /// (release health on): see [`register_eagerly`]. + eager_session: bool, } /// What the launch-path recovery pass needs, fixed at launch. @@ -432,6 +439,8 @@ impl Recorder { launch_done: AtomicBool::new(false), launch_wait_ms: AtomicU64::new(0), health: ForkLock::new(None), + health_errors: AtomicU64::new(0), + eager_session: config.release_health, }); crate::fork::install_fork_hook(); @@ -595,6 +604,7 @@ impl Recorder { // back to memory tracks nothing. { let mut health = self.shared.health.lock(); + self.shared.health_errors.store(0, Ordering::SeqCst); let enabled = health.is_some(); *health = match (&session, enabled) { (Some(s), true) => crate::health::HealthTracker::start( @@ -725,17 +735,14 @@ impl Recorder { /// The running session's health record, when release health is on. pub fn session_health(&self) -> Option { - self.shared - .health - .lock() - .as_ref() - .map(|h| h.record().clone()) + self.handle().session_health() } /// Finished sessions recorded in this data directory (clean ends and - /// recovered crashes), oldest first, until [`crate::health::acknowledge`]d. + /// recovered crashes), oldest first, until acknowledged. Empty unless the + /// session is tracked on disk. pub fn finished_sessions(&self) -> Vec { - crate::health::finished(&self.data_dir) + self.handle().finished_sessions() } /// Data directory root for this recorder. @@ -830,13 +837,21 @@ impl Drop for Recorder { // A panic that was merely *contained* never reaches here while // unwinding, and its snapshot is deleted by `report_caught`, so this // cannot turn a handled panic into a phantom crash. + // + // The health record is the exception: it is finished whenever the stop + // is not a death by panic, whatever the threads acked. A detached + // thread leaves the marker, so the next launch recovers the session + // too, and finds this record already there (`finalize_dead` is + // idempotent), so the clean end is counted once and as what it was. + if !std::thread::panicking() { + if let Some(health) = self.shared.health.lock().take() { + health.finish_clean(self.shared.health_errors.load(Ordering::SeqCst)); + } + } if worker_acked && uploader_acked && !std::thread::panicking() { if let Some(session) = &self.session { // The finished health record first, then the marker: a death in // between leaves a marker that recovery settles idempotently. - if let Some(health) = self.shared.health.lock().take() { - health.finish_clean(); - } session.end(); } } else if !(worker_acked && uploader_acked) { @@ -862,6 +877,38 @@ pub struct RecorderHandle { } impl RecorderHandle { + /// See [`Recorder::session_health`]. The error count is read live. + pub fn session_health(&self) -> Option { + let mut record = self.shared.health.lock().as_ref()?.record().clone(); + record.errors = self.shared.health_errors.load(Ordering::SeqCst); + Some(record) + } + + /// The data directory of the tracked session, cloned out so file I/O runs + /// outside every lock. `None` when nothing is tracked (release health off, + /// or memory mode, whose recovery-less sessions could never show a crash). + fn health_dir(&self) -> Option { + self.shared + .health + .lock() + .as_ref() + .map(|h| h.data_dir().to_path_buf()) + } + + /// See [`Recorder::finished_sessions`]. + pub fn finished_sessions(&self) -> Vec { + self.health_dir() + .map(|d| crate::health::finished(&d)) + .unwrap_or_default() + } + + /// Forget a finished session (see [`crate::health::acknowledge`]). + pub fn acknowledge_session(&self, generation: u64) { + if let Some(dir) = self.health_dir() { + crate::health::acknowledge(&dir, generation); + } + } + /// See [`Recorder::recover_on_launch_path`]. pub fn recover_on_launch_path(&self) { let Some(plan) = self.shared.launch_plan.lock().take() else { @@ -996,9 +1043,7 @@ fn report_into( // A panic on a non-main thread arrives here as an `Error` too, never as a // crash: the process survives it. if meta.issue_type == IssueType::Error { - if let Some(health) = shared.health.lock().as_mut() { - health.note_error(); - } + shared.health_errors.fetch_add(1, Ordering::Relaxed); } // Every enqueued report increments the counter (the worker decrements on // drain), but only NON-crash reports are subject to the drop-newest cap. @@ -1440,6 +1485,9 @@ fn worker_loop( // elapsed β€” via a message OR a timeout β€” so steady traffic can't starve it. if Instant::now() >= next_rotate { rotate_tick(store.as_mut(), &shared, &mut sampler); + if let Some(health) = shared.health.lock().as_mut() { + health.sync_errors(shared.health_errors.load(Ordering::SeqCst)); + } if let Some(marker) = &marker { ensure_marker(&shared, marker); } @@ -1726,6 +1774,28 @@ fn assemble_from_disk( .ok() } +/// Minimum spacing between eager session-registration attempts while offline. +const EAGER_RETRY: Duration = Duration::from_secs(30); + +/// Register the backend session now, once: a no-op when a token is already held +/// (an earlier attempt, or a delivery, got one), so there is exactly one +/// `POST /v2/sessions` per launch that is online. Never fails the caller: errors +/// are dropped and the lazy path in [`transport::deliver`] registers later. The +/// lock is not held across the network call. +fn register_eagerly(shared: &Shared) { + if shared.session.lock().is_some() { + return; + } + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if let Ok(token) = shared + .transport + .register_session(&shared.app_token, &shared.environment_json) + { + shared.session.lock().get_or_insert(token); + } + })); +} + fn uploader_loop( rx: Receiver, shared: Arc, @@ -1737,6 +1807,16 @@ fn uploader_loop( ) { let poll = Duration::from_secs(5); let mut legacy = legacy_data_dirs; + // Release health: the backend counts a session when it is registered, and + // registration is otherwise lazy (only a delivery registers), so a run that + // never uploads would never be counted. Best effort, on this thread, before + // anything else; failure (offline) is silent and retried at most every + // `EAGER_RETRY` from the poll below, falling back to the lazy path. + let eager = fresh_launch && shared.eager_session; + let mut last_eager = Instant::now(); + if eager { + register_eagerly(&shared); + } // A fresh launch retries any queued report immediately (force), ignoring // backoff scheduled by the prior session. A fork child skips this: the // parent's uploader is already working the shared queue. Guarded so a @@ -1782,6 +1862,10 @@ fn uploader_loop( break; } Err(RecvTimeoutError::Timeout) => { + if eager && last_eager.elapsed() >= EAGER_RETRY { + last_eager = Instant::now(); + register_eagerly(&shared); + } let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { drain_all(&shared, data_dir.as_deref(), &legacy, backoff_base, false) })); diff --git a/crates/bugsee-core/src/session.rs b/crates/bugsee-core/src/session.rs index 6a8218e..225df98 100644 --- a/crates/bugsee-core/src/session.rs +++ b/crates/bugsee-core/src/session.rs @@ -151,6 +151,9 @@ fn floor_generation(data_dir: &Path) -> u64 { Some(".alive"), )); floor = floor.max(max_numeric_entry(&data_dir.join("parts"), None)); + // Session-health records name generations too (a finished or acknowledged + // one must never be shadowed by a reused number). + floor = floor.max(crate::health::max_generation(data_dir)); floor } diff --git a/crates/bugsee-core/tests/release_health.rs b/crates/bugsee-core/tests/release_health.rs new file mode 100644 index 0000000..2ef331a --- /dev/null +++ b/crates/bugsee-core/tests/release_health.rs @@ -0,0 +1,187 @@ +// +// release_health.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// +//! Issue #12 at the recorder level: eager backend-session registration, health +//! records on a slow stop, and memory mode. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use bugsee_core::health::{self, SessionStatus}; +use bugsee_core::model::enums::{IssueType, Severity, TriggerType}; +use bugsee_core::model::report::Source; +use bugsee_core::transport::{Transport, TransportError}; +use bugsee_core::{MockTransport, Recorder, RecorderConfig, ReportMeta}; + +fn dir(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!( + "bugsee-health-core-{name}-{}", + bugsee_core::util::random_hex(6) + )); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn config(dir: &std::path::Path, health: bool) -> RecorderConfig { + let mut c = RecorderConfig::new(dir, "tok"); + c.release_health = health; + c +} + +fn error_meta() -> ReportMeta { + ReportMeta { + issue_type: IssueType::Error, + summary: Some("e".into()), + description: None, + labels: Vec::new(), + severity: Severity::default(), + email: None, + signatures: vec!["sig-health".into()], + source: Source { + trigger: TriggerType::Error, + origin: None, + }, + attrs: Default::default(), + attachments: Vec::new(), + } +} + +fn wait_until(what: &str, f: impl Fn() -> bool) { + let deadline = Instant::now() + Duration::from_secs(10); + while !f() { + assert!(Instant::now() < deadline, "timed out waiting for {what}"); + std::thread::sleep(Duration::from_millis(10)); + } +} + +#[test] +fn release_health_registers_the_backend_session_once_at_launch() { + let d = dir("eager"); + let mock = Arc::new(MockTransport::default()); + let rec = Recorder::launch(config(&d, true), mock.clone()).unwrap(); + // Nothing was reported: only the eager registration can have run. + wait_until("the eager registration", || { + *mock.sessions.lock().unwrap() == 1 + }); + + // A later delivery reuses that session instead of registering another. + rec.report(error_meta(), None); + assert!(rec.flush(Duration::from_secs(10))); + assert_eq!(mock.created_requests.lock().unwrap().len(), 1); + assert_eq!(*mock.sessions.lock().unwrap(), 1, "one session per launch"); +} + +#[test] +fn without_release_health_registration_stays_lazy() { + let d = dir("lazy"); + let mock = Arc::new(MockTransport::default()); + let rec = Recorder::launch(config(&d, false), mock.clone()).unwrap(); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!(*mock.sessions.lock().unwrap(), 0); + drop(rec); +} + +/// Offline at launch, then online: launch is unaffected and the session is still +/// registered exactly once, by the lazy path of the first delivery. +struct FlakyRegister { + inner: MockTransport, + attempts: AtomicU32, + fail_first: u32, +} +impl Transport for FlakyRegister { + fn register_session(&self, t: &str, e: &[u8]) -> Result { + if self.attempts.fetch_add(1, Ordering::SeqCst) < self.fail_first { + return Err(TransportError::Transient("offline".into())); + } + self.inner.register_session(t, e) + } + fn create_issue(&self, t: &str, a: Option<&str>, r: &[u8]) -> Result { + self.inner.create_issue(t, a, r) + } + fn upload_bundle(&self, u: &str, z: &[u8]) -> Result<(), TransportError> { + self.inner.upload_bundle(u, z) + } +} + +#[test] +fn a_failed_eager_registration_never_fails_launch_and_is_not_repeated_by_delivery() { + let d = dir("offline"); + let flaky = Arc::new(FlakyRegister { + inner: MockTransport::default(), + attempts: AtomicU32::new(0), + fail_first: 1, + }); + let started = Instant::now(); + let rec = Recorder::launch(config(&d, true), flaky.clone()).expect("launch must not fail"); + assert!(started.elapsed() < Duration::from_secs(5)); + wait_until("the failed eager attempt", || { + flaky.attempts.load(Ordering::SeqCst) >= 1 + }); + rec.report(error_meta(), None); + assert!(rec.flush(Duration::from_secs(10))); + assert_eq!(flaky.inner.created_requests.lock().unwrap().len(), 1); + assert_eq!(*flaky.inner.sessions.lock().unwrap(), 1); +} + +/// A transport whose delivery hangs longer than a stop waits. +struct Hung; +impl Transport for Hung { + fn register_session(&self, _: &str, _: &[u8]) -> Result { + std::thread::sleep(Duration::from_millis(3500)); + Err(TransportError::Transient("hung".into())) + } + fn create_issue(&self, _: &str, _: Option<&str>, _: &[u8]) -> Result { + Err(TransportError::Transient("hung".into())) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), TransportError> { + Err(TransportError::Transient("hung".into())) + } +} + +#[test] +fn a_stop_whose_uploader_is_hung_still_finishes_the_health_record() { + let d = dir("hung"); + let rec = Recorder::launch(config(&d, true), Arc::new(Hung)).unwrap(); + rec.report(error_meta(), None); + let started = Instant::now(); + drop(rec); // the uploader is stuck in the transport: the stop gives up on it + assert!(started.elapsed() < Duration::from_secs(10)); + let all = health::finished(&d); + assert_eq!(all.len(), 1, "{all:?}"); + assert_eq!(all[0].status, SessionStatus::Errored); + assert_eq!(all[0].errors, 1); +} + +#[test] +fn memory_mode_tracks_nothing_and_lists_nothing() { + let d = dir("memory"); + // A finished record from a disk run must not be listed by a memory-mode run. + let first = Recorder::launch(config(&d, true), Arc::new(MockTransport::default())).unwrap(); + drop(first); + assert_eq!(health::finished(&d).len(), 1); + + let mut cfg = config(&d, true); + cfg.memory_only = Some("test".into()); + let rec = Recorder::launch(cfg, Arc::new(MockTransport::default())).unwrap(); + assert!(rec.session_health().is_none()); + assert!(rec.finished_sessions().is_empty()); + rec.handle().acknowledge_session(1); + drop(rec); + assert_eq!( + health::finished(&d).len(), + 1, + "memory mode must not touch records" + ); +} + +#[test] +fn an_empty_data_dir_is_rejected() { + let env = bugsee_core::Environment::detect_with("0", &Default::default(), &Default::default()); + assert!(health::HealthTracker::start(std::path::Path::new(""), 1, &env).is_none()); + assert!(health::finished(std::path::Path::new("")).is_empty()); +} diff --git a/crates/bugsee-panic/src/lib.rs b/crates/bugsee-panic/src/lib.rs index e0e285d..6a89204 100644 --- a/crates/bugsee-panic/src/lib.rs +++ b/crates/bugsee-panic/src/lib.rs @@ -235,6 +235,16 @@ pub fn install(reporter: Arc) { handled: is_worker, thread: worker_thread, }); + // A worker-thread panic is reported and the process lives + // on: its snapshot must not outlast it, or a later SIGKILL + // would be read as this panic having killed the process. + if is_worker { + if let Some(guard) = SNAPSHOT_PATH.try_lock() { + if let Some(path) = current_snapshot_path(&guard) { + let _ = std::fs::remove_file(path); + } + } + } } } diff --git a/crates/bugsee/src/api.rs b/crates/bugsee/src/api.rs index b268802..c16a816 100644 --- a/crates/bugsee/src/api.rs +++ b/crates/bugsee/src/api.rs @@ -268,24 +268,27 @@ impl Bugsee { }) } - /// The running session's health record, or `None` when the SDK is not - /// launched, release health is off ([`LaunchOptions::release_health`]) or - /// the SDK is in memory mode. - pub fn session_health() -> Option { - Self::with_recorder(|r| r.session_health()).flatten() + // Release-health diagnostics (#12). Unstable and hidden: there is no backend + // contract for session health, so no stable API is promised for it. They + // clone a handle out first, so no file I/O runs under the global lock. + + #[doc(hidden)] + pub fn __session_health() -> Option { + Self::handle()?.session_health() } - /// Sessions of this data directory that have ended, oldest first: clean ends - /// (`ok` / `errored`) and earlier runs this launch recovered (`crashed` / - /// `abnormal`), each exactly once. Take one with - /// [`Bugsee::acknowledge_session`]; the newest 256 are kept. - pub fn finished_sessions() -> Vec { - Self::with_recorder(|r| r.finished_sessions()).unwrap_or_default() + #[doc(hidden)] + pub fn __finished_sessions() -> Vec { + Self::handle() + .map(|h| h.finished_sessions()) + .unwrap_or_default() } - /// Forget a finished session (by its `generation`) once it has been used. - pub fn acknowledge_session(generation: u64) { - Self::with_recorder(|r| bugsee_core::health::acknowledge(r.data_dir(), generation)); + #[doc(hidden)] + pub fn __acknowledge_session(generation: u64) { + if let Some(h) = Self::handle() { + h.acknowledge_session(generation); + } } /// Stop the SDK and flush pending work (dropping the recorder joins the worker). diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index 99b1480..dda9089 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -253,15 +253,17 @@ impl LaunchOptions { self } - /// Record how each session ends (clean, errored, crashed, abnormal exit) in - /// the data directory, for [`Bugsee::finished_sessions`](crate::Bugsee::finished_sessions). - /// Off by default. + /// **Experimental.** Release health: register this launch with the backend + /// as a session immediately (rather than with the first report), so it + /// counts toward crash-free sessions even if the run never uploads + /// anything, and keep a local record of how the session ended (clean, + /// errored, crashed, abnormal exit) in the data directory. Off by default. /// - /// **Local only.** The backend has no endpoint for session health yet (it - /// derives crash-free rates from session creation and crash issues), so - /// nothing is sent. Requires a usable data directory: in memory mode a crash - /// could never be seen, so no session is tracked. See DESIGN.md, release - /// health. + /// The backend has no endpoint for session end/status yet, so the record is + /// local only and not exposed as stable API; the registration is best + /// effort and never delays or fails launch. In memory mode no record is + /// kept (a crash there could never be seen). The behaviour may change when + /// a backend contract exists. See DESIGN.md, release health. pub fn release_health(mut self, enabled: bool) -> Self { self.release_health = enabled; self diff --git a/crates/bugsee/tests/release_health.rs b/crates/bugsee/tests/release_health.rs index b2469f4..4bd9a90 100644 --- a/crates/bugsee/tests/release_health.rs +++ b/crates/bugsee/tests/release_health.rs @@ -78,6 +78,18 @@ fn run_child(kind: &str, dir: &str) -> ! { .join(); std::thread::sleep(Duration::from_millis(300)); } + // The panic's snapshot must not outlive it: the process is then killed, + // which is an abnormal exit, not the panic's crash. The pause lets the + // worker persist the error count before the kill. + "thread_panic_then_exit" => { + let _ = std::thread::Builder::new() + .name("worker".into()) + .spawn(|| panic!("worker-thread-panic")) + .unwrap() + .join(); + std::thread::sleep(Duration::from_millis(1500)); + std::process::exit(3); + } "segv" => fault_null(), "unwinding_panic" => { let _keep = &guard; @@ -122,6 +134,12 @@ fn cases() -> Vec { status: SessionStatus::Errored, min_errors: 1, }, + Case { + kind: "thread_panic_then_exit", + dies: true, + status: SessionStatus::Abnormal, + min_errors: 1, + }, Case { kind: "segv", dies: true, @@ -149,7 +167,7 @@ fn relaunch(dir: &Path) -> Vec { let guard = launch(dir.to_str().unwrap(), true); std::thread::sleep(Duration::from_millis(300)); Bugsee::flush(Duration::from_secs(10)); - let all = Bugsee::finished_sessions(); + let all = Bugsee::__finished_sessions(); drop(guard); all } From 45300246e8108b28a9f2feb3a1e12a2a73f8c5b3 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:52:29 +0500 Subject: [PATCH 28/52] fix(core): tried sessions first, and a queue-cap refusal is not an attempt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - find_pending_bounded returns sessions with prior attempts before newer ones, so MAX_RECOVERY_ATTEMPTS progresses even if recovering the newest keeps faulting - Admission::Full gives the attempt back (undo_attempt) - make the full-queue test race-free πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/recovery.rs | 58 +++++++++++++++++---- crates/bugsee-core/src/runtime.rs | 3 ++ crates/bugsee-core/tests/launch_recovery.rs | 7 ++- 4 files changed, 56 insertions(+), 14 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 6d87306..b8c05f3 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -222,7 +222,7 @@ Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.r - **Threading contract.** The pass runs on a helper thread (`bugsee-launch-recovery`) and `launch` waits at most `launch_recovery_budget` (500 ms default, `LaunchOptions::launch_recovery_budget`; zero disables the pass) by polling a flag (never parking: a parker used before a `fork` is inherited by the child, and on macOS the child then traps in libdispatch when it parks). That is a hard upper bound whatever a `before_send` callback does: on timeout `launch` returns and the pass finishes in the background, so a host that holds its own lock while calling `launch` cannot deadlock against a callback that takes it. The callback can run on an SDK thread before `launch` returns; this is documented on `LaunchOptions::before_send`. - **Ordering with the crash handlers.** The pass is the last thing `launch_with` does, after the native handler and the panic hook are installed and outside every lock, so a fault inside it is recorded with a stack against THIS session (its marker and `crash.info` path are already this generation's) and reported by the next launch. The previous session's evidence is untouched until its report is queued, and `bump_attempts` runs before processing, so a session that crashes the assemble path is abandoned after `MAX_RECOVERY_ATTEMPTS` launches instead of looping. On a re-launch within one process the native handler is replaced with one for the new generation before the pass runs; a fault in the pass is therefore the new generation's. - **Worker coordination.** With the pass armed, the worker holds its own recovery back until the pass is done (bounded by budget plus 5 s, so a wedged callback cannot stall capture for good), then skips generations the pass claimed (`launch_attempted`, recorded only after winning the #14 claim). Claims keep the two exclusive in any interleaving. A host using `Recorder::launch` directly with a non-zero budget must call `recover_on_launch_path` or the worker waits that grace out; the core default budget is zero, i.e. no pass. -- **Bounds.** At most 8 sessions (`launch_recovery_max_sessions`), no new session after the budget (checked between sessions, so one huge session can overrun it, and the newest is always attempted), and at most 16 times that many candidates probed. Every session marker NAME is read (`readdir` order is not generation order, so a capped scan could miss the newest), sorted, and only the newest are probed; up to as many older sessions that already have an `.attempts` sidecar are included too, so a session that faults during recovery itself, whose failure leaves a newer session behind each time, still reaches `MAX_RECOVERY_ATTEMPTS` and is abandoned. **Newest first**: the latest crash is the one most likely to belong to the build now running, and a crash loop makes only recent evidence reachable. What the pass leaves is recovered by the worker's unbounded pass, so nothing is lost; recovered reports may therefore upload newest first, which is harmless because the server deduplicates by signature. +- **Bounds.** At most 8 sessions (`launch_recovery_max_sessions`), no new session after the budget (checked between sessions, so one huge session can overrun it, and the newest is always attempted), and at most 16 times that many candidates probed. Every session marker NAME is read (`readdir` order is not generation order, so a capped scan could miss the newest), sorted, and only the newest are probed; up to as many older sessions that already have an `.attempts` sidecar are included too and are processed FIRST (the worker's pass uses the same order), and a queue-cap refusal gives its attempt back (`undo_attempt`), so a session that faults during recovery itself, whose failure leaves a newer session behind each time, still reaches `MAX_RECOVERY_ATTEMPTS` and is abandoned. **Newest first**: the latest crash is the one most likely to belong to the build now running, and a crash loop makes only recent evidence reachable. What the pass leaves is recovered by the worker's unbounded pass, so nothing is lost; recovered reports may therefore upload newest first, which is harmless because the server deduplicates by signature. - **Crash-loop escalation.** A fast loop never lets the uploader finish, so the server's `12004` verdict and the local blacklist are never reached. In the pass, a recovered crash whose signature set equals one already queued is dropped (the first copy is kept; once it has been delivered a recurrence is queued again), and the launch pass stops queuing recovered crashes when the queue holds 64 reports or 64 MiB, counting every queued report (`queue::admit_recovered`). A unique crash the cap refuses is NOT discarded: its evidence stays and it is left to the worker's pass, which does not cap (the cap bounds launch-path work, not total retention). Not done: a repeat counter on the kept report. The worker pass does not dedup or cap. - **Scope.** A recovered crash carries no scope (email, labels, attributes) from the new process, on either path: the scope describes the new run, and merging it made the two paths disagree. Persisting scope per session is possible, not done. - **Not covered.** Sessions of the legacy data directory are recovered by the worker only. A fork child (prefork worker boot loop) recovers on its worker, not on a launch path, so a prefork worker that dies before its worker thread finishes recovery leaves the evidence for a later launch (nothing is lost, it is reported late); the master's pass covers only the master's own launch. In memory mode with a recovery directory (#9), a crash that cannot be queued on disk is delivered from memory and its evidence stays, so a later launch reports it again (the backend deduplicates). diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 76e7e33..3a4f3ae 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -99,6 +99,7 @@ pub fn find_pending_bounded( } } candidates.sort_unstable_by(|a, b| b.cmp(a)); + let mut retried = Vec::new(); let (mut fresh, mut retries) = (0usize, 0usize); for generation in candidates.into_iter().take(limit.saturating_mul(16)) { if fresh >= limit && retries >= limit { @@ -116,20 +117,43 @@ pub fn find_pending_bounded( let minidump = exists_opt(parts_dir.join(MINIDUMP_NAME)); let crash_info = exists_opt(parts_dir.join(CRASH_INFO_NAME)); let panic_info = exists_opt(parts_dir.join(PANIC_INFO_NAME)); - if retrying { - retries += 1; - } else { - fresh += 1; - } - pending.push(PendingSession { + let session = PendingSession { generation, parts_dir, minidump, crash_info, panic_info, - }); + }; + if retrying { + retries += 1; + retried.push(session); + } else { + fresh += 1; + pending.push(session); + } + } + // Sessions already tried come FIRST. If recovering the newest keeps killing + // the process, each death leaves another newer session behind and the older + // one would never get its next attempt counted (and never be queued, or + // abandoned at the cap). + retried.extend(pending); + retried +} + +/// Give back the attempt [`bump_attempts`] just counted, for a session that was +/// refused for a reason that says nothing about its evidence (the launch path had +/// no queue room), so such refusals cannot use up `MAX_RECOVERY_ATTEMPTS`. +pub fn undo_attempt(data_dir: &Path, pending: &PendingSession) { + let path = attempts_path(data_dir, pending.generation); + let current = std::fs::read_to_string(&path) + .ok() + .and_then(|s| s.trim().parse::().ok()) + .unwrap_or(0); + if current <= 1 { + let _ = std::fs::remove_file(&path); + } else { + let _ = std::fs::write(&path, (current - 1).to_string()); } - pending } fn exists_opt(p: PathBuf) -> Option { @@ -868,6 +892,20 @@ mod tests { assert!(got(u64::MAX, 0).is_empty()); } + #[test] + fn an_undone_attempt_is_not_counted() { + let dir = tmp(); + marker(&dir, 1, "0"); + let pending = find_pending(&dir, u64::MAX).pop().unwrap(); + assert_eq!(bump_attempts(&dir, &pending), 1); + assert_eq!(bump_attempts(&dir, &pending), 2); + undo_attempt(&dir, &pending); + assert_eq!(bump_attempts(&dir, &pending), 2); + undo_attempt(&dir, &pending); + undo_attempt(&dir, &pending); + assert_eq!(bump_attempts(&dir, &pending), 1); + } + #[test] fn a_session_with_failed_attempts_is_not_starved_by_newer_ones() { let dir = tmp(); @@ -880,7 +918,9 @@ mod tests { .iter() .map(|p| p.generation) .collect(); - assert_eq!(got, vec![6, 5, 2]); + // The one already tried comes first, so it advances even if the newest + // keeps killing the process. + assert_eq!(got, vec![2, 6, 5]); } #[test] diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 43e3768..5439354 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1251,6 +1251,9 @@ fn recover_one( // cap, rather than destroy it. queue::Admission::Full => { shared.launch_attempted.lock().remove(&pending.generation); + // Not a failed attempt: a supervisor crash loop would otherwise + // burn the session's attempts on queue refusals and abandon it. + recovery::undo_attempt(data_dir, pending); return; } } diff --git a/crates/bugsee-core/tests/launch_recovery.rs b/crates/bugsee-core/tests/launch_recovery.rs index 96b887d..cb9f134 100644 --- a/crates/bugsee-core/tests/launch_recovery.rs +++ b/crates/bugsee-core/tests/launch_recovery.rs @@ -207,10 +207,9 @@ fn a_full_queue_defers_a_unique_crash_to_the_worker_instead_of_discarding_it() { let rec = Recorder::launch(config(&dir, Duration::from_millis(500)), Arc::new(Offline)).unwrap(); rec.recover_on_launch_path(); - // Not queued by the launch pass, and its evidence is still on disk. - assert_eq!(queue::list_pending(&dir).len(), queue::MAX_QUEUED_REPORTS); - assert!(dir.join("sessions").join("1.alive").exists()); - + // The launch pass refused it (the worker may already be queueing it, so + // only the outcome is asserted): it ends up queued, not discarded, and + // without having used up its recovery attempts. let deadline = Instant::now() + Duration::from_secs(10); while queue::list_pending(&dir).len() <= queue::MAX_QUEUED_REPORTS { assert!(Instant::now() < deadline, "the worker never recovered it"); From 979c6e8b5836b30d886c04f784cf4c6953508ce1 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:53:55 +0500 Subject: [PATCH 29/52] fix(native): address review of the in-flight flush (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - P1: a dying process skips the worker's recovery scan, orphan GC and legacy drains and reaches worker_loop at once, so the queued Flush is answered. - P2: the flush target stays registered until the worker has acked Stop, so a crash during shutdown still waits for queued entries. - P2: Windows wait uses integer QPC math (no floating point in the filter). πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/crash_flush.rs | 14 +++++++++----- crates/bugsee-core/src/runtime.rs | 20 +++++++++++++++++--- crates/bugsee-core/tests/crash_flush_e2e.rs | 12 ++++++++++++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/crates/bugsee-core/src/crash_flush.rs b/crates/bugsee-core/src/crash_flush.rs index e639433..7ea36eb 100644 --- a/crates/bugsee-core/src/crash_flush.rs +++ b/crates/bugsee-core/src/crash_flush.rs @@ -237,7 +237,8 @@ fn serve_request() { Some(flush) => { std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| flush())).unwrap_or(false) } - // Nothing registered: nothing is queued anywhere that we could drain. + // Nothing registered: no recorder is accepting entries (a recorder stays + // registered until its worker has stopped), so there is nothing to drain. None => true, }; if ok { @@ -445,6 +446,8 @@ mod imp { } } + /// `(counter, frequency)`; integer math only downstream (no floating point + /// in an exception filter). fn now_ticks() -> (i64, i64) { let (mut t, mut f) = (0i64, 1i64); // SAFETY: valid out-pointers; both calls cannot fail on supported Windows. @@ -463,18 +466,19 @@ mod imp { } let ticket = REQUESTED.fetch_add(1, Ordering::AcqRel) + 1; let (start, freq) = now_ticks(); - let budget_ticks = (budget.as_secs_f64() * freq as f64) as i64; + let budget_ms = budget.as_millis().min(i64::MAX as u128 / freq as u128 / 2) as i64; // SAFETY: SetEvent/WaitForSingleObject take no locks we could be holding. unsafe { SetEvent(wake) }; while ACKED.load(Ordering::Acquire) < ticket { let (now, _) = now_ticks(); - let left = budget_ticks - (now - start); - if left <= 0 { + let elapsed_ms = (now - start).saturating_mul(1000) / freq; + let left_ms = budget_ms - elapsed_ms; + if left_ms <= 0 { return false; } // Wake on the helper's ack; the slice cap covers another waiter having // consumed the (auto-reset) signal. - let ms = ((left as f64 / freq as f64 * 1000.0).ceil() as u32).clamp(1, 5); + let ms = left_ms.clamp(1, 5) as u32; // SAFETY: as above. unsafe { WaitForSingleObject(ack, ms) }; } diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 1fbf47e..de32e6e 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -749,8 +749,6 @@ impl Recorder { impl Drop for Recorder { fn drop(&mut self) { - // Stop answering crash handlers before the worker goes away. - drop(self.crash_flush.take()); // A worker still holding its recovery for the launch-path pass must not // make the stop below time out. self.shared.launch_done.store(true, Ordering::SeqCst); @@ -766,6 +764,9 @@ impl Drop for Recorder { let _ = handle.join(); } } + // Only now stop answering crash handlers: until the worker has acked + // `Stop` it can still hold queued entries a crash flush must wait for. + drop(self.crash_flush.take()); let (utx, urx) = channel(); let uploader_acked = self.upload_tx.send(UploadMsg::Stop(utx)).is_ok() && urx.recv_timeout(Duration::from_secs(2)).is_ok(); @@ -1069,6 +1070,9 @@ fn spawn_pipeline( // Best effort, once: sessions a previous default location left // behind. No generation of ours lives there, so none is skipped. for dir in &legacy_data_dirs { + if crate::crash_flush::is_dying() { + break; + } queue::gc_orphans(dir); run_recovery(&worker_shared, dir, u64::MAX, false); } @@ -1119,14 +1123,24 @@ fn install_crash_flush(tx: &Sender) -> Option { /// With `skip_launch_pass`, generations the launch-path pass already claimed are /// left alone. fn run_recovery(shared: &Shared, data_dir: &Path, current_generation: u64, skip_launch_pass: bool) { + // A dying process starts no scan or GC a kill cannot finish: the worker must + // reach its loop to answer the crash handler's in-flight flush (crash_flush). + if crate::crash_flush::is_dying() { + return; + } for pending in recovery::find_pending(data_dir, current_generation) { + if crate::crash_flush::is_dying() { + return; + } if skip_launch_pass && shared.launch_attempted.lock().contains(&pending.generation) { continue; } recover_claimed(shared, data_dir, &pending, false); } // Reap parts/attempts left by cleanly-exited or already-recovered generations. - recovery::gc_orphans(data_dir, current_generation); + if !crate::crash_flush::is_dying() { + recovery::gc_orphans(data_dir, current_generation); + } } /// Claim one pending session and recover it. Returns whether this process won the diff --git a/crates/bugsee-core/tests/crash_flush_e2e.rs b/crates/bugsee-core/tests/crash_flush_e2e.rs index 4c77acb..316bfae 100644 --- a/crates/bugsee-core/tests/crash_flush_e2e.rs +++ b/crates/bugsee-core/tests/crash_flush_e2e.rs @@ -338,6 +338,18 @@ fn a_dying_process_recovers_nothing_and_leaves_the_session_pending() { assert!(dir.0.join("sessions/1.alive").exists(), "still pending"); drop(r1); + // The worker reached its loop at once (no scan/GC ahead of it), so a flush + // request is answered promptly and what was queued is on disk. + let r1 = Recorder::launch(RecorderConfig::new(&dir.0, "TOKEN"), transport.clone()).unwrap(); + for i in 0..200 { + r1.capture(event(i)); + } + let t = Instant::now(); + assert!(flush_in_flight(Duration::from_secs(2))); + assert!(t.elapsed() < Duration::from_millis(500)); + assert_eq!(records_on_disk(&dir.0), 200); + drop(r1); + bugsee_core::crash_flush::reset_dying_for_test(); let r2 = Recorder::launch(RecorderConfig::new(&dir.0, "TOKEN"), transport.clone()).unwrap(); r2.recover_on_launch_path(); From 1d84feb8f754a2153964072a3b550e6510d66f4e Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:56:09 +0500 Subject: [PATCH 30/52] fix: never recover the same directory twice in one launch (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - legacy_is_adoptable compares against the chosen dir AND recovery_dirs by canonical path (Windows: temp fallback is the legacy location) - spawn path dedups legacy/extra dirs against the data dir and each other - seed the in-memory blacklist from legacy/extra dirs too - test: an existing temp tree is offered in exactly one role πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/runtime.rs | 34 ++++++++++++-- crates/bugsee/src/options.rs | 77 +++++++++++++++++++++++++++---- 2 files changed, 98 insertions(+), 13 deletions(-) diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 49fe3dc..15cdd3d 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -419,11 +419,14 @@ impl Recorder { config.upload_backoff_base, config.sampler, true, - config - .legacy_data_dir - .into_iter() - .chain(config.extra_data_dirs) - .collect(), + distinct_extra_dirs( + &data_dir, + config + .legacy_data_dir + .into_iter() + .chain(config.extra_data_dirs) + .collect(), + ), )?; Ok(Recorder { @@ -829,6 +832,23 @@ fn report_into( } } +/// `extra` without duplicates and without `data_dir` itself, compared by +/// canonical path. Recovering one directory twice in a launch would deliver its +/// sessions twice and burn their attempt cap. +fn distinct_extra_dirs(data_dir: &Path, extra: Vec) -> Vec { + let key = |p: &Path| p.canonicalize().unwrap_or_else(|_| p.to_path_buf()); + let mut seen = vec![key(data_dir)]; + let mut out = Vec::new(); + for dir in extra { + let k = key(&dir); + if !seen.contains(&k) { + seen.push(k); + out.push(dir); + } + } + out +} + /// Claim a session and open its part store, or fail without leaving a liveness /// marker behind (a marker with no store would be recovered as a phantom crash). fn open_disk(data_dir: &Path, caps: WindowCaps) -> std::io::Result<(Session, PartStore)> { @@ -925,6 +945,10 @@ fn spawn_pipeline( // Best effort, once: sessions a previous default location left // behind. No generation of ours lives there, so none is skipped. for dir in &legacy_data_dirs { + let known = queue::blacklisted_signatures(dir); + if !known.is_empty() { + worker_shared.mem_blacklist.lock().extend(known); + } queue::gc_orphans(dir); run_recovery(&worker_shared, dir, u64::MAX); } diff --git a/crates/bugsee/src/options.rs b/crates/bugsee/src/options.rs index 6688638..914f50d 100644 --- a/crates/bugsee/src/options.rs +++ b/crates/bugsee/src/options.rs @@ -386,8 +386,16 @@ pub(crate) fn resolve_data_dir( let legacy = temp.join("bugsee").join(&token); // Also with no usable directory: leftovers there are exactly what a launch // that cannot write anywhere new should still deliver. - let chosen = found.as_ref().map(|(_, dir)| dir.as_path()); - let legacy_dir = legacy_is_adoptable(&legacy, chosen).then_some(legacy); + // It must not be a directory we already use or recover from (on Windows the + // temp fallback IS the legacy location): a second recovery of the same + // session in one launch would deliver it twice and burn its attempt cap. + let others: Vec<&Path> = found + .as_ref() + .map(|(_, dir)| dir.as_path()) + .into_iter() + .chain(recovery_dirs.iter().map(PathBuf::as_path)) + .collect(); + let legacy_dir = legacy_is_adoptable(&legacy, &others).then_some(legacy); match found { Some((source, dir)) => DataDirResolution { dir: Some(dir), @@ -515,7 +523,7 @@ fn owned_by_us(_meta: &std::fs::Metadata) -> bool { /// would let someone else's tree, or our own live data directory, be processed /// (recovering our live session as a crash). Paths are compared canonicalised, /// not as strings, so another spelling of our own directory is rejected too. -fn legacy_is_adoptable(legacy: &Path, chosen: Option<&Path>) -> bool { +fn legacy_is_adoptable(legacy: &Path, others: &[&Path]) -> bool { let Some(parent) = legacy.parent() else { return false; }; @@ -525,11 +533,10 @@ fn legacy_is_adoptable(legacy: &Path, chosen: Option<&Path>) -> bool { let Ok(legacy) = legacy.canonicalize() else { return false; }; - match chosen { - // No live directory of ours to collide with. - None => true, - Some(chosen) => chosen.canonicalize().is_ok_and(|c| c != legacy), - } + // An unresolvable path cannot be shown distinct, so it counts as a collision. + others + .iter() + .all(|other| other.canonicalize().is_ok_and(|c| c != legacy)) } /// Prove a file can be created in `dir`. `create_new` with an unpredictable name @@ -919,6 +926,60 @@ mod tests { let _ = std::fs::remove_dir_all(&s); } + /// Platform-agnostic: wherever the temp fallback coincides with the legacy + /// location (Windows), an unwritable existing tree there must be offered for + /// recovery once, as a recovery dir or as the legacy dir, never both. + #[test] + fn an_unwritable_temp_tree_is_recovered_from_exactly_once() { + let s = scratch(); + let tmp = s.join("tmp"); + // Make BOTH the private-named tree and the legacy tree exist; on unix + // they differ, on Windows they are the same directory. + for tree in [ + tmp.join(private_temp_name()).join("tok"), + tmp.join("bugsee").join("tok"), + ] { + std::fs::create_dir_all(&tree).unwrap(); + } + // Block writes everywhere: no home, and the writability probe cannot + // create files because the trees are read-only (unix) -- elsewhere the + // dirs simply win, which is fine: the invariant below still holds. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for tree in [ + tmp.join(private_temp_name()).join("tok"), + tmp.join("bugsee").join("tok"), + ] { + std::fs::set_permissions(&tree, std::fs::Permissions::from_mode(0o500)).unwrap(); + } + } + let r = resolve_data_dir(None, "tok", &env_of(&[]), &|| tmp.clone()); + let mut all: Vec = r.recovery_dirs.clone(); + all.extend(r.legacy_dir.clone()); + all.extend(r.dir.clone()); + let canon: Vec = all.iter().filter_map(|p| p.canonicalize().ok()).collect(); + let mut unique = canon.clone(); + unique.sort(); + unique.dedup(); + assert_eq!( + canon.len(), + unique.len(), + "one directory must not be in two roles: {all:?}" + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for tree in [ + tmp.join(private_temp_name()).join("tok"), + tmp.join("bugsee").join("tok"), + ] { + let _ = std::fs::set_permissions(&tree, std::fs::Permissions::from_mode(0o755)); + } + } + let _ = std::fs::remove_dir_all(&s); + } + #[test] fn an_explicit_directory_never_asks_for_the_temp_location() { let s = scratch(); From 3b90a28b699597f8636cad576f14b86f3678a9bf Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:56:17 +0500 Subject: [PATCH 31/52] fix(core): a lock file that vanishes under a scan is not evidence of a crash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit probe returned Free both for an unheld lock and for a file that disappeared between a caller's exists() and the open. A session without kernel locks unlinks its .lock before writing its heartbeat, so a peer that saw the file could classify that live session as crashed. probe now reports Probe::Vanished for a missing file (no pre-check), and session_is_crashed treats it like Unknown and falls through to the heartbeat/pid decision. Test injects the vanish between the existence check and the probe's open. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/claim.rs | 31 +++++++++++++++++++++++------- crates/bugsee-core/src/recovery.rs | 25 +++++++++++++++++++++++- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 2154dd9..6dbacd7 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -207,7 +207,7 @@ Crash recovery needs the data directory to **outlive the process**: on a contain - **Costs of per-process locks, handled in `claim.rs`.** They do not exclude threads, so a registry of held paths does (keyed by pid and spin-locked with the holder's pid stamped in, so a `fork` child that inherits it mid-update breaks the lock instead of deadlocking). POSIX drops a process's locks on a file when *any* descriptor for it closes, so a lock file is never opened twice by one process: the registry is checked before each open, and a claim closes its descriptor before leaving the registry. - **Ordering at launch.** `Session::begin` creates the marker (`O_EXCL`), takes `.lock`, then writes the pid. Until the pid is written the marker is empty and fresh, which peers skip even if the lock reads free. A lock that cannot be held with kernel enforcement (contended past a short retry, or the filesystem rejects locks) is *removed*: an existing but unheld lock file reads as "that session died", so peers fall back to the marker's pid instead. On unix a probe asks `F_GETLK` and never takes the lock, so it cannot steal one from a launching peer. - **Unclean shutdown.** If the worker or uploader thread is detached at shutdown it may still be writing the session, so the session's lock is kept until the process exits (the kernel then drops it) rather than released on drop. -- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown (as it does for a leftover lock file that cannot be opened, e.g. on a read-only volume, so a crash is never hidden by it). A session that could not take a lock keeps a `.beat` heartbeat file, touched every 2 s by the worker (and during launch-time recovery): a beat younger than 3 minutes, or dated in the future (generous: NFS caches attributes for up to a minute and its mtime is the server's clock) means alive whatever pid the marker names, which is the only signal that works across pid namespaces without locks; with no beat the marker's pid is probed as before. The price is that a crash on such a filesystem is recovered only after its heartbeat has been quiet for three minutes, so a faster restart defers it to a later launch. Under the recovery claim the liveness decision is re-run, so a session that finished launching since the scan is never torn down. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. +- **Filesystems without locks.** If the filesystem rejects byte-range locks (`ENOLCK` on NFS without lockd; `EOPNOTSUPP`/`ENOSYS`/`EINVAL` on FUSE, 9p) a claim degrades to thread-only exclusion with a one-time message on stderr, and a probe reports unknown, and a lock file that vanishes under a peer's scan is likewise not taken as evidence of death (as a probe does for a leftover lock file that cannot be opened, e.g. on a read-only volume, so a crash is never hidden by it). A session that could not take a lock keeps a `.beat` heartbeat file, touched every 2 s by the worker (and during launch-time recovery): a beat younger than 3 minutes, or dated in the future (generous: NFS caches attributes for up to a minute and its mtime is the server's clock) means alive whatever pid the marker names, which is the only signal that works across pid namespaces without locks; with no beat the marker's pid is probed as before. The price is that a crash on such a filesystem is recovered only after its heartbeat has been quiet for three minutes, so a faster restart defers it to a later launch. Under the recovery claim the liveness decision is re-run, so a session that finished launching since the scan is never torn down. The queue keeps draining and sessions keep recovering, but processes sharing that directory are no longer kept from delivering a report twice. Locks that *succeed but do not exclude* (Docker Desktop bind mounts, WSL2 drvfs/9p, NFS mounted `nolock`) cannot be detected and behave the same way: run one SDK host per directory there. - **Paths.** The in-process registry canonicalises the lock file's directory, so `/tmp` vs `/private/tmp` or relative vs absolute paths are one claim. A `fork` child clears the registry (`claim::reset_after_fork`, from the fork hook) before claiming anything. - **Stale scans.** After winning a session claim the marker is re-checked, since a peer may have recovered and discarded it since the scan (this caused phantom duplicate reports). `queue::gc_orphans` only reaps files untouched for 60 s: a peer's in-flight `enqueue` looks like an orphan, and reaping it lost the report. Lock files of finished items and dead sessions are swept when free. - **Compatibility.** The `.alive` marker is still a bare pid and the lock files are new names, so older builds sharing the directory keep working. They do not take part in the locking, so a mixed-version directory (a rolling upgrade) gives no exactly-once guarantee between old and new processes, and a new process treats an old build's session by its pid as before. Run one SDK version per directory for the guarantee. diff --git a/crates/bugsee-core/src/claim.rs b/crates/bugsee-core/src/claim.rs index 5c2c730..f3d0956 100644 --- a/crates/bugsee-core/src/claim.rs +++ b/crates/bugsee-core/src/claim.rs @@ -111,11 +111,15 @@ pub fn try_claim_lenient(path: &Path) -> Option> { pub(crate) enum Probe { Held, Free, + /// There is no lock file (any longer). Not the same as [`Probe::Free`]: a + /// caller that saw the file exist and finds it gone must not conclude that its + /// holder died, since a session without kernel locks unlinks its lock file. + Vanished, /// The filesystem cannot say (it rejects locks): use another liveness signal. Unknown, } -/// Is `path` currently claimed? A missing file is free. A file that cannot be +/// Is `path` currently claimed? A missing file is [`Probe::Vanished`]. A file that cannot be /// inspected is [`Probe::Unknown`], never assumed held: a crashed session's /// leftover lock file must not hide the crash on a volume that has become /// read-only. @@ -123,15 +127,18 @@ pub(crate) enum Probe { /// On unix this asks (`F_GETLK`) without taking the lock, so a probe can never /// steal the lock from a creator that has made the file but not yet locked it. pub(crate) fn probe(path: &Path) -> Probe { - if !path.exists() { - return Probe::Free; - } let key = key_of(path); if !registry().insert(&key) { return Probe::Held; // held by a thread of this process, or being probed } // Read-only: a querying lock call needs no write access, and a leftover lock // file on a read-only volume (or with 0444 mode) must still be probeable. + #[cfg(test)] + BEFORE_PROBE_OPEN.with(|h| { + if let Some(hook) = h.borrow().as_ref() { + hook(); + } + }); let result = match OpenOptions::new().read(true).open(path) { Ok(file) => { let r = match test_is_locked(&file) { @@ -145,8 +152,7 @@ pub(crate) fn probe(path: &Path) -> Probe { drop(file); // close before leaving the registry r } - // Vanished since the existence check: nothing holds it. - Err(_) if !path.exists() => Probe::Free, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Probe::Vanished, Err(_) => Probe::Unknown, }; registry().remove(&key); @@ -263,6 +269,17 @@ thread_local! { pub(crate) static FORCE_UNSUPPORTED: std::cell::Cell = const { std::cell::Cell::new(false) }; } +#[cfg(test)] +type Hook = Box; + +#[cfg(test)] +thread_local! { + /// Test hook: runs inside `probe` just before it opens the file, to simulate + /// the lock file vanishing between a caller's existence check and the probe. + pub(crate) static BEFORE_PROBE_OPEN: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + fn test_try_lock(file: &File) -> std::io::Result { #[cfg(test)] if FORCE_UNSUPPORTED.with(|f| f.get()) { @@ -569,7 +586,7 @@ mod tests { fn a_missing_lock_file_probes_free_and_is_not_created() { let dir = tmp(); let path = dir.join("nope.lock"); - assert!(matches!(probe(&path), Probe::Free)); + assert!(matches!(probe(&path), Probe::Vanished)); assert!(!path.exists()); } diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 526717b..fe50447 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -120,7 +120,10 @@ fn session_is_crashed(data_dir: &Path, generation: u64, marker: &Path, self_pid: crate::claim::Probe::Free => { return !(marker_is_empty(marker) && marker_is_fresh(marker)); } - crate::claim::Probe::Unknown => {} + // No lock file after all (it vanished since `exists`, e.g. a session + // without kernel locks removing it), or the filesystem cannot say: + // that is not evidence of death. Use the heartbeat / pid. + crate::claim::Probe::Vanished | crate::claim::Probe::Unknown => {} } } legacy_marker_is_dead(marker, self_pid) @@ -890,6 +893,26 @@ mod tests { assert_eq!(gens(&dir), vec![1], "quiet heartbeat: crashed"); } + #[test] + fn a_lock_file_that_vanishes_under_the_scan_is_not_evidence_of_a_crash() { + // A lock-less session unlinks its `.lock` before it writes its + // heartbeat; a peer that saw the file must fall back to the beat. + let dir = tmp(); + let m = marker(&dir, 1, "4294967290"); + let lock = Session::lock_path(&dir, 1); + std::fs::write(&lock, "").unwrap(); + std::fs::write(heartbeat_path(&m), "").unwrap(); + let lock2 = lock.clone(); + crate::claim::BEFORE_PROBE_OPEN.with(|h| { + *h.borrow_mut() = Some(Box::new(move || { + let _ = std::fs::remove_file(&lock2); + })); + }); + let found = gens(&dir); + crate::claim::BEFORE_PROBE_OPEN.with(|h| *h.borrow_mut() = None); + assert!(found.is_empty(), "fresh heartbeat: the session is alive"); + } + #[test] fn a_future_dated_heartbeat_counts_as_alive() { let dir = tmp(); From 546024b788a1c22a3701d698399b20889f4abd5d Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 16:57:42 +0500 Subject: [PATCH 32/52] fix: a stop during a flush's memory pass must not start a disk upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-check `stopping` after the memory drain and after each claim, so the uploader reaches its stop arm and the stop-time memory drain runs. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/runtime.rs | 9 ++ crates/bugsee-core/tests/storage_failures.rs | 141 +++++++++++++++++++ 2 files changed, 150 insertions(+) diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index e14587d..182f9a9 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1838,11 +1838,20 @@ fn drain_until_empty( Some(start + deadline), true, ); + // A stop that arrived during the memory pass must not be followed by a + // blocking disk upload: the uploader has to reach `UploadMsg::Stop` and + // run the stop-time memory drain, which a hung HTTP call would starve. + if shared.stopping.load(Ordering::SeqCst) { + return false; + } 'dirs: for &data_dir in dirs { for report in pending_reports(shared, data_dir) { let Some(_claim) = queue::try_claim_lenient(&report) else { continue; // a peer is delivering it; the loop below waits it out }; + if shared.stopping.load(Ordering::SeqCst) { + return false; // checked after the claim so no upload starts + } let Ok((zip, request_json, retry, _next)) = queue::load(&report) else { continue; }; diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index 616fb67..f1cced9 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -699,3 +699,144 @@ fn signatures_blacklisted_on_disk_are_honoured_in_memory_mode() { "only the report without the blacklisted signature is sent" ); } + +/// A stop that lands while a flush is in its memory pass must not be followed by +/// a (possibly hanging) disk upload: the uploader has to reach its stop arm and +/// give the in-memory backlog its last attempt. +#[test] +fn a_stop_during_a_flush_skips_disk_uploads_so_memory_gets_its_last_attempt() { + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{Condvar, Mutex}; + + struct T { + mem_calls: AtomicUsize, + mem_in_flight: AtomicBool, + disk_calls: AtomicUsize, + delivered: AtomicUsize, + gate: (Mutex, Condvar), + } + struct Handle(Arc); + impl bugsee_core::Transport for Handle { + fn register_session( + &self, + _: &str, + _: &[u8], + ) -> Result { + Ok("token".into()) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + request: &[u8], + ) -> Result { + let transient = || bugsee_core::TransportError::Transient("offline".into()); + if String::from_utf8_lossy(request).contains("disk-report") { + self.0.disk_calls.fetch_add(1, Ordering::SeqCst); + return Err(transient()); + } + match self.0.mem_calls.fetch_add(1, Ordering::SeqCst) { + 0 => Err(transient()), + 1 => { + // The flush's memory pass: held until the stop has begun. + self.0.mem_in_flight.store(true, Ordering::SeqCst); + let mut open = self.0.gate.0.lock().unwrap(); + while !*open { + open = self.0.gate.1.wait(open).unwrap(); + } + Err(transient()) + } + _ => Ok("https://uploads.example/x".into()), + } + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), bugsee_core::TransportError> { + self.0.delivered.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + fn wait_until(what: &str, f: impl Fn() -> bool) { + let start = std::time::Instant::now(); + while !f() { + assert!( + start.elapsed() < Duration::from_secs(10), + "timed out: {what}" + ); + std::thread::sleep(Duration::from_millis(5)); + } + } + + let dir = TempDir::new(); + let data = dir.path.join("data"); + std::fs::create_dir_all(&data).unwrap(); + // A queued disk report that is backing off, so only a forced flush tries it. + { + use bugsee_core::model::report::TimeWindow; + let env = bugsee_core::model::environment::Environment::detect_with( + "t", + &Default::default(), + &Default::default(), + ); + let mut meta = manual_upload_meta(); + meta.summary = Some("disk-report".into()); + let docs = bugsee_core::reporting::assemble_docs( + Vec::new(), + TimeWindow { start: 0, end: 1 }, + &meta, + &env, + None, + &[], + "T", + 1, + ) + .unwrap(); + bugsee_core::queue::enqueue(&data, &docs).unwrap(); + for r in bugsee_core::queue::list_pending(&data) { + bugsee_core::queue::set_meta(&r, 0, epoch_ms() + 3_600_000); + } + } + + let state = Arc::new(T { + mem_calls: AtomicUsize::new(0), + mem_in_flight: AtomicBool::new(false), + disk_calls: AtomicUsize::new(0), + delivered: AtomicUsize::new(0), + gate: (Mutex::new(false), Condvar::new()), + }); + let mut config = RecorderConfig::new(&data, "T"); + config.memory_only = Some("test".into()); + let recorder = Recorder::launch(config, Arc::new(Handle(state.clone()))).expect("launch"); + let handle = recorder.handle(); + + let mut meta = manual_upload_meta(); + meta.summary = Some("mem-report".into()); + recorder.report(meta, None); + wait_until("first memory attempt", || { + state.mem_calls.load(Ordering::SeqCst) >= 1 + }); + + // Let the startup/scheduled passes settle; only what follows is the flush's. + std::thread::sleep(Duration::from_millis(500)); + let disk_before = state.disk_calls.load(Ordering::SeqCst); + let flusher = std::thread::spawn(move || handle.flush(Duration::from_secs(10))); + wait_until("flush in its memory pass", || { + state.mem_in_flight.load(Ordering::SeqCst) + }); + let stopper = std::thread::spawn(move || drop(recorder)); + // `stopping` is set well within this. + std::thread::sleep(Duration::from_millis(300)); + *state.gate.0.lock().unwrap() = true; + state.gate.1.notify_all(); + stopper.join().unwrap(); + assert!(!flusher.join().unwrap()); + + assert_eq!( + state.disk_calls.load(Ordering::SeqCst), + disk_before, + "a disk upload was started after the stop" + ); + assert_eq!( + state.delivered.load(Ordering::SeqCst), + 1, + "the stop-time drain must give the memory backlog its last attempt" + ); +} From 69378f148dcccda463129eab513d127c93342be0 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 17:03:20 +0500 Subject: [PATCH 33/52] fix(health): keep panic.info for abort correlation, no eager registration in memory mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review on #35: the worker-thread snapshot is expired by a detached thread after the process survived 1 s instead of deleted in the hook, and the eager session registration requires a disk session. Refs #12 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 6 +-- crates/bugsee-core/src/runtime.rs | 11 ++++-- crates/bugsee-core/tests/release_health.rs | 14 +++++++ crates/bugsee-panic/src/lib.rs | 43 ++++++++++++++++++---- crates/bugsee/tests/release_health.rs | 2 +- 5 files changed, 61 insertions(+), 15 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 747b020..64e08da 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -294,7 +294,7 @@ That last case previously produced **no report whatsoever** β€” the SDK shut dow **What the backend does today** (`appserver` `components/shared/statistics/index.js`, `formatter.js`): a session is counted when it is created (`POST /v2/sessions`, `statsInc.sessions`), a crash when a crash **report** is received (`logIssue`, `issues.crashes`), and `crashes_free_sessions = (sessions - crashes) / sessions`, per day and per release (app `version`/`build`). There is **no endpoint** that accepts a session end, status, duration or error count, no `ok/errored/crashed/abnormal` model, and no spec (`specs/`, `report-bundle-structure/`) mentions one. -**The design gap that matters most.** The Rust transport registers its session lazily, inside `deliver`, so only a process that uploads something ever creates a backend session. A healthy run that reports nothing is never counted while its crash report is, and the rate collapses toward 0%. With `LaunchOptions::release_health(true)` (experimental, **off by default**) the uploader thread therefore registers the session **eagerly, once per launch**, right after launch: best effort, never on the caller's thread, never an error for launch, and offline-tolerant (a failure is dropped, retried at most every 30 s, and the first delivery registers lazily if it is still unregistered). The token is the one `deliver` uses, so there is exactly one `POST /v2/sessions` per online launch. The session of a run that died is registered by that run itself; its crash report arrives with the next process, which counts it as a crash on its own day and release (the backend has no link between the two), as for any recovered crash. A fork child inherits the parent's token and registers nothing of its own. +**The design gap that matters most.** The Rust transport registers its session lazily, inside `deliver`, so only a process that uploads something ever creates a backend session. A healthy run that reports nothing is never counted while its crash report is, and the rate collapses toward 0%. With `LaunchOptions::release_health(true)` (experimental, **off by default**) the uploader thread therefore registers the session **eagerly, once per launch**, right after launch (only when the launch has a disk session: a memory-mode run could never contribute its crash, so counting it would read healthy): best effort, never on the caller's thread, never an error for launch, and offline-tolerant (a failure is dropped, retried at most every 30 s, and the first delivery registers lazily if it is still unregistered). The token is the one `deliver` uses, so there is exactly one `POST /v2/sessions` per online launch. The session of a run that died is registered by that run itself; its crash report arrives with the next process, which counts it as a crash on its own day and release (the backend has no link between the two), as for any recovered crash. A fork child inherits the parent's token and registers nothing of its own. **Local record (no backend half).** The same option keeps a durable record per session in `health/`: `.live` while running and `.json` when finished (`ok`, `errored`, `crashed`, `abnormal`; duration, error count, release). Nothing is sent. It is unstable by design: the types live in a `doc(hidden)` module, are `#[non_exhaustive]`, and the facade offers only `doc(hidden)` accessors (`Bugsee::__finished_sessions` and friends); there is no stable API for a contract that does not exist. @@ -302,12 +302,12 @@ Rules. - A session is one SDK launch (desktop app, server and CLI alike; no idle timeout). - A handled error, including a panic on a non-main thread, makes it `errored`, never `crashed`. Counting happens with an atomic on the reporting path (no file I/O under the facade's lock); the worker's rotation tick persists it. - A stop that is not a death by panic finishes the record (`ok`/`errored`) whatever the worker and uploader acked. A detached thread leaves the liveness marker, so the next launch recovers the session too and finds the finished record (idempotent). -- A session that died is finished by the next launch's recovery as `crashed` (native crash info, a minidump, or a panic snapshot that is **fresh**: the session did not keep capturing more than 10 s after it) or `abnormal` (none). This happens in `recovery::discard`, under the recover claim and whatever became of the report (delivered, deduplicated, dropped by `before_send`). A worker-thread panic deletes its snapshot after the inline report, and `build_report` applies the same freshness rule, so a later SIGKILL is not misread as that panic's crash. +- A session that died is finished by the next launch's recovery as `crashed` (native crash info, a minidump, or a panic snapshot that is **fresh**: the session did not keep capturing more than 10 s after it) or `abnormal` (none). This happens in `recovery::discard`, under the recover claim and whatever became of the report (delivered, deduplicated, dropped by `before_send`). A worker-thread panic's snapshot is deleted by a detached thread once the process has survived it for 1 s (never from the hook itself: under `panic = "abort"` the snapshot is what correlates the SIGABRT with the panic), and `build_report` applies the same freshness rule, so a later SIGKILL is not misread as that panic's crash. - Exactly once: the finished record is written by rename before the marker goes and rewritten if unreadable; `acknowledge` leaves a `.acked` tombstone that a late finalizer honours; gc removes tombstones and orphan `.live` files once their marker is gone. Records raise the generation floor, so a number is not reused. - Memory mode (#9) tracks nothing and lists nothing (a crash there is invisible, so every session would read healthy); an empty data directory is refused. A fork child is a session of its own and never finishes the parent's. - Durations of clean sessions come from a monotonic clock; a dead session's end is the later of its newest capture-part mtime and its live record's last write. -Documented limits. Records of legacy and extra recovery directories are not surfaced; the newest 256 finished records are kept and older ones are dropped even if unacknowledged; right after launch a recovered earlier-run crash may not be listed yet (recovery past the launch-path budget is asynchronous); a server that is ended by SIGTERM, `exec` or `_exit` without dropping its guard is `abnormal`, which inflates the crashed rate, so end the run with a clean stop: call `Bugsee::stop` (or drop the guard) from the termination handler; once the shutdown contract of #8 (PR #31) lands, `Bugsee::shutdown` and the opt-in `Bugsee::install_termination_flush(timeout)` do this on SIGTERM/SIGINT (neither exists on this branch yet); error counts of a session that died are approximate (up to one tick behind). The crash handler and panic hook add no work of their own for this; the one new hook-side action is the worker-thread snapshot deletion above, and the inline worker-thread report (already there) now also bumps the error counter. +Documented limits. Records of legacy and extra recovery directories are not surfaced; the newest 256 finished records are kept and older ones are dropped even if unacknowledged; right after launch a recovered earlier-run crash may not be listed yet (recovery past the launch-path budget is asynchronous); a server that is ended by SIGTERM, `exec` or `_exit` without dropping its guard is `abnormal`, which inflates the crashed rate, so end the run with a clean stop: call `Bugsee::stop` (or drop the guard) from the termination handler; once the shutdown contract of #8 (PR #31) lands, `Bugsee::shutdown` and the opt-in `Bugsee::install_termination_flush(timeout)` do this on SIGTERM/SIGINT (neither exists on this branch yet); error counts of a session that died are approximate (up to one tick behind). The crash handler and panic hook add no work of their own for this; the one new hook-side action is scheduling the worker-thread snapshot expiry above, and the inline worker-thread report (already there) now also bumps the error counter. **Blocked on the backend.** (1) An ingest endpoint, or fields on session-create/close, for `{session_id, started_at, ended_at, duration, status, errors, release, environment}`, authenticated like the other calls and idempotent on `session_id`; until then crashed and abnormal sessions can only be counted through crash reports. (2) How `crashed`/`abnormal` should feed `crashes_free_sessions`: today a session whose crash report was deduplicated (`12003`) or blacklisted (`12004`) counts as crash-free server-side while the SDK records it `crashed`. (3) Crash-free **users**: the backend's user counts key on `environment.hardware.device_id` (`session.service.js`), which the Rust environment does not send; a stable anonymous id is needed first. (4) Sampling for servers with many short sessions (each launch is one `POST /v2/sessions`), to be decided with the endpoint. (5) A `Transport::report_session` method and offline queueing of finished records, which the durable queue and recover-lock claims can carry once the shape exists. diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 9e82c0b..bcbd3da 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -304,7 +304,7 @@ struct Shared { health_errors: AtomicU64, /// Register the backend session at launch instead of with the first report /// (release health on): see [`register_eagerly`]. - eager_session: bool, + eager_session: AtomicBool, } /// What the launch-path recovery pass needs, fixed at launch. @@ -442,7 +442,7 @@ impl Recorder { launch_wait_ms: AtomicU64::new(0), health: ForkLock::new(None), health_errors: AtomicU64::new(0), - eager_session: config.release_health, + eager_session: AtomicBool::new(false), }); crate::fork::install_fork_hook(); @@ -475,6 +475,11 @@ impl Recorder { Some((session, store)) => (Some(session), Some(store)), None => (None, None), }; + // Only a session that can be recovered is registered eagerly: a memory-mode + // run could never contribute its crash, so counting it would read healthy. + shared + .eager_session + .store(config.release_health && session.is_some(), Ordering::SeqCst); if config.release_health { if let Some(session) = &session { *shared.health.lock() = crate::health::HealthTracker::start( @@ -1864,7 +1869,7 @@ fn uploader_loop( // never uploads would never be counted. Best effort, on this thread, before // anything else; failure (offline) is silent and retried at most every // `EAGER_RETRY` from the poll below, falling back to the lazy path. - let eager = fresh_launch && shared.eager_session; + let eager = fresh_launch && shared.eager_session.load(Ordering::SeqCst); let mut last_eager = Instant::now(); if eager { register_eagerly(&shared); diff --git a/crates/bugsee-core/tests/release_health.rs b/crates/bugsee-core/tests/release_health.rs index 2ef331a..3ea954d 100644 --- a/crates/bugsee-core/tests/release_health.rs +++ b/crates/bugsee-core/tests/release_health.rs @@ -185,3 +185,17 @@ fn an_empty_data_dir_is_rejected() { assert!(health::HealthTracker::start(std::path::Path::new(""), 1, &env).is_none()); assert!(health::finished(std::path::Path::new("")).is_empty()); } + +#[test] +fn a_memory_mode_launch_is_not_registered_eagerly() { + // It could never contribute a recovered crash, so counting it as a session + // would read healthy. + let d = dir("memory-eager"); + let mock = Arc::new(MockTransport::default()); + let mut cfg = config(&d, true); + cfg.memory_only = Some("test".into()); + let rec = Recorder::launch(cfg, mock.clone()).unwrap(); + std::thread::sleep(Duration::from_millis(400)); + assert_eq!(*mock.sessions.lock().unwrap(), 0); + drop(rec); +} diff --git a/crates/bugsee-panic/src/lib.rs b/crates/bugsee-panic/src/lib.rs index 6a89204..6c1ac40 100644 --- a/crates/bugsee-panic/src/lib.rs +++ b/crates/bugsee-panic/src/lib.rs @@ -166,6 +166,34 @@ fn worker_thread_name() -> Option { } } +/// How long a process must outlive a worker-thread panic for the persisted +/// snapshot to be dropped as stale. An abort follows the hook within +/// milliseconds, far inside this. +const SNAPSHOT_SURVIVAL: std::time::Duration = std::time::Duration::from_millis(1000); + +/// After [`SNAPSHOT_SURVIVAL`], delete the snapshot just written, unless a later +/// panic has replaced it. Best effort on a detached thread: if the process dies +/// first (an abort), the snapshot is there for next-launch correlation. +fn expire_snapshot_if_survived() { + let Some(path) = SNAPSHOT_PATH + .try_lock() + .and_then(|g| current_snapshot_path(&g).cloned()) + else { + return; + }; + let Some(written) = PanicInfo::read_from(&path).map(|i| i.timestamp) else { + return; + }; + let _ = std::thread::Builder::new() + .name("bugsee-panic-expire".into()) + .spawn(move || { + std::thread::sleep(SNAPSHOT_SURVIVAL); + if PanicInfo::read_from(&path).is_some_and(|i| i.timestamp == written) { + let _ = std::fs::remove_file(&path); + } + }); +} + /// Install the chained global panic observer and register `reporter`. /// Idempotent β€” the hook is installed at most once; the first reporter wins. pub fn install(reporter: Arc) { @@ -235,15 +263,14 @@ pub fn install(reporter: Arc) { handled: is_worker, thread: worker_thread, }); - // A worker-thread panic is reported and the process lives - // on: its snapshot must not outlast it, or a later SIGKILL - // would be read as this panic having killed the process. + // A worker-thread panic is reported and the process may live + // on: if it is still running a moment later its snapshot is + // stale, and a later SIGKILL must not be read as this panic + // having killed it. Not deleted here: under `panic = "abort"` + // the process dies right after the hook, and the snapshot is + // what correlates the SIGABRT with the panic. if is_worker { - if let Some(guard) = SNAPSHOT_PATH.try_lock() { - if let Some(path) = current_snapshot_path(&guard) { - let _ = std::fs::remove_file(path); - } - } + expire_snapshot_if_survived(); } } } diff --git a/crates/bugsee/tests/release_health.rs b/crates/bugsee/tests/release_health.rs index 4bd9a90..88f2b2f 100644 --- a/crates/bugsee/tests/release_health.rs +++ b/crates/bugsee/tests/release_health.rs @@ -87,7 +87,7 @@ fn run_child(kind: &str, dir: &str) -> ! { .spawn(|| panic!("worker-thread-panic")) .unwrap() .join(); - std::thread::sleep(Duration::from_millis(1500)); + std::thread::sleep(Duration::from_millis(2500)); std::process::exit(3); } "segv" => fault_null(), From 4002558198c153bd6b0fd6e0bb8f8d561b3a5e97 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 17:08:31 +0500 Subject: [PATCH 34/52] fix(core): a failed enqueue write is not a recovery attempt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/runtime.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 5439354..6bc3314 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1280,6 +1280,13 @@ fn recover_one( true } else { hold_in_memory(shared, assembled); + // A failed WRITE says nothing about the evidence (the volume is + // full or read-only), unlike an assemble failure, so it must not + // use up the session's attempts: a supervisor crash loop would + // kill the process before the memory copy is sent, and three such + // launches would discard a crash that was never delivered. The + // launch pass keeps its claim so the worker does not hold it twice. + recovery::undo_attempt(data_dir, pending); false } } From 46872ff497a93a1f95c8cf140d37731c02b1e6e0 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 17:20:12 +0500 Subject: [PATCH 35/52] fix(health): do not expire panic.info from a hook timer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address re-review on #35: leave the worker-thread snapshot alone (abort correlation) and rule out a surviving process's leftover at recovery with a 5 s tail. Adds a worker-thread aborting-panic case to the crash matrix. Refs #12 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 4 +-- crates/bugsee-core/src/recovery.rs | 2 +- crates/bugsee-panic/src/lib.rs | 42 +++---------------------- crates/bugsee/tests/crash_e2e_matrix.rs | 15 +++++++++ crates/bugsee/tests/release_health.rs | 2 +- 5 files changed, 24 insertions(+), 41 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 64e08da..93689c6 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -302,12 +302,12 @@ Rules. - A session is one SDK launch (desktop app, server and CLI alike; no idle timeout). - A handled error, including a panic on a non-main thread, makes it `errored`, never `crashed`. Counting happens with an atomic on the reporting path (no file I/O under the facade's lock); the worker's rotation tick persists it. - A stop that is not a death by panic finishes the record (`ok`/`errored`) whatever the worker and uploader acked. A detached thread leaves the liveness marker, so the next launch recovers the session too and finds the finished record (idempotent). -- A session that died is finished by the next launch's recovery as `crashed` (native crash info, a minidump, or a panic snapshot that is **fresh**: the session did not keep capturing more than 10 s after it) or `abnormal` (none). This happens in `recovery::discard`, under the recover claim and whatever became of the report (delivered, deduplicated, dropped by `before_send`). A worker-thread panic's snapshot is deleted by a detached thread once the process has survived it for 1 s (never from the hook itself: under `panic = "abort"` the snapshot is what correlates the SIGABRT with the panic), and `build_report` applies the same freshness rule, so a later SIGKILL is not misread as that panic's crash. +- A session that died is finished by the next launch's recovery as `crashed` (native crash info, a minidump, or a panic snapshot that is **fresh**: the session did not keep capturing more than 5 s after it) or `abnormal` (none). This happens in `recovery::discard`, under the recover claim and whatever became of the report (delivered, deduplicated, dropped by `before_send`). A worker-thread panic's snapshot is **kept** (under `panic = "abort"` it is what correlates the SIGABRT with the panic, and a chained hook can delay the abort arbitrarily); a leftover one is ruled out at recovery instead, and `build_report` applies the same freshness rule, so a later SIGKILL is not misread as that panic's crash. - Exactly once: the finished record is written by rename before the marker goes and rewritten if unreadable; `acknowledge` leaves a `.acked` tombstone that a late finalizer honours; gc removes tombstones and orphan `.live` files once their marker is gone. Records raise the generation floor, so a number is not reused. - Memory mode (#9) tracks nothing and lists nothing (a crash there is invisible, so every session would read healthy); an empty data directory is refused. A fork child is a session of its own and never finishes the parent's. - Durations of clean sessions come from a monotonic clock; a dead session's end is the later of its newest capture-part mtime and its live record's last write. -Documented limits. Records of legacy and extra recovery directories are not surfaced; the newest 256 finished records are kept and older ones are dropped even if unacknowledged; right after launch a recovered earlier-run crash may not be listed yet (recovery past the launch-path budget is asynchronous); a server that is ended by SIGTERM, `exec` or `_exit` without dropping its guard is `abnormal`, which inflates the crashed rate, so end the run with a clean stop: call `Bugsee::stop` (or drop the guard) from the termination handler; once the shutdown contract of #8 (PR #31) lands, `Bugsee::shutdown` and the opt-in `Bugsee::install_termination_flush(timeout)` do this on SIGTERM/SIGINT (neither exists on this branch yet); error counts of a session that died are approximate (up to one tick behind). The crash handler and panic hook add no work of their own for this; the one new hook-side action is scheduling the worker-thread snapshot expiry above, and the inline worker-thread report (already there) now also bumps the error counter. +Documented limits. Records of legacy and extra recovery directories are not surfaced; the newest 256 finished records are kept and older ones are dropped even if unacknowledged; right after launch a recovered earlier-run crash may not be listed yet (recovery past the launch-path budget is asynchronous); a server that is ended by SIGTERM, `exec` or `_exit` without dropping its guard is `abnormal`, which inflates the crashed rate, so end the run with a clean stop: call `Bugsee::stop` (or drop the guard) from the termination handler; once the shutdown contract of #8 (PR #31) lands, `Bugsee::shutdown` and the opt-in `Bugsee::install_termination_flush(timeout)` do this on SIGTERM/SIGINT (neither exists on this branch yet); error counts of a session that died are approximate (up to one tick behind). The crash handler and panic hook add no work of their own for this; the hook gains no new action, and the inline worker-thread report (already there) now also bumps the error counter. **Blocked on the backend.** (1) An ingest endpoint, or fields on session-create/close, for `{session_id, started_at, ended_at, duration, status, errors, release, environment}`, authenticated like the other calls and idempotent on `session_id`; until then crashed and abnormal sessions can only be counted through crash reports. (2) How `crashed`/`abnormal` should feed `crashes_free_sessions`: today a session whose crash report was deduplicated (`12003`) or blacklisted (`12004`) counts as crash-free server-side while the SDK records it `crashed`. (3) Crash-free **users**: the backend's user counts key on `environment.hardware.device_id` (`session.service.js`), which the Rust environment does not send; a stable anonymous id is needed first. (4) Sampling for servers with many short sessions (each launch is one `POST /v2/sessions`), to be decided with the endpoint. (5) A `Transport::report_session` method and offline queueing of finished records, which the durable queue and recover-lock claims can carry once the shape exists. diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 90b4c76..2e51829 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -430,7 +430,7 @@ fn dead_status(pending: &PendingSession) -> crate::health::SessionStatus { /// How long after a panic snapshot the session may keep writing and still be /// the one the panic killed: unwinding out of `main` and the recorder's final /// flush take well under this. -const PANIC_TAIL_MS: i64 = 10_000; +const PANIC_TAIL_MS: i64 = 5_000; /// Whether a panic snapshot with no native crash beside it can be what ended /// this session. A snapshot is written for every panic and deleted only when a diff --git a/crates/bugsee-panic/src/lib.rs b/crates/bugsee-panic/src/lib.rs index 6c1ac40..6c6dc38 100644 --- a/crates/bugsee-panic/src/lib.rs +++ b/crates/bugsee-panic/src/lib.rs @@ -166,34 +166,6 @@ fn worker_thread_name() -> Option { } } -/// How long a process must outlive a worker-thread panic for the persisted -/// snapshot to be dropped as stale. An abort follows the hook within -/// milliseconds, far inside this. -const SNAPSHOT_SURVIVAL: std::time::Duration = std::time::Duration::from_millis(1000); - -/// After [`SNAPSHOT_SURVIVAL`], delete the snapshot just written, unless a later -/// panic has replaced it. Best effort on a detached thread: if the process dies -/// first (an abort), the snapshot is there for next-launch correlation. -fn expire_snapshot_if_survived() { - let Some(path) = SNAPSHOT_PATH - .try_lock() - .and_then(|g| current_snapshot_path(&g).cloned()) - else { - return; - }; - let Some(written) = PanicInfo::read_from(&path).map(|i| i.timestamp) else { - return; - }; - let _ = std::thread::Builder::new() - .name("bugsee-panic-expire".into()) - .spawn(move || { - std::thread::sleep(SNAPSHOT_SURVIVAL); - if PanicInfo::read_from(&path).is_some_and(|i| i.timestamp == written) { - let _ = std::fs::remove_file(&path); - } - }); -} - /// Install the chained global panic observer and register `reporter`. /// Idempotent β€” the hook is installed at most once; the first reporter wins. pub fn install(reporter: Arc) { @@ -263,15 +235,11 @@ pub fn install(reporter: Arc) { handled: is_worker, thread: worker_thread, }); - // A worker-thread panic is reported and the process may live - // on: if it is still running a moment later its snapshot is - // stale, and a later SIGKILL must not be read as this panic - // having killed it. Not deleted here: under `panic = "abort"` - // the process dies right after the hook, and the snapshot is - // what correlates the SIGABRT with the panic. - if is_worker { - expire_snapshot_if_survived(); - } + // The snapshot stays even for a worker-thread panic: under + // `panic = "abort"` the process dies after this hook chain + // and the snapshot is what correlates the SIGABRT with the + // panic. One a surviving process leaves behind is ruled out + // at recovery (`panic_is_recent`), not here. } } diff --git a/crates/bugsee/tests/crash_e2e_matrix.rs b/crates/bugsee/tests/crash_e2e_matrix.rs index 8949236..f28d167 100644 --- a/crates/bugsee/tests/crash_e2e_matrix.rs +++ b/crates/bugsee/tests/crash_e2e_matrix.rs @@ -127,6 +127,17 @@ fn run_child(kind: &str, dir: &str) -> ! { })); panic!("{PANIC_TEXT}"); } + // The same on a worker thread: the SDK reports it inline (the process + // "survives" as far as the hook knows) and must still keep the snapshot, + // or the abort is recovered as a thin SIGABRT without the panic. + "aborting_panic_thread" => { + let previous = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + previous(info); + std::process::abort(); + })); + let _ = std::thread::spawn(|| panic!("{PANIC_TEXT}")).join(); + } // Dies without running any destructor: no clean shutdown, no signal. "exit_without_shutdown" => std::process::exit(3), other => panic!("unknown crash kind {other}"), @@ -220,6 +231,10 @@ fn cases() -> Vec { kind: "aborting_panic", expect: Expect::Panic, }); + v.push(Case { + kind: "aborting_panic_thread", + expect: Expect::Panic, + }); v } diff --git a/crates/bugsee/tests/release_health.rs b/crates/bugsee/tests/release_health.rs index 88f2b2f..fda2f9e 100644 --- a/crates/bugsee/tests/release_health.rs +++ b/crates/bugsee/tests/release_health.rs @@ -87,7 +87,7 @@ fn run_child(kind: &str, dir: &str) -> ! { .spawn(|| panic!("worker-thread-panic")) .unwrap() .join(); - std::thread::sleep(Duration::from_millis(2500)); + std::thread::sleep(Duration::from_millis(6500)); std::process::exit(3); } "segv" => fault_null(), From b6069384c7eb9cd3e04ac303b49a7e549fff5aa6 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 17:25:21 +0500 Subject: [PATCH 36/52] fix(core): a session the launch pass defers is picked up even if the pass outlived the worker's wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/runtime.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 6bc3314..bd6747b 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -289,6 +289,9 @@ struct Shared { /// over (it is armed and the pass has not finished). launch_armed: AtomicBool, launch_done: AtomicBool, + /// Set when the launch pass left a session for the worker (queue cap), so a + /// worker that already did its one pass runs another. + launch_deferred: AtomicBool, launch_wait_ms: AtomicU64, } @@ -422,6 +425,7 @@ impl Recorder { launch_attempted: ForkLock::new(BTreeSet::new()), launch_armed: AtomicBool::new(false), launch_done: AtomicBool::new(false), + launch_deferred: AtomicBool::new(false), launch_wait_ms: AtomicU64::new(0), }); @@ -1084,6 +1088,7 @@ fn spawn_pipeline( caps, sampler, worker_upload_tx, + generation, ); })?; @@ -1254,6 +1259,7 @@ fn recover_one( // Not a failed attempt: a supervisor crash loop would otherwise // burn the session's attempts on queue refusals and abandon it. recovery::undo_attempt(data_dir, pending); + shared.launch_deferred.store(true, Ordering::SeqCst); return; } } @@ -1326,6 +1332,7 @@ fn worker_loop( caps: WindowCaps, mut sampler: Option>, upload_tx: Sender, + generation: u64, ) { // Rotation/eviction/telemetry must run on a wall-clock cadence, NOT only when // `recv_timeout` times out. Under sustained capture traffic every `recv` @@ -1421,6 +1428,18 @@ fn worker_loop( rotate_tick(store.as_mut(), &shared, &mut sampler); next_rotate = Instant::now() + rotate_interval; } + // The launch-path pass left a session for us (queue cap) after our one + // pass had already run, because it outlived the wait: take it now. + if shared.launch_done.load(Ordering::SeqCst) + && shared.launch_deferred.swap(false, Ordering::SeqCst) + { + if let Some(dir) = &data_dir { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run_recovery(&shared, dir, generation, marker.as_deref(), true); + })); + let _ = upload_tx.send(UploadMsg::Wake); + } + } // Independent of the rotation interval (which a host may set large). if Instant::now() >= next_heartbeat { if let Some(marker) = &marker { From 168b672450a366b30293b674d363e7a25a2d1478 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 17:31:12 +0500 Subject: [PATCH 37/52] fix(health): mark a panicking drop so a slow unwind is still a crash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address re-review on #35: panic_is_recent no longer guesses from the clock when the recorder was dropped while unwinding; empty rotation directories no longer count as capture activity. Refs #12 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/health.rs | 7 ++++++- crates/bugsee-core/src/panic_info.rs | 4 ++++ crates/bugsee-core/src/recovery.rs | 20 +++++++++++++++++++- crates/bugsee-core/src/runtime.rs | 9 +++++++++ crates/bugsee/tests/release_health.rs | 19 +++++++++++++++++++ 6 files changed, 58 insertions(+), 3 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 93689c6..ccc3789 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -302,7 +302,7 @@ Rules. - A session is one SDK launch (desktop app, server and CLI alike; no idle timeout). - A handled error, including a panic on a non-main thread, makes it `errored`, never `crashed`. Counting happens with an atomic on the reporting path (no file I/O under the facade's lock); the worker's rotation tick persists it. - A stop that is not a death by panic finishes the record (`ok`/`errored`) whatever the worker and uploader acked. A detached thread leaves the liveness marker, so the next launch recovers the session too and finds the finished record (idempotent). -- A session that died is finished by the next launch's recovery as `crashed` (native crash info, a minidump, or a panic snapshot that is **fresh**: the session did not keep capturing more than 5 s after it) or `abnormal` (none). This happens in `recovery::discard`, under the recover claim and whatever became of the report (delivered, deduplicated, dropped by `before_send`). A worker-thread panic's snapshot is **kept** (under `panic = "abort"` it is what correlates the SIGABRT with the panic, and a chained hook can delay the abort arbitrarily); a leftover one is ruled out at recovery instead, and `build_report` applies the same freshness rule, so a later SIGKILL is not misread as that panic's crash. +- A session that died is finished by the next launch's recovery as `crashed` (native crash info, a minidump, or a panic snapshot that is **fresh**: the session did not keep capturing more than 5 s after it, unless the recorder was dropped while unwinding from a panic, which leaves a `panic.unwinding` mark beside the parts and settles it regardless of how slow the unwind was) or `abnormal` (none). This happens in `recovery::discard`, under the recover claim and whatever became of the report (delivered, deduplicated, dropped by `before_send`). A worker-thread panic's snapshot is **kept** (under `panic = "abort"` it is what correlates the SIGABRT with the panic, and a chained hook can delay the abort arbitrarily); a leftover one is ruled out at recovery instead, and `build_report` applies the same freshness rule, so a later SIGKILL is not misread as that panic's crash. - Exactly once: the finished record is written by rename before the marker goes and rewritten if unreadable; `acknowledge` leaves a `.acked` tombstone that a late finalizer honours; gc removes tombstones and orphan `.live` files once their marker is gone. Records raise the generation floor, so a number is not reused. - Memory mode (#9) tracks nothing and lists nothing (a crash there is invisible, so every session would read healthy); an empty data directory is refused. A fork child is a session of its own and never finishes the parent's. - Durations of clean sessions come from a monotonic clock; a dead session's end is the later of its newest capture-part mtime and its live record's last write. diff --git a/crates/bugsee-core/src/health.rs b/crates/bugsee-core/src/health.rs index 09d8a8c..ec0348e 100644 --- a/crates/bugsee-core/src/health.rs +++ b/crates/bugsee-core/src/health.rs @@ -395,10 +395,15 @@ pub(crate) fn newest_mtime(dir: &Path, depth: u32) -> Option { let name = entry.file_name(); if matches!( name.to_str(), - Some("panic.info" | "crash.info" | "crash.modules") + Some("panic.info" | "panic.unwinding" | "crash.info" | "crash.modules") ) { continue; } + // Rotation creates empty part directories whether or not anything was + // captured; only files are capture activity. + if meta.is_dir() && depth == 0 { + continue; + } let mut t = meta .modified() .ok() diff --git a/crates/bugsee-core/src/panic_info.rs b/crates/bugsee-core/src/panic_info.rs index 9834087..8c4efb8 100644 --- a/crates/bugsee-core/src/panic_info.rs +++ b/crates/bugsee-core/src/panic_info.rs @@ -17,6 +17,10 @@ use crate::model::crash::Frame; /// File name of the persisted panic snapshot under a session's part dir. pub const PANIC_INFO_NAME: &str = "panic.info"; +/// Written beside the capture parts when the recorder is dropped while its +/// thread is unwinding from a panic: the process is dying of that panic, which +/// settles a panic snapshot's staleness without any clock. +pub const PANIC_UNWINDING_NAME: &str = "panic.unwinding"; /// A captured panic, persisted for next-launch correlation. #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/bugsee-core/src/recovery.rs b/crates/bugsee-core/src/recovery.rs index 2e51829..be0d446 100644 --- a/crates/bugsee-core/src/recovery.rs +++ b/crates/bugsee-core/src/recovery.rs @@ -437,8 +437,20 @@ const PANIC_TAIL_MS: i64 = 5_000; /// boundary of ours catches it (or the hook reports a worker-thread panic), so /// one left by a panic the app survived, caught by a foreign `catch_unwind`, is /// still there when the process is later SIGKILLed. A session that kept -/// capturing long after the snapshot was not killed by it. +/// capturing long after the snapshot was not killed by it. A slow unwind keeps +/// the worker capturing too, so the recorder marks that case explicitly +/// ([`crate::panic_info::PANIC_UNWINDING_NAME`]) and the clock only decides when +/// there is no such mark. fn panic_is_recent(pending: &PendingSession, info: &PanicInfo) -> bool { + // The recorder was dropped while unwinding from a panic: the process died of + // it, however long the unwind took. + if pending + .parts_dir + .join(crate::panic_info::PANIC_UNWINDING_NAME) + .exists() + { + return true; + } match crate::health::newest_mtime(&pending.parts_dir, 3) { Some(last) => info.timestamp == 0 || last - info.timestamp <= PANIC_TAIL_MS, None => true, @@ -980,6 +992,12 @@ mod tests { let report = build_report(&pending, 5_000); assert!(String::from_utf8_lossy(&report.crash_json).contains("AppExit")); + // A slow unwind (the recorder dropped while panicking, long after the + // panic) is marked, and the mark settles it without the clock. + std::fs::write(parts.join(crate::panic_info::PANIC_UNWINDING_NAME), b"").unwrap(); + assert_eq!(dead_status(&pending), SessionStatus::Crashed); + std::fs::remove_file(parts.join(crate::panic_info::PANIC_UNWINDING_NAME)).unwrap(); + // A fresh snapshot (the panic that killed it) is still a crash. PanicInfo { reason: "fatal".into(), diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index bcbd3da..5429b1a 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -853,6 +853,15 @@ impl Drop for Recorder { // thread leaves the marker, so the next launch recovers the session // too, and finds this record already there (`finalize_dead` is // idempotent), so the clean end is counted once and as what it was. + if std::thread::panicking() && self.session.is_some() { + // Dying of a panic: say so beside the parts, so recovery does not + // have to guess from the clock whether the panic snapshot is stale + // (a slow unwind keeps the worker capturing for a while). + let _ = std::fs::write( + self.gen_dir().join(crate::panic_info::PANIC_UNWINDING_NAME), + b"", + ); + } if !std::thread::panicking() { if let Some(health) = self.shared.health.lock().take() { health.finish_clean(self.shared.health_errors.load(Ordering::SeqCst)); diff --git a/crates/bugsee/tests/release_health.rs b/crates/bugsee/tests/release_health.rs index fda2f9e..be185a5 100644 --- a/crates/bugsee/tests/release_health.rs +++ b/crates/bugsee/tests/release_health.rs @@ -95,6 +95,19 @@ fn run_child(kind: &str, dir: &str) -> ! { let _keep = &guard; panic!("health-e2e-deliberate-panic"); } + // A destructor that takes longer than the panic-snapshot freshness tail + // runs before the guard drops, with the recorder still capturing. + "slow_unwinding_panic" => { + struct SlowDrop; + impl Drop for SlowDrop { + fn drop(&mut self) { + std::thread::sleep(Duration::from_millis(6500)); + } + } + let _keep = &guard; + let _slow = SlowDrop; + panic!("health-e2e-slow-panic"); + } "exit_without_shutdown" => std::process::exit(3), other => panic!("unknown kind {other}"), } @@ -152,6 +165,12 @@ fn cases() -> Vec { status: SessionStatus::Crashed, min_errors: 0, }, + Case { + kind: "slow_unwinding_panic", + dies: true, + status: SessionStatus::Crashed, + min_errors: 0, + }, Case { kind: "exit_without_shutdown", dies: true, From 12d4672be587123e7a0e17255c6420c5648100bf Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 17:36:12 +0500 Subject: [PATCH 38/52] fix(core): deferred launch-pass recovery uses the recovery directory, not only the capture tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/runtime.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index bd6747b..9bf705b 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1024,6 +1024,7 @@ fn spawn_pipeline( let worker_shared = Arc::clone(shared); let worker_data_dir = data_dir.map(Path::to_path_buf); let worker_drain_dir = drain_dir.map(Path::to_path_buf); + let worker_recover_dir = worker_drain_dir.clone(); let marker = data_dir.map(|d| Session::marker_path(d, generation)); // Recovery can run long (a host `before_send`, many sessions); it keeps this // session's heartbeat going meanwhile, as the worker loop is not yet running. @@ -1089,6 +1090,7 @@ fn spawn_pipeline( sampler, worker_upload_tx, generation, + worker_recover_dir, ); })?; @@ -1333,6 +1335,7 @@ fn worker_loop( mut sampler: Option>, upload_tx: Sender, generation: u64, + recover_dir: Option, ) { // Rotation/eviction/telemetry must run on a wall-clock cadence, NOT only when // `recv_timeout` times out. Under sustained capture traffic every `recv` @@ -1430,10 +1433,12 @@ fn worker_loop( } // The launch-path pass left a session for us (queue cap) after our one // pass had already run, because it outlived the wait: take it now. - if shared.launch_done.load(Ordering::SeqCst) - && shared.launch_deferred.swap(false, Ordering::SeqCst) - { - if let Some(dir) = &data_dir { + // `recover_dir` is the tree startup recovery used: it exists in memory + // mode too, where `data_dir` (the capture tree) does not. + if let Some(dir) = &recover_dir { + if shared.launch_done.load(Ordering::SeqCst) + && shared.launch_deferred.swap(false, Ordering::SeqCst) + { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { run_recovery(&shared, dir, generation, marker.as_deref(), true); })); From 1748ada9238752a8847732984e43b24cbc679bde Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 17:42:45 +0500 Subject: [PATCH 39/52] fix(scrub): do not redact public OIDC/OAuth metadata keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit authorization_endpoint/url/uri, oauth* names (oauth_token and oauth_client_secret still hit the token/secret stems), auth_method and token_endpoint_auth_* are not secrets. Tests in bugsee-core and bugsee-reqwest (query params and headers). Refs #11 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/scrub.rs | 30 ++++++++++++++++++++++++++++++ crates/bugsee-reqwest/src/lib.rs | 20 ++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/crates/bugsee-core/src/scrub.rs b/crates/bugsee-core/src/scrub.rs index a48fd7f..f64429d 100644 --- a/crates/bugsee-core/src/scrub.rs +++ b/crates/bugsee-core/src/scrub.rs @@ -317,9 +317,21 @@ fn stem_hit(n: &[u8], stem: &[u8]) -> bool { let excluded = match stem { // `author`/`authority` (but not `authorization`), and // `www-authenticate`/`authentication`. + // Public OIDC/OAuth metadata is excluded too: `authorization_endpoint` + // /`_url`/`_uri`, `oauth*` names (`oauth_client_id`; token- and + // secret-bearing ones such as `oauth_token`/`oauth_client_secret` + // still hit the `token`/`secret` stems), and `auth_method`/ + // `token_endpoint_auth_signing_alg`. b"auth" => { (rest.starts_with(b"or") && !rest.starts_with(b"oriz")) || rest.starts_with(b"enticat") + || (rest.starts_with(b"orization") + && [&b"endpoint"[..], b"url", b"uri"] + .iter() + .any(|x| rest[9..].starts_with(x))) + || before.ends_with(b"o") + || rest.starts_with(b"method") + || rest.starts_with(b"signingalg") } // `tokenizer`, `token_type`/`_count`/`_ttl`/`_endpoint`/`_url`, and // `cancellation_token` (a cooperative-cancel handle, not a secret). @@ -2043,6 +2055,20 @@ mod tests { "session_timeout", "keyboard", "monkey", + "authorization_endpoint", + "authorization_url", + "authorization_uri", + "oauth", + "oauth_client_id", + "oauth2_redirect_uri", + "token_endpoint_auth_method", + "token_endpoint_auth_signing_alg_values_supported", + "auth_method", + "client_id", + "redirect_uri", + "issuer", + "jwks_uri", + "scope", "cancellation_token", "CancellationToken", "www-authenticate", @@ -2061,6 +2087,10 @@ mod tests { } for k in [ "authorization", + "oauth_token", + "oauth_client_secret", + "oauth_access_token", + "id_token", "auth", "oauth_token", "access_token", diff --git a/crates/bugsee-reqwest/src/lib.rs b/crates/bugsee-reqwest/src/lib.rs index a353b11..809fc32 100644 --- a/crates/bugsee-reqwest/src/lib.rs +++ b/crates/bugsee-reqwest/src/lib.rs @@ -421,6 +421,26 @@ mod tests { assert!(s.contains("token_ttl=9") && !s.contains("zzz"), "{s}"); assert!(!is_sensitive_header("WWW-Authenticate")); assert!(!is_sensitive_header("token_url")); + // Public OIDC/OAuth metadata stays readable; credentials do not. + let url = reqwest::Url::parse( + "https://idp.example.com/cb?authorization_endpoint=a&oauth_client_id=b\ + &token_endpoint_auth_method=c&client_id=d&redirect_uri=e&oauth_token=zzz\ + &oauth_client_secret=yyy", + ) + .unwrap(); + let s = sanitize_url(&url); + for keep in [ + "authorization_endpoint=a", + "oauth_client_id=b", + "token_endpoint_auth_method=c", + "client_id=d", + "redirect_uri=e", + ] { + assert!(s.contains(keep), "{keep} in {s}"); + } + assert!(!s.contains("zzz") && !s.contains("yyy"), "{s}"); + assert!(!is_sensitive_header("authorization_endpoint")); + assert!(is_sensitive_header("oauth_client_secret")); assert!(is_sensitive_header("Authorization")); } From fbd6644169a9a087e46d0a4325a21344f62eebf3 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 17:44:30 +0500 Subject: [PATCH 40/52] fix: remember every undeletable retired bundle, not just delivered ones (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One retire_queued helper for all arms (delivered, TooManySimilar, duplicate, permanent, retries exhausted); recovered crashes consult the in-memory blacklist. Test: terminal verdicts on a read-only queue are sent once. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/runtime.rs | 30 ++++--- crates/bugsee-core/tests/storage_failures.rs | 94 ++++++++++++++++++++ 3 files changed, 114 insertions(+), 12 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 6b3cdbe..4d05c12 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -187,7 +187,7 @@ Global base scope behind `RwLock`; `with_scope` pushes a thread-local overlay. * |---|---| | Default candidate unusable, a later default works | falls through; one stderr line; `StorageStatus::skipped` lists why | | Requested directory unusable, or no default usable, or `Session::begin`/`PartStore::new` fails (also in a fork child) | **memory mode** (`StorageMode::Memory`): no capture window (entries dropped), no liveness marker, native crash handler and panic snapshot not installed; handled errors and manual reports are assembled without capture context and delivered from an in-memory backlog capped at 16 (an entry counts until its delivery settles; overflow drops the newest with `DropReason::StorageUnavailable`); a `TooManySimilar` verdict is remembered in memory; a clean stop (guard drop) gives the backlog one last bounded (1.5 s) attempt, but a report still unsent when the process dies is lost; a fork child starts with an empty backlog (the parent's is the parent's to send) | -| Directory exists but cannot take NEW files (full, read-only remount) | memory mode for new reports, but the directory is still **drained and recovered from**: its queued bundles are delivered and its crashed sessions recovered (claims are best-effort there; a crash that cannot be re-queued on disk is delivered from memory). Bundles delivered but undeletable are remembered and not resent in this process; a later launch with a writable disk may resend them and the backend deduplicates. Every such rejected candidate is drained, including a full default that lost to a later writable one and the private temp tree (which must be a real directory of ours); a stale claim that cannot be unlinked counts as "no claim possible", not "held elsewhere"; the on-disk `.blacklist` seeds the in-memory one. Only a missing or non-directory path is skipped | +| Directory exists but cannot take NEW files (full, read-only remount) | memory mode for new reports, but the directory is still **drained and recovered from**: its queued bundles are delivered and its crashed sessions recovered (claims are best-effort there; a crash that cannot be re-queued on disk is delivered from memory). Bundles retired by ANY verdict (delivered, `TooManySimilar`, duplicate, permanent, retries exhausted) but undeletable are remembered and not resent in this process; a recovered crash that is dropped by a `TooManySimilar` verdict on such a tree keeps its `.alive` marker, so the next launch rebuilds it once more; a later launch with a writable disk may resend them and the backend deduplicates. Every such rejected candidate is drained, including a full default that lost to a later writable one and the private temp tree (which must be a real directory of ours); a stale claim that cannot be unlinked counts as "no claim possible", not "held elsewhere"; the on-disk `.blacklist` seeds the in-memory one. Only a missing or non-directory path is skipped | | Directory vanishes / becomes unwritable mid-run | capture appends are best-effort and dropped; the rotation tick rewrites a missing `sessions/.alive` (a wiped-and-recreated tree would otherwise make the next launch's orphan sweep drop this session's crash); a report that cannot snapshot or enqueue is rebuilt without context and held in memory, so it still goes out; `StorageStatus::degraded` turns true and `crash_recovery_available()` false | | Disk full / permission denied at enqueue, snapshot, marker | same as above; the `reports/` snapshot dir is reclaimed by `SnapshotHandle::drop` | | Crash handler cannot write its marker | the raw `open`/`write` return values are ignored; the process still dies with the original signal (watchdog bounds the handler) | diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 15cdd3d..45bffa0 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1017,7 +1017,10 @@ fn recover_one(shared: &Shared, data_dir: &Path, pending: &recovery::PendingSess Some(hook) => hook(&mut built.meta), None => true, }; - if !keep || queue::any_blacklisted(data_dir, &built.meta.signatures) { + if !keep + || mem_blacklisted(shared, &built.meta.signatures) + || queue::any_blacklisted(data_dir, &built.meta.signatures) + { recovery::discard(data_dir, pending); return; } @@ -1525,6 +1528,17 @@ fn uploader_loop( } } +/// Retire a queued report whose work is over (delivered, or abandoned for any +/// reason). If the directory will not let us delete it (read-only or full), it is +/// remembered so no later drain selects it again: otherwise a terminal verdict on +/// such a directory would be re-sent every 20 ms of a flush and every poll. +fn retire_queued(shared: &Shared, report: &queue::QueuedReport) { + queue::remove(report); + if report.bundle.exists() { + shared.delivered_stuck.lock().insert(report.bundle.clone()); + } +} + /// Queued reports in `dir` still awaiting delivery: everything on disk except /// bundles already delivered whose deletion failed. fn pending_reports(shared: &Shared, dir: &Path) -> Vec { @@ -1646,13 +1660,7 @@ fn deliver_queued( cached_endpoint.as_deref(), &mut endpoint_out, ) { - Ok(()) => { - queue::remove(report); - if report.bundle.exists() { - // Delivered, but the directory will not let us delete it. - shared.delivered_stuck.lock().insert(report.bundle.clone()); - } - } + Ok(()) => retire_queued(shared, report), // A transient failure OR a still-expired session are both retryable. Err(TransportError::Transient(_)) | Err(TransportError::SessionExpired) => { // Persist the presigned endpoint if the failure left one to resume @@ -1664,7 +1672,7 @@ fn deliver_queued( if count_failure { let n = retry + 1; if n >= UPLOAD_RETRY_CAP { - queue::remove(report); + retire_queued(shared, report); notify_dropped(shared, DropReason::RetriesExhausted); } else { queue::set_meta(report, n, now + backoff_delay(n, backoff_base)); @@ -1683,12 +1691,12 @@ fn deliver_queued( }; queue::blacklist_add(data_dir, &sigs); shared.mem_blacklist.lock().extend(sigs); - queue::remove(report); + retire_queued(shared, report); notify_dropped(shared, DropReason::TooManySimilar); } // Duplicate / permanent β€” abandon the report. Err(e) => { - queue::remove(report); + retire_queued(shared, report); notify_dropped(shared, drop_reason_for(e)); } } diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index 616fb67..58a81da 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -699,3 +699,97 @@ fn signatures_blacklisted_on_disk_are_honoured_in_memory_mode() { "only the report without the blacklisted signature is sent" ); } + +/// A terminal verdict on a queued report in a directory we cannot delete from +/// must not be re-sent on every drain (a flush retries every 20 ms, the uploader +/// polls every 5 s). +#[cfg(unix)] +#[test] +fn terminal_verdicts_on_an_undeletable_queue_are_not_resent() { + use bugsee_core::model::report::TimeWindow; + use bugsee_core::reporting::assemble_docs; + use bugsee_core::TransportError; + use std::os::unix::fs::PermissionsExt; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[derive(Clone, Copy)] + enum Verdict { + TooManySimilar, + Duplicate, + Permanent, + } + struct Terminal(Verdict, Arc); + impl bugsee_core::Transport for Terminal { + fn register_session(&self, _: &str, _: &[u8]) -> Result { + Ok("token".into()) + } + fn create_issue( + &self, + _: &str, + _: Option<&str>, + _: &[u8], + ) -> Result { + self.1.fetch_add(1, Ordering::SeqCst); + Err(match self.0 { + Verdict::TooManySimilar => TransportError::TooManySimilar { + signatures: vec!["sig".into()], + }, + Verdict::Duplicate => TransportError::DuplicateDropped, + Verdict::Permanent => TransportError::Permanent("rejected".into()), + }) + } + fn upload_bundle(&self, _: &str, _: &[u8]) -> Result<(), TransportError> { + Ok(()) + } + } + + for verdict in [ + Verdict::TooManySimilar, + Verdict::Duplicate, + Verdict::Permanent, + ] { + let dir = TempDir::new(); + let data = dir.path.join("data"); + let env = bugsee_core::model::environment::Environment::detect_with( + "t", + &Default::default(), + &Default::default(), + ); + let queued = assemble_docs( + Vec::new(), + TimeWindow { start: 0, end: 1 }, + &manual_upload_meta(), + &env, + None, + &[], + "T", + 1, + ) + .unwrap(); + bugsee_core::queue::enqueue(&data, &queued).unwrap(); + for d in [data.join("queue"), data.clone()] { + std::fs::set_permissions(d, std::fs::Permissions::from_mode(0o555)).unwrap(); + } + if std::fs::write(data.join("probe"), b"x").is_ok() { + eprintln!("skipped: permission bits are not enforced for this user"); + return; + } + + let issues = Arc::new(AtomicUsize::new(0)); + let mut config = RecorderConfig::new(&data, "T"); + config.memory_only = Some("write probe failed".into()); + let recorder = + Recorder::launch(config, Arc::new(Terminal(verdict, issues.clone()))).expect("launch"); + // A flush spins its drain loop until the deadline, the case that stormed. + let _ = recorder.flush(Duration::from_millis(600)); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!( + issues.load(Ordering::SeqCst), + 1, + "the verdict is final; the undeletable bundle must not be re-sent" + ); + drop(recorder); + restore_permissions(&data); + restore_permissions(&data.join("queue")); + } +} From bb281f1c5b3e3cc69fcf370272c9d23a454e9e8c Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 17:47:38 +0500 Subject: [PATCH 41/52] test(core): wait on an observable condition in the full-queue test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/tests/launch_recovery.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/bugsee-core/tests/launch_recovery.rs b/crates/bugsee-core/tests/launch_recovery.rs index cb9f134..387c672 100644 --- a/crates/bugsee-core/tests/launch_recovery.rs +++ b/crates/bugsee-core/tests/launch_recovery.rs @@ -210,9 +210,16 @@ fn a_full_queue_defers_a_unique_crash_to_the_worker_instead_of_discarding_it() { // The launch pass refused it (the worker may already be queueing it, so // only the outcome is asserted): it ends up queued, not discarded, and // without having used up its recovery attempts. - let deadline = Instant::now() + Duration::from_secs(10); + // Observable condition, generous timeout: a loaded CI machine can delay the + // helper thread, the worker's wait (budget plus a 5 s grace) and its pass. + let deadline = Instant::now() + Duration::from_secs(60); while queue::list_pending(&dir).len() <= queue::MAX_QUEUED_REPORTS { - assert!(Instant::now() < deadline, "the worker never recovered it"); + assert!( + Instant::now() < deadline, + "the worker never recovered it: {} queued, session marker present: {}", + queue::list_pending(&dir).len(), + dir.join("sessions").join("1.alive").exists() + ); std::thread::sleep(Duration::from_millis(20)); } drop(rec); From 97dc58bce33444bb293d65c4ae2470897c00d4fa Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 17:55:31 +0500 Subject: [PATCH 42/52] fix(scrub): narrow the oauth exclusion to public metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit oauth_verifier, oauth_pin and a bare oauth key stay redacted again; oauth_client_id/redirect/scope/issuer/endpoint/... and auth_url/uri/endpoint are public. Tests in bugsee-core and bugsee-reqwest. Refs #11 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/scrub.rs | 39 ++++++++++++++++++++++++++++++-- crates/bugsee-reqwest/src/lib.rs | 9 ++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/crates/bugsee-core/src/scrub.rs b/crates/bugsee-core/src/scrub.rs index f64429d..90bda74 100644 --- a/crates/bugsee-core/src/scrub.rs +++ b/crates/bugsee-core/src/scrub.rs @@ -305,6 +305,32 @@ pub fn is_sensitive_key(name: &str) -> bool { }) } +/// After `oauth` (the `o` is the stem's `before`), a name that is public +/// metadata (`oauth_client_id`, `oauth2_redirect_uri`, `oauth_scope`), as opposed +/// to `oauth_verifier`/`oauth_pin`/bare `oauth`, which stay secret. Token- and +/// secret-bearing names are caught by their own stems regardless. +fn public_oauth_suffix(rest: &[u8]) -> bool { + let rest = &rest[rest.iter().take_while(|c| c.is_ascii_digit()).count()..]; + [ + &b"clientid"[..], + b"clientname", + b"redirect", + b"callback", + b"scope", + b"issuer", + b"endpoint", + b"url", + b"uri", + b"version", + b"provider", + b"enabled", + b"flow", + b"config", + ] + .iter() + .any(|x| rest.starts_with(x)) +} + /// Whether `stem` occurs in `n` in a sense that is not excluded by its context. fn stem_hit(n: &[u8], stem: &[u8]) -> bool { let mut from = 0; @@ -329,7 +355,10 @@ fn stem_hit(n: &[u8], stem: &[u8]) -> bool { && [&b"endpoint"[..], b"url", b"uri"] .iter() .any(|x| rest[9..].starts_with(x))) - || before.ends_with(b"o") + || (before.ends_with(b"o") && public_oauth_suffix(rest)) + || [&b"url"[..], b"uri", b"endpoint"] + .iter() + .any(|x| rest.starts_with(x)) || rest.starts_with(b"method") || rest.starts_with(b"signingalg") } @@ -2058,8 +2087,10 @@ mod tests { "authorization_endpoint", "authorization_url", "authorization_uri", - "oauth", "oauth_client_id", + "auth_url", + "auth_endpoint", + "oauth_scope", "oauth2_redirect_uri", "token_endpoint_auth_method", "token_endpoint_auth_signing_alg_values_supported", @@ -2087,6 +2118,10 @@ mod tests { } for k in [ "authorization", + "oauth", + "oauth_verifier", + "oauth_pin", + "oauth_signature", "oauth_token", "oauth_client_secret", "oauth_access_token", diff --git a/crates/bugsee-reqwest/src/lib.rs b/crates/bugsee-reqwest/src/lib.rs index 809fc32..331d016 100644 --- a/crates/bugsee-reqwest/src/lib.rs +++ b/crates/bugsee-reqwest/src/lib.rs @@ -441,6 +441,15 @@ mod tests { assert!(!s.contains("zzz") && !s.contains("yyy"), "{s}"); assert!(!is_sensitive_header("authorization_endpoint")); assert!(is_sensitive_header("oauth_client_secret")); + let url = reqwest::Url::parse( + "https://idp.example.com/cb?oauth_verifier=vvv&auth_url=u&oauth_pin=1234", + ) + .unwrap(); + let s = sanitize_url(&url); + assert!( + s.contains("auth_url=u") && !s.contains("vvv") && !s.contains("1234"), + "{s}" + ); assert!(is_sensitive_header("Authorization")); } From 19a26ee6b28ac0d1ff7f52529284310c663cbce0 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 17:56:42 +0500 Subject: [PATCH 43/52] Merge origin/fix/issue-9 into fix/issue-11 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adapt the new storage_failures test to assemble_docs' scrubber argument. Refs #11 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/tests/storage_failures.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/bugsee-core/tests/storage_failures.rs b/crates/bugsee-core/tests/storage_failures.rs index c5fcecb..eb6502f 100644 --- a/crates/bugsee-core/tests/storage_failures.rs +++ b/crates/bugsee-core/tests/storage_failures.rs @@ -765,6 +765,7 @@ fn terminal_verdicts_on_an_undeletable_queue_are_not_resent() { &[], "T", 1, + None, ) .unwrap(); bugsee_core::queue::enqueue(&data, &queued).unwrap(); From 7ab8a9045b8b8428168d88bfe16c86954d1dc682 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 18:07:46 +0500 Subject: [PATCH 44/52] fix(core): declare kernel32 calls once (clashing_extern_declarations on Windows) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit process.rs and crash_flush.rs declared WaitForSingleObject with different handle types. Both now use one win32 module; no other duplicate extern declarations exist in bugsee-core, bugsee-native or bugsee. Verified with a standalone windows-msvc clippy of the affected modules. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/crash_flush.rs | 27 ++++++++----------- crates/bugsee-core/src/lib.rs | 2 ++ crates/bugsee-core/src/process.rs | 17 +----------- crates/bugsee-core/src/win32.rs | 39 +++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 32 deletions(-) create mode 100644 crates/bugsee-core/src/win32.rs diff --git a/crates/bugsee-core/src/crash_flush.rs b/crates/bugsee-core/src/crash_flush.rs index 7ea36eb..afd9d78 100644 --- a/crates/bugsee-core/src/crash_flush.rs +++ b/crates/bugsee-core/src/crash_flush.rs @@ -386,20 +386,13 @@ mod imp { #[cfg(windows)] mod imp { use super::{serve_request, ACKED, REQUESTED}; + use crate::win32::{ + CreateEventW, GetCurrentThreadId, Handle, QueryPerformanceCounter, + QueryPerformanceFrequency, SetEvent, WaitForSingleObject, WAIT_OBJECT_0, + }; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; - type Hnd = usize; - - extern "system" { - fn CreateEventW(attrs: *const u8, manual: i32, initial: i32, name: *const u16) -> Hnd; - fn SetEvent(h: Hnd) -> i32; - fn WaitForSingleObject(h: Hnd, ms: u32) -> u32; - fn GetCurrentThreadId() -> u32; - fn QueryPerformanceCounter(v: *mut i64) -> i32; - fn QueryPerformanceFrequency(v: *mut i64) -> i32; - } - /// Auto-reset events, never closed: the handler signals `WAKE`, the helper /// signals `ACK`. (`0` when none.) static WAKE: AtomicUsize = AtomicUsize::new(0); @@ -418,16 +411,18 @@ mod imp { CreateEventW(core::ptr::null(), 0, 0, core::ptr::null()), ) }; - if wake == 0 || ack == 0 { + if wake.is_null() || ack.is_null() { return Err(std::io::Error::last_os_error()); } + // Raw handles are not `Send`; carry them as integers. + let (wake, ack) = (wake as usize, ack as usize); std::thread::Builder::new() .name("bugsee-crash-flush".into()) .spawn(move || { let _me = super::register_pipeline_thread(); loop { // SAFETY: waits on an event that is never closed. - if unsafe { WaitForSingleObject(wake, u32::MAX) } != 0 { + if unsafe { WaitForSingleObject(wake as Handle, u32::MAX) } != WAIT_OBJECT_0 { break; } serve_request(); @@ -442,7 +437,7 @@ mod imp { let h = ACK.load(Ordering::Acquire); if h != 0 { // SAFETY: signals a live event. - unsafe { SetEvent(h) }; + unsafe { SetEvent(h as Handle) }; } } @@ -468,7 +463,7 @@ mod imp { let (start, freq) = now_ticks(); let budget_ms = budget.as_millis().min(i64::MAX as u128 / freq as u128 / 2) as i64; // SAFETY: SetEvent/WaitForSingleObject take no locks we could be holding. - unsafe { SetEvent(wake) }; + unsafe { SetEvent(wake as Handle) }; while ACKED.load(Ordering::Acquire) < ticket { let (now, _) = now_ticks(); let elapsed_ms = (now - start).saturating_mul(1000) / freq; @@ -480,7 +475,7 @@ mod imp { // consumed the (auto-reset) signal. let ms = left_ms.clamp(1, 5) as u32; // SAFETY: as above. - unsafe { WaitForSingleObject(ack, ms) }; + unsafe { WaitForSingleObject(ack as Handle, ms) }; } true } diff --git a/crates/bugsee-core/src/lib.rs b/crates/bugsee-core/src/lib.rs index 8a70967..bd11cbf 100644 --- a/crates/bugsee-core/src/lib.rs +++ b/crates/bugsee-core/src/lib.rs @@ -33,6 +33,8 @@ pub mod session; pub mod signature; pub mod transport; pub mod util; +#[cfg(windows)] +mod win32; pub use model::{ entry::{ diff --git a/crates/bugsee-core/src/process.rs b/crates/bugsee-core/src/process.rs index c857e46..a30b8bc 100644 --- a/crates/bugsee-core/src/process.rs +++ b/crates/bugsee-core/src/process.rs @@ -98,22 +98,7 @@ pub(crate) fn is_alive(_pid: u32) -> bool { } #[cfg(windows)] -mod win { - //! The few kernel32 calls needed, declared directly: kernel32 is always - //! linked, so this avoids a dependency for three functions. - use std::ffi::c_void; - - pub type Handle = *mut c_void; - pub const SYNCHRONIZE: u32 = 0x0010_0000; - pub const WAIT_TIMEOUT: u32 = 258; - pub const ERROR_ACCESS_DENIED: i32 = 5; - - extern "system" { - pub fn OpenProcess(access: u32, inherit: i32, pid: u32) -> Handle; - pub fn CloseHandle(handle: Handle) -> i32; - pub fn WaitForSingleObject(handle: Handle, millis: u32) -> u32; - } -} +use crate::win32 as win; #[cfg(test)] mod tests { diff --git a/crates/bugsee-core/src/win32.rs b/crates/bugsee-core/src/win32.rs new file mode 100644 index 0000000..066123b --- /dev/null +++ b/crates/bugsee-core/src/win32.rs @@ -0,0 +1,39 @@ +// +// win32.rs +// bugsee-core +// +// Copyright Β© 2026 Bugsee. All rights reserved. +// + +//! The few kernel32 calls this crate needs, declared ONCE. +//! +//! kernel32 is always linked, so declaring them directly avoids a dependency. +//! They live in one module on purpose: two declarations of the same symbol with +//! different Rust types (a `HANDLE` as `*mut c_void` in one place, `usize` in +//! another) trip `clashing_extern_declarations`, a crate-wide lint. Any new +//! kernel32 call in this crate goes here. + +use std::ffi::c_void; + +pub type Handle = *mut c_void; + +pub const SYNCHRONIZE: u32 = 0x0010_0000; +pub const WAIT_OBJECT_0: u32 = 0; +pub const WAIT_TIMEOUT: u32 = 258; +pub const ERROR_ACCESS_DENIED: i32 = 5; + +extern "system" { + pub fn OpenProcess(access: u32, inherit: i32, pid: u32) -> Handle; + pub fn CloseHandle(handle: Handle) -> i32; + pub fn WaitForSingleObject(handle: Handle, millis: u32) -> u32; + pub fn CreateEventW( + attrs: *const c_void, + manual_reset: i32, + initial: i32, + name: *const u16, + ) -> Handle; + pub fn SetEvent(event: Handle) -> i32; + pub fn GetCurrentThreadId() -> u32; + pub fn QueryPerformanceCounter(value: *mut i64) -> i32; + pub fn QueryPerformanceFrequency(value: *mut i64) -> i32; +} From 7e1adabccb395a632805c0f7c1b97ab387d1f239 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 18:08:34 +0500 Subject: [PATCH 45/52] fix(scrub): free-text oauth_verifier/oauth_pin and short pin/otp codes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Free-text keyed matching now follows the structured matcher for the OAuth 1.0a verifier/PIN and redacts 4-10 digit pin=/otp= values. Refs #11 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/scrub.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/bugsee-core/src/scrub.rs b/crates/bugsee-core/src/scrub.rs index 90bda74..3afafc1 100644 --- a/crates/bugsee-core/src/scrub.rs +++ b/crates/bugsee-core/src/scrub.rs @@ -241,6 +241,9 @@ const TEXT_STRONG: &[&str] = &[ "bearer", "sessionid", "sessiontoken", + // OAuth 1.0a verifier/PIN: structured matching redacts these via `auth`. + "verifier", + "oauthpin", ]; /// Free-text names that are also ordinary words; see the module docs. @@ -516,6 +519,8 @@ enum KeyClass { Cookie, /// A key the host listed in `sensitive_keys`: always redacted. Explicit, + /// `pin` / `otp`: short numeric codes, redacted after `=` or a quoted key. + Pin, } impl Scrubber { @@ -935,6 +940,8 @@ impl Scrubber { KeyClass::Explicit } else if TEXT_STRONG.iter().any(|k| n.ends_with(k.as_bytes())) { KeyClass::Strong + } else if n == b"pin" || n == b"otp" { + KeyClass::Pin } else if EXACT.iter().any(|k| n == k.as_bytes()) || TEXT_AMBIGUOUS_SUFFIX .iter() @@ -1244,6 +1251,13 @@ impl Scrubber { if !(sep == b'=' || quoted_key) || !secretish { continue; } + } else if class == KeyClass::Pin { + // 4-10 digits; `pin=home` or "otp: sent" are prose. + let v = &b[vs..ve]; + let codeish = (4..=10).contains(&v.len()) && v.iter().all(u8::is_ascii_digit); + if !(sep == b'=' || quoted_key) || !codeish { + continue; + } } else if class == KeyClass::Strong && sep == b':' && !quoted_key @@ -1815,6 +1829,18 @@ mod tests { // Ambiguous names need `=`/quoted key and a secret-looking value. assert_eq!(s("session=abcd1234efgh"), "session=[redacted]"); assert_eq!(s("key=short"), "key=short"); + // OAuth 1.0a verifier/PIN and short OTP/PIN codes in free text. + assert_eq!( + s("oauth_verifier=abcXYZ&x=1"), + "oauth_verifier=[redacted]&x=1" + ); + assert_eq!(s("oauth_pin=1234"), "oauth_pin=[redacted]"); + assert_eq!(s("pin=1234 otp=654321"), "pin=[redacted] otp=[redacted]"); + assert_eq!(s(r#"{"otp": "654321"}"#), r#"{"otp": "[redacted]"}"#); + assert_eq!( + s("pin=home otp: sent spinner=12"), + "pin=home otp: sent spinner=12" + ); assert_eq!(s("auth: failed for user"), "auth: failed for user"); } From 605126d00ddb2ee8585a4bc6e34bc93bbfb0e186 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 18:19:52 +0500 Subject: [PATCH 46/52] fix(scrub): structured matching redacts code_verifier (PKCE) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #11 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/scrub.rs | 6 +++++- crates/bugsee-reqwest/src/lib.rs | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/bugsee-core/src/scrub.rs b/crates/bugsee-core/src/scrub.rs index 3afafc1..631f045 100644 --- a/crates/bugsee-core/src/scrub.rs +++ b/crates/bugsee-core/src/scrub.rs @@ -214,6 +214,8 @@ const STEMS: &[&str] = &[ "xsrf", "bearer", "jwt", + // OAuth 1.0a `oauth_verifier` and PKCE `code_verifier`. + "verifier", ]; /// Names matched only exactly (too ambiguous to match as substrings). @@ -241,7 +243,7 @@ const TEXT_STRONG: &[&str] = &[ "bearer", "sessionid", "sessiontoken", - // OAuth 1.0a verifier/PIN: structured matching redacts these via `auth`. + // OAuth 1.0a verifier/PIN and PKCE `code_verifier` (kept in step with `STEMS`). "verifier", "oauthpin", ]; @@ -1835,6 +1837,7 @@ mod tests { "oauth_verifier=[redacted]&x=1" ); assert_eq!(s("oauth_pin=1234"), "oauth_pin=[redacted]"); + assert_eq!(s("code_verifier=abcXYZ123"), "code_verifier=[redacted]"); assert_eq!(s("pin=1234 otp=654321"), "pin=[redacted] otp=[redacted]"); assert_eq!(s(r#"{"otp": "654321"}"#), r#"{"otp": "[redacted]"}"#); assert_eq!( @@ -2146,6 +2149,7 @@ mod tests { "authorization", "oauth", "oauth_verifier", + "code_verifier", "oauth_pin", "oauth_signature", "oauth_token", diff --git a/crates/bugsee-reqwest/src/lib.rs b/crates/bugsee-reqwest/src/lib.rs index 331d016..5412de4 100644 --- a/crates/bugsee-reqwest/src/lib.rs +++ b/crates/bugsee-reqwest/src/lib.rs @@ -441,6 +441,7 @@ mod tests { assert!(!s.contains("zzz") && !s.contains("yyy"), "{s}"); assert!(!is_sensitive_header("authorization_endpoint")); assert!(is_sensitive_header("oauth_client_secret")); + assert!(is_sensitive_header("code_verifier")); let url = reqwest::Url::parse( "https://idp.example.com/cb?oauth_verifier=vvv&auth_url=u&oauth_pin=1234", ) From acee6dbd3b788adbc2e62801da03842fccda7bd1 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 19:08:16 +0500 Subject: [PATCH 47/52] docs: keep the corrected native feature comment from main --- crates/bugsee/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bugsee/Cargo.toml b/crates/bugsee/Cargo.toml index 5cf37df..814f254 100644 --- a/crates/bugsee/Cargo.toml +++ b/crates/bugsee/Cargo.toml @@ -34,7 +34,7 @@ default = ["net", "panic", "native", "telemetry"] net = ["dep:ureq"] # Global Rust panic observer + catch_unwind boundary guards. panic = ["dep:bugsee-panic"] -# Native fatal-crash handler (signal/Mach exception) + next-launch recovery. +# Native fatal-crash handler (POSIX signal / Windows SEH) + next-launch recovery. native = ["dep:bugsee-native"] # System/process telemetry sampling (traces.system) via sysinfo. telemetry = ["dep:sysinfo"] From 1ecfca866ecccdf02dbe5a7f20edd65ca6dfcc2c Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 19:17:23 +0500 Subject: [PATCH 48/52] fix: de-duplicate the unix libc dependency after merging main (#27's optional form vs the unconditional one) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee/Cargo.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/bugsee/Cargo.toml b/crates/bugsee/Cargo.toml index a7fc135..068b3f2 100644 --- a/crates/bugsee/Cargo.toml +++ b/crates/bugsee/Cargo.toml @@ -17,6 +17,11 @@ backtrace = "0.3" ureq = { version = "2", optional = true, features = ["json"] } sysinfo = { version = "0.33", optional = true, default-features = false, features = ["system", "disk"] } +# Unconditional on unix: the data-directory ownership checks (`options.rs`) call +# `geteuid`, and the `telemetry` feature's local-UTC-offset / hardware-model probes +# (`hostinfo.rs`) call libc too. It is already in the tree via `bugsee-core` / +# `bugsee-native`, so this adds nothing; keeping it non-optional is what lets +# `--no-default-features` build. [target.'cfg(unix)'.dependencies] libc = "0.2" @@ -32,7 +37,7 @@ panic = ["dep:bugsee-panic"] # Native fatal-crash handler (POSIX signal / Windows SEH) + next-launch recovery. native = ["dep:bugsee-native"] # System/process telemetry sampling (traces.system) via sysinfo. -telemetry = ["dep:sysinfo", "dep:libc"] +telemetry = ["dep:sysinfo"] # The crash matrix needs the REAL main thread: an uncaught panic must unwind out # of `main`, and the panic observer treats any other thread as "a thread died", @@ -43,10 +48,5 @@ name = "crash_e2e_matrix" harness = false required-features = ["native"] -# Local UTC offset and the hardware model are libc calls on unix; `libc` is -# already in the tree via `bugsee-core` / `bugsee-native`, so this adds nothing. -[target.'cfg(unix)'.dependencies] -libc = { version = "0.2", optional = true } - [target.'cfg(unix)'.dev-dependencies] libc = "0.2" From 0f94b90b90a6e491f519d6f63d7d779c70f3bc21 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 19:33:36 +0500 Subject: [PATCH 49/52] fix(core): make the crash-flush handshake robust on slow runners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the windows flake in a_stuck_worker_costs_only_the_budget: the helper's ack wait (250 ms) was shorter than the 5 s budget the test asked for. After un-wedging, the worker had to drain a backlog (file creation on Windows is slow) and the helper gave up before the ack, never retrying, so the handler waited out its whole budget. The helper now waits 10 s (it only bounds a permanently wedged worker). The test also stopped assuming a 150 ms sleep wedges the worker (it waits for the sampler to report entry), uses generous budgets and bounds, and no longer relies on sleeps. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- crates/bugsee-core/src/runtime.rs | 8 ++- crates/bugsee-core/tests/crash_flush_e2e.rs | 72 ++++++++++++--------- 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 95548af..06cdf86 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1147,8 +1147,12 @@ fn spawn_pipeline( } /// How long the helper waits for the worker to acknowledge a crash-time flush. -/// The crash handler gives up on its own, sooner; this only bounds the helper. -const CRASH_FLUSH_ACK_TIMEOUT: Duration = Duration::from_millis(250); +/// The crash handler gives up on its own, far sooner (its budget); this only +/// bounds how long a permanently wedged worker can occupy the helper. It must +/// outlast any legitimate drain (the queue is capped), or a slow-but-healthy +/// worker would make the helper report failure for a request whose budget was +/// longer than this. +const CRASH_FLUSH_ACK_TIMEOUT: Duration = Duration::from_secs(10); /// Register this pipeline as the process's crash-time flush target. A failure /// (no pipe/event available) just forgoes the best-effort flush. diff --git a/crates/bugsee-core/tests/crash_flush_e2e.rs b/crates/bugsee-core/tests/crash_flush_e2e.rs index 316bfae..790dd2d 100644 --- a/crates/bugsee-core/tests/crash_flush_e2e.rs +++ b/crates/bugsee-core/tests/crash_flush_e2e.rs @@ -98,10 +98,15 @@ fn everything_queued_before_the_request_is_on_disk_when_it_confirms() { drop(recorder); } -struct Blocker(Arc); +/// Blocks the worker inside `sample` until released, and says when it got there. +struct Blocker { + release: Arc, + entered: Arc, +} impl TelemetrySampler for Blocker { fn sample(&mut self) -> Vec<(String, Value)> { - while !self.0.load(Ordering::SeqCst) { + self.entered.store(true, Ordering::SeqCst); + while !self.release.load(Ordering::SeqCst) { std::thread::sleep(Duration::from_millis(5)); } Vec::new() @@ -113,12 +118,20 @@ fn a_stuck_worker_costs_only_the_budget() { let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); let dir = TempDir::new(); let release = Arc::new(AtomicBool::new(false)); + let entered = Arc::new(AtomicBool::new(false)); let mut cfg = RecorderConfig::new(&dir.0, "TOKEN"); cfg.rotate_interval = Duration::from_millis(20); - cfg.sampler = Some(Box::new(Blocker(Arc::clone(&release)))); + cfg.sampler = Some(Box::new(Blocker { + release: Arc::clone(&release), + entered: Arc::clone(&entered), + })); let recorder = Recorder::launch(cfg, Arc::new(MockTransport::default())).unwrap(); - // Let a tick start and wedge the worker inside the sampler. - std::thread::sleep(Duration::from_millis(150)); + // Wait (observably, not by sleeping) until the worker is wedged in the sampler. + let deadline = Instant::now() + Duration::from_secs(60); + while !entered.load(Ordering::SeqCst) { + assert!(Instant::now() < deadline, "the worker never reached a tick"); + std::thread::sleep(Duration::from_millis(5)); + } for i in 0..100 { recorder.capture(event(i)); } @@ -126,11 +139,13 @@ fn a_stuck_worker_costs_only_the_budget() { let t = Instant::now(); assert!(!flush_in_flight(Duration::from_millis(50))); let took = t.elapsed(); - assert!(took < Duration::from_millis(500), "bounded, took {took:?}"); + assert!(took < Duration::from_secs(5), "bounded, took {took:?}"); - // Un-wedge: the same request now succeeds (the helper is not poisoned). + // Un-wedge: the same request now succeeds (the helper is not poisoned). The + // budget is generous: this proves the handshake recovers, not how fast a + // loaded CI machine drains. release.store(true, Ordering::SeqCst); - assert!(flush_in_flight(Duration::from_secs(5))); + assert!(flush_in_flight(Duration::from_secs(60))); assert_eq!(records_on_disk(&dir.0), 100); drop(recorder); } @@ -156,7 +171,7 @@ fn dropping_the_recorder_stops_flushing_into_it() { Arc::new(MockTransport::default()), ) .unwrap(); - assert!(flush_in_flight(Duration::from_secs(2))); + assert!(flush_in_flight(Duration::from_secs(30))); assert_eq!( calls.load(Ordering::SeqCst), 0, @@ -166,8 +181,8 @@ fn dropping_the_recorder_stops_flushing_into_it() { // Nothing is registered now, so a request is answered at once rather than // waited out, and the replaced sentinel is not called. let t = Instant::now(); - assert!(flush_in_flight(Duration::from_secs(2))); - assert!(t.elapsed() < Duration::from_millis(200), "no wait at all"); + assert!(flush_in_flight(Duration::from_secs(30))); + assert!(t.elapsed() < Duration::from_secs(10), "no wait at all"); assert_eq!(calls.load(Ordering::SeqCst), 0); } @@ -180,7 +195,7 @@ fn memory_mode_does_not_take_over_the_target() { let mut cfg = RecorderConfig::new(&dir.0, "TOKEN"); cfg.memory_only = Some("test".into()); let recorder = Recorder::launch(cfg, Arc::new(MockTransport::default())).unwrap(); - assert!(flush_in_flight(Duration::from_secs(2))); + assert!(flush_in_flight(Duration::from_secs(30))); assert_eq!(calls.load(Ordering::SeqCst), 1, "still the sentinel"); drop(recorder); } @@ -193,11 +208,11 @@ fn a_responsive_flusher_is_confirmed_promptly() { let (calls, f) = counter(); let cf = CrashFlush::install(f).unwrap(); let t = Instant::now(); - assert!(flush_in_flight(Duration::from_millis(500))); - assert!(t.elapsed() < Duration::from_millis(250)); + assert!(flush_in_flight(Duration::from_secs(30))); + assert!(t.elapsed() < Duration::from_secs(5)); assert_eq!(calls.load(Ordering::SeqCst), 1); drop(cf); - assert!(flush_in_flight(Duration::from_millis(500))); + assert!(flush_in_flight(Duration::from_secs(30))); assert_eq!(calls.load(Ordering::SeqCst), 1, "deregistered: not called"); } @@ -208,7 +223,7 @@ fn the_last_registered_target_wins() { let (b_calls, b) = counter(); let a = CrashFlush::install(a).unwrap(); let b = CrashFlush::install(b).unwrap(); - assert!(flush_in_flight(Duration::from_millis(500))); + assert!(flush_in_flight(Duration::from_secs(30))); assert_eq!( ( a_calls.load(Ordering::SeqCst), @@ -218,7 +233,7 @@ fn the_last_registered_target_wins() { ); // Dropping the one that is not current must not deregister the current one. drop(a); - assert!(flush_in_flight(Duration::from_millis(500))); + assert!(flush_in_flight(Duration::from_secs(30))); assert_eq!(b_calls.load(Ordering::SeqCst), 2); drop(b); } @@ -239,11 +254,9 @@ fn a_stuck_flusher_costs_the_budget_and_no_more() { assert!(!flush_in_flight(Duration::from_millis(50))); let took = t.elapsed(); assert!(took >= Duration::from_millis(45), "{took:?}"); - assert!(took < Duration::from_millis(400), "{took:?}"); + assert!(took < Duration::from_secs(5), "{took:?}"); release.store(true, Ordering::SeqCst); drop(cf); - // Let the helper finish the stuck call before the next test. - std::thread::sleep(Duration::from_millis(50)); } #[test] @@ -264,8 +277,7 @@ fn a_panicking_flusher_does_not_kill_the_helper() { while n.load(Ordering::SeqCst) == 0 { std::thread::sleep(Duration::from_millis(5)); } - std::thread::sleep(Duration::from_millis(50)); - assert!(flush_in_flight(Duration::from_millis(500))); + assert!(flush_in_flight(Duration::from_secs(30))); drop(cf); } @@ -277,12 +289,12 @@ fn a_pipeline_thread_does_not_wait_for_itself() { let t = Instant::now(); let r = std::thread::spawn(|| { let _p = bugsee_core::crash_flush::register_pipeline_thread(); - flush_in_flight(Duration::from_secs(2)) + flush_in_flight(Duration::from_secs(30)) }) .join() .unwrap(); assert!(!r, "skipped"); - assert!(t.elapsed() < Duration::from_millis(200)); + assert!(t.elapsed() < Duration::from_secs(10)); assert_eq!(calls.load(Ordering::SeqCst), 0); drop(cf); } @@ -345,8 +357,8 @@ fn a_dying_process_recovers_nothing_and_leaves_the_session_pending() { r1.capture(event(i)); } let t = Instant::now(); - assert!(flush_in_flight(Duration::from_secs(2))); - assert!(t.elapsed() < Duration::from_millis(500)); + assert!(flush_in_flight(Duration::from_secs(30))); + assert!(t.elapsed() < Duration::from_secs(30)); assert_eq!(records_on_disk(&dir.0), 200); drop(r1); @@ -366,14 +378,14 @@ fn a_fork_child_gets_its_own_helper() { let _g = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); let (_calls, f) = counter(); let cf = CrashFlush::install(f).unwrap(); - assert!(flush_in_flight(Duration::from_millis(500))); + assert!(flush_in_flight(Duration::from_secs(30))); // SAFETY: the child only calls the code under test and `_exit`s. let pid = unsafe { libc::fork() }; if pid == 0 { - let before = flush_in_flight(Duration::from_millis(500)); + let before = flush_in_flight(Duration::from_secs(30)); let (_c, g) = counter(); let mine = CrashFlush::install(g).unwrap(); - let after = flush_in_flight(Duration::from_millis(500)); + let after = flush_in_flight(Duration::from_secs(30)); std::mem::forget(mine); let code = if !before && after { 0 } else { 1 }; // SAFETY: leaving a forked child. @@ -387,7 +399,7 @@ fn a_fork_child_gets_its_own_helper() { "child status {status}" ); assert!( - flush_in_flight(Duration::from_millis(500)), + flush_in_flight(Duration::from_secs(30)), "parent unaffected" ); drop(cf); From 54c5414e9e377b7f478747b44935cfae83162bc9 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 19:42:42 +0500 Subject: [PATCH 50/52] docs: describe backend behaviour generically in release-health notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #12 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 8 ++++---- crates/bugsee-core/src/health.rs | 9 ++++----- crates/bugsee-core/src/runtime.rs | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 3503ee0..1083522 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -292,9 +292,9 @@ That last case previously produced **no report whatsoever** β€” the SDK shut dow ### Release health: sessions and crash-free rates (#12) β€” experimental -**What the backend does today** (`appserver` `components/shared/statistics/index.js`, `formatter.js`): a session is counted when it is created (`POST /v2/sessions`, `statsInc.sessions`), a crash when a crash **report** is received (`logIssue`, `issues.crashes`), and `crashes_free_sessions = (sessions - crashes) / sessions`, per day and per release (app `version`/`build`). There is **no endpoint** that accepts a session end, status, duration or error count, no `ok/errored/crashed/abnormal` model, and no spec (`specs/`, `report-bundle-structure/`) mentions one. +**What the backend does today.** It counts a session once, when the session is registered, and a crash when a crash **report** is received. The crash-free session rate is derived from those two counts (sessions minus crashes, over sessions), per day and per release (app version and build). It has **no way for an SDK to report** a session's end, status, duration or error count, and no `ok/errored/crashed/abnormal` model; no document of the wire contract mentions one. -**The design gap that matters most.** The Rust transport registers its session lazily, inside `deliver`, so only a process that uploads something ever creates a backend session. A healthy run that reports nothing is never counted while its crash report is, and the rate collapses toward 0%. With `LaunchOptions::release_health(true)` (experimental, **off by default**) the uploader thread therefore registers the session **eagerly, once per launch**, right after launch (only when the launch has a disk session: a memory-mode run could never contribute its crash, so counting it would read healthy): best effort, never on the caller's thread, never an error for launch, and offline-tolerant (a failure is dropped, retried at most every 30 s, and the first delivery registers lazily if it is still unregistered). The token is the one `deliver` uses, so there is exactly one `POST /v2/sessions` per online launch. The session of a run that died is registered by that run itself; its crash report arrives with the next process, which counts it as a crash on its own day and release (the backend has no link between the two), as for any recovered crash. A fork child inherits the parent's token and registers nothing of its own. +**The design gap that matters most.** The Rust transport registers its session lazily, inside `deliver`, so only a process that uploads something ever creates a backend session. A healthy run that reports nothing is never counted while its crash report is, and the rate collapses toward 0%. With `LaunchOptions::release_health(true)` (experimental, **off by default**) the uploader thread therefore registers the session **eagerly, once per launch**, right after launch (only when the launch has a disk session: a memory-mode run could never contribute its crash, so counting it would read healthy): best effort, never on the caller's thread, never an error for launch, and offline-tolerant (a failure is dropped, retried at most every 30 s, and the first delivery registers lazily if it is still unregistered). The token is the one `deliver` uses, so there is exactly one session registration per online launch. The session of a run that died is registered by that run itself; its crash report arrives with the next process, which counts it as a crash on its own day and release (the backend has no link between the two), as for any recovered crash. A fork child inherits the parent's token and registers nothing of its own. **Local record (no backend half).** The same option keeps a durable record per session in `health/`: `.live` while running and `.json` when finished (`ok`, `errored`, `crashed`, `abnormal`; duration, error count, release). Nothing is sent. It is unstable by design: the types live in a `doc(hidden)` module, are `#[non_exhaustive]`, and the facade offers only `doc(hidden)` accessors (`Bugsee::__finished_sessions` and friends); there is no stable API for a contract that does not exist. @@ -309,7 +309,7 @@ Rules. Documented limits. Records of legacy and extra recovery directories are not surfaced; the newest 256 finished records are kept and older ones are dropped even if unacknowledged; right after launch a recovered earlier-run crash may not be listed yet (recovery past the launch-path budget is asynchronous); a server that is ended by SIGTERM, `exec` or `_exit` without dropping its guard is `abnormal`, which inflates the crashed rate, so end the run with a clean stop: call `Bugsee::stop` (or drop the guard) from the termination handler; once the shutdown contract of #8 (PR #31) lands, `Bugsee::shutdown` and the opt-in `Bugsee::install_termination_flush(timeout)` do this on SIGTERM/SIGINT (neither exists on this branch yet); error counts of a session that died are approximate (up to one tick behind). The crash handler and panic hook add no work of their own for this; the hook gains no new action, and the inline worker-thread report (already there) now also bumps the error counter. -**Blocked on the backend.** (1) An ingest endpoint, or fields on session-create/close, for `{session_id, started_at, ended_at, duration, status, errors, release, environment}`, authenticated like the other calls and idempotent on `session_id`; until then crashed and abnormal sessions can only be counted through crash reports. (2) How `crashed`/`abnormal` should feed `crashes_free_sessions`: today a session whose crash report was deduplicated (`12003`) or blacklisted (`12004`) counts as crash-free server-side while the SDK records it `crashed`. (3) Crash-free **users**: the backend's user counts key on `environment.hardware.device_id` (`session.service.js`), which the Rust environment does not send; a stable anonymous id is needed first. (4) Sampling for servers with many short sessions (each launch is one `POST /v2/sessions`), to be decided with the endpoint. (5) A `Transport::report_session` method and offline queueing of finished records, which the durable queue and recover-lock claims can carry once the shape exists. +**Blocked on the backend.** (1) An ingest endpoint, or fields on session-create/close, for `{session_id, started_at, ended_at, duration, status, errors, release, environment}`, authenticated like the other calls and idempotent on `session_id`; until then crashed and abnormal sessions can only be counted through crash reports. (2) How `crashed`/`abnormal` should feed the crash-free rate: today a session whose crash report the server dropped as a duplicate or refused as too similar counts as crash-free server-side while the SDK records it `crashed`. (3) Crash-free **users**: the backend counts users by a stable per-device identifier in the environment, which the Rust environment does not send; a stable anonymous id is needed first. (4) Sampling for servers with many short sessions (each launch is one session registration), to be decided with the endpoint. (5) A `Transport::report_session` method and offline queueing of finished records, which the durable queue and recover-lock claims can carry once the shape exists. ## 11. Concurrency & threading @@ -446,7 +446,7 @@ worker jobs (alongside `ios`/`javascript`), not Android's `mappings/` + ProGuard `mapping.*` path β€” the worker resolves a Rust minidump against that unified store, so the two halves must agree or every Rust minidump stays unsymbolicated. -**Still open, release health (#12):** the backend has no session end/status/duration ingest and counts a session only at session-create; see Β§10, *Release health*, for the contract the Rust SDK would need and why crash-free users also needs `hardware.device_id`. +**Still open, release health (#12):** the backend has no session end/status/duration ingest and counts a session only at session-create; see Β§10, *Release health*, for the contract the Rust SDK would need and why crash-free users also needs a stable anonymous device identifier. **Still open (not required for ingestion):** the `viewer` dashboard has no Rust-specific rendering yet (reports display through the generic managed path), diff --git a/crates/bugsee-core/src/health.rs b/crates/bugsee-core/src/health.rs index ec0348e..1034700 100644 --- a/crates/bugsee-core/src/health.rs +++ b/crates/bugsee-core/src/health.rs @@ -9,11 +9,10 @@ //! (issue #12), the SDK half of release health / crash-free sessions. //! //! **Experimental and unstable; there is no backend contract for it.** The -//! backend derives crash-free sessions itself: it counts one session per -//! `POST /v2/sessions` and one crash per crash report received (`appserver` -//! `shared/statistics`: `sessions`, `issues.crashes`; crash-free = -//! `(sessions - crashes) / sessions`), and it has no endpoint that accepts a -//! session's end, status, duration or error count. These records are therefore +//! backend derives crash-free sessions itself: it counts a session once, when +//! it is registered, and a crash when a crash report is received (the rate is +//! sessions minus crashes, over sessions), and it has no way for an SDK to +//! report a session's end, status, duration or error count. These records are therefore //! kept locally and never sent; what release health does send is the session //! registration itself, made eagerly at launch (see `runtime`), because a //! session the backend never hears of cannot be counted. Off by default. diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index a41c4e9..5ecc6c5 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1961,7 +1961,7 @@ const EAGER_RETRY: Duration = Duration::from_secs(30); /// Register the backend session now, once: a no-op when a token is already held /// (an earlier attempt, or a delivery, got one), so there is exactly one -/// `POST /v2/sessions` per launch that is online. Never fails the caller: errors +/// session registration per launch that is online. Never fails the caller: errors /// are dropped and the lazy path in [`transport::deliver`] registers later. The /// lock is not held across the network call. fn register_eagerly(shared: &Shared) { From 4858ce7c3fd30d55c4f1048c1e3d0041390e34c3 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 21:05:50 +0500 Subject: [PATCH 51/52] docs: keep design notes generic (no private-component references) (#36) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generalise the mobile native capture and out-of-process monitor design notes, the FFI audit and a few code comments so they describe behaviour in generic terms (the backend, the ingestion pipeline, existing report variants) instead of naming components of other projects. No behaviour change. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 2 +- crates/bugsee-core/src/runtime.rs | 2 +- crates/bugsee/src/hostinfo.rs | 2 +- docs/design/mobile-native-capture.md | 94 +++++++++++++-------------- docs/design/out-of-process-monitor.md | 6 +- docs/ffi-audit.md | 2 +- 6 files changed, 54 insertions(+), 54 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 34321f5..74c6955 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -223,7 +223,7 @@ Tests: `bugsee-core/tests/storage_failures.rs`, `bugsee/tests/storage_fallback.r - **Ordering with the crash handlers.** The pass is the last thing `launch_with` does, after the native handler and the panic hook are installed and outside every lock, so a fault inside it is recorded with a stack against THIS session (its marker and `crash.info` path are already this generation's) and reported by the next launch. The previous session's evidence is untouched until its report is queued, and `bump_attempts` runs before processing, so a session that crashes the assemble path is abandoned after `MAX_RECOVERY_ATTEMPTS` launches instead of looping. On a re-launch within one process the native handler is replaced with one for the new generation before the pass runs; a fault in the pass is therefore the new generation's. - **Worker coordination.** With the pass armed, the worker holds its own recovery back until the pass is done (bounded by budget plus 5 s, so a wedged callback cannot stall capture for good), then skips generations the pass claimed (`launch_attempted`, recorded only after winning the #14 claim). Claims keep the two exclusive in any interleaving. A host using `Recorder::launch` directly with a non-zero budget must call `recover_on_launch_path` or the worker waits that grace out; the core default budget is zero, i.e. no pass. - **Bounds.** At most 8 sessions (`launch_recovery_max_sessions`), no new session after the budget (checked between sessions, so one huge session can overrun it, and the newest is always attempted), and at most 16 times that many candidates probed. Every session marker NAME is read (`readdir` order is not generation order, so a capped scan could miss the newest), sorted, and only the newest are probed; up to as many older sessions that already have an `.attempts` sidecar are included too and are processed FIRST (the worker's pass uses the same order), and a queue-cap refusal gives its attempt back (`undo_attempt`), so a session that faults during recovery itself, whose failure leaves a newer session behind each time, still reaches `MAX_RECOVERY_ATTEMPTS` and is abandoned. **Newest first**: the latest crash is the one most likely to belong to the build now running, and a crash loop makes only recent evidence reachable. What the pass leaves is recovered by the worker's unbounded pass, so nothing is lost; recovered reports may therefore upload newest first, which is harmless because the server deduplicates by signature. -- **Crash-loop escalation.** A fast loop never lets the uploader finish, so the server's `12004` verdict and the local blacklist are never reached. In the pass, a recovered crash whose signature set equals one already queued is dropped (the first copy is kept; once it has been delivered a recurrence is queued again), and the launch pass stops queuing recovered crashes when the queue holds 64 reports or 64 MiB, counting every queued report (`queue::admit_recovered`). A unique crash the cap refuses is NOT discarded: its evidence stays and it is left to the worker's pass, which does not cap (the cap bounds launch-path work, not total retention). Not done: a repeat counter on the kept report. The worker pass does not dedup or cap. +- **Crash-loop escalation.** A fast loop never lets the uploader finish, so the server's too-many-similar verdict and the local blacklist are never reached. In the pass, a recovered crash whose signature set equals one already queued is dropped (the first copy is kept; once it has been delivered a recurrence is queued again), and the launch pass stops queuing recovered crashes when the queue holds 64 reports or 64 MiB, counting every queued report (`queue::admit_recovered`). A unique crash the cap refuses is NOT discarded: its evidence stays and it is left to the worker's pass, which does not cap (the cap bounds launch-path work, not total retention). Not done: a repeat counter on the kept report. The worker pass does not dedup or cap. - **Scope.** A recovered crash carries no scope (email, labels, attributes) from the new process, on either path: the scope describes the new run, and merging it made the two paths disagree. Persisting scope per session is possible, not done. - **Not covered.** Sessions of the legacy data directory are recovered by the worker only. A fork child (prefork worker boot loop) recovers on its worker, not on a launch path, so a prefork worker that dies before its worker thread finishes recovery leaves the evidence for a later launch (nothing is lost, it is reported late); the master's pass covers only the master's own launch. In memory mode with a recovery directory (#9), a crash that cannot be queued on disk is delivered from memory and its evidence stays, so a later launch reports it again (the backend deduplicates). - **Measured** (`launch_recovery.rs::measure_launch_latency`, Apple silicon, synthetic log-only windows at the default 8 MiB / 60 s caps, a pass that waits for completion): empty directory about 0.1 to 0.3 ms; one crashed session about 7 ms release, 21 ms debug; eight identical sessions about 17 ms release, 43 ms debug. The eight-session figure undercounts assembly, since seven of them are dropped as duplicates before being assembled; scrubbing (#11) will add per-session cost. `Recorder::launch` itself is about 0.5 to 0.8 ms of that. diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index 06cdf86..7e707c7 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -1756,7 +1756,7 @@ fn process_report( return Ok(()); } - // Suppress reports the server blacklisted (12004 TooManySimilar), from the + // Suppress reports the server blacklisted (the too-many-similar verdict), from the // in-memory set (the only one with no usable disk) or the on-disk one. if mem_blacklisted(shared, &meta.signatures) || data_dir.is_some_and(|dir| queue::any_blacklisted(dir, &meta.signatures)) diff --git a/crates/bugsee/src/hostinfo.rs b/crates/bugsee/src/hostinfo.rs index 5234aa6..25ceaec 100644 --- a/crates/bugsee/src/hostinfo.rs +++ b/crates/bugsee/src/hostinfo.rs @@ -151,7 +151,7 @@ mod imp { /// macOS: `hw.model` is the model identifier ("MacBookPro18,3"). Every other /// Apple OS: `hw.model` is the *board* id ("D63AP"), and the product - /// identifier ("iPhone14,2") is `hw.machine`, the same value iOS SDKs read + /// identifier ("iPhone14,2") is `hw.machine`, the same value iOS apps read /// from `utsname.machine`. In the simulator `hw.machine` is the host's /// architecture, so the simulated device's identifier is taken from /// `SIMULATOR_MODEL_IDENTIFIER` first. diff --git a/docs/design/mobile-native-capture.md b/docs/design/mobile-native-capture.md index 57aa224..d94606a 100644 --- a/docs/design/mobile-native-capture.md +++ b/docs/design/mobile-native-capture.md @@ -4,9 +4,9 @@ **Base:** `main` @ `a1564ae`. The `CrashBackend` contract is read from `origin/refactor/crash-backend` (crate `bugsee-native-api`, commit `6189ed6`, not merged). The out-of-process design is read from `origin/docs/issue-17-oop-monitor-design` (`docs/design/out-of-process-monitor.md`, not merged). **Provenance legend.** -- **[V]** verified in this repo, in the Bugsee backend worker source, or in the local sources of Bugsee's own iOS and Android SDKs (described generically here, per #4's rule that private repositories are not named, linked or quoted). +- **[V]** verified in this repo, or against the behaviour of the existing wire format and the backend (described generically here, per #4's rule that private repositories are not named, linked or quoted). - **[U]** from memory, from the issue text, or from general platform knowledge; not verified. Confirm before relying on it. -- No third-party source (PLCrashReporter, Crashpad, ART, bionic, Apple headers) was available locally. Everything about them is **[U]** unless it was observed through Bugsee's own SDK code. +- No third-party source (PLCrashReporter, Crashpad, ART, bionic, Apple headers) was available locally. Everything about them is **[U]**. --- @@ -44,14 +44,14 @@ ### 2.3 Android-specific observations from reading the handler (not from running it) -1. **`sigaltstack` is not left alone.** `ensure_alt_stack` keeps the installing thread's alternate stack only if it is at least 64 KiB; otherwise it maps a 128 KiB one and installs it (restored on uninstall) **[V]**. Bugsee's Android SDK states the opposite rule as hard-won: never replace ART's `sigaltstack` **[V]**. ART's per-thread stacks are small **[U]**, so on a Java thread this would likely replace ART's. It is also per-thread: only the installing thread is covered. +1. **`sigaltstack` is not left alone.** `ensure_alt_stack` keeps the installing thread's alternate stack only if it is at least 64 KiB; otherwise it maps a 128 KiB one and installs it (restored on uninstall) **[V]**. Established Android crash reporters follow the opposite rule: never replace ART's `sigaltstack` **[U]**. ART's per-thread stacks are small **[U]**, so on a Java thread this would likely replace ART's. It is also per-thread: only the installing thread is covered. 2. **Registration is plain `libc::sigaction`.** Whether that call reaches ART's sigchain, so ART orders and chains us, depends on symbol interposition in the app's namespace **[U]**. The chaining logic fits a "previous handler is a real handler" model, but only a device run shows that ART's own `SIGSEGV` use (implicit null checks, stack-overflow probes) still works and that we do not report faults ART recovers. -3. **The module map is an install-time snapshot.** Android apps routinely call `System.loadLibrary` after SDK start; frames in those libraries are dropped by design **[V]**. Bugsee's Android SDK keeps its own module table in the handler for the equivalent problem **[V]**. -4. **Frames are native only.** `_Unwind_Backtrace` cannot see Java frames and may not unwind through ART-generated code **[U]**. Bugsee's Android SDK captures Java stacks inside its handler via JNI **[V]**; the Rust SDK has no JNI surface. -5. **Only one ABI is built in CI** (`aarch64`). `armeabi-v7a`, `x86` and `x86_64` have `cfg` branches in the handler (pc/sp extraction) that CI never compiles **[V]**. Bugsee's Android SDK ships all four **[V]**. +3. **The module map is an install-time snapshot.** Android apps routinely call `System.loadLibrary` after SDK start; frames in those libraries are dropped by design **[V]**. Mature Android engines keep their own module table in the handler for the equivalent problem **[U]**. +4. **Frames are native only.** `_Unwind_Backtrace` cannot see Java frames and may not unwind through ART-generated code **[U]**. A JVM-aware reporter can capture Java stacks inside its handler via JNI **[U]**; the Rust SDK has no JNI surface. +5. **Only one ABI is built in CI** (`aarch64`). `armeabi-v7a`, `x86` and `x86_64` have `cfg` branches in the handler (pc/sp extraction) that CI never compiles **[V]**. Android SDKs commonly ship all four **[U]**. 6. **Data directory: the Rust API can set it, the C ABI cannot, and only iOS gets a sandbox-correct default.** `LaunchOptions::data_dir` exists **[V]** (`options.rs`), so a Rust-API host (scenario A) can point it at the app sandbox. `bugsee-ffi` exposes only `bugsee_launch(token)` and `bugsee_launch_with_endpoint(token, endpoint)` **[V]**, so an FFI host cannot pass one. On `main` such a host gets `std::env::temp_dir()/bugsee/`, and an unwritable directory makes `launch_with` return an error (`create_dir_all(&data_dir)?`) **[V]**. Open PR #24 (#9), read from `origin/fix/issue-9` (`resolve_data_dir`, `platform_data_base`, `DataDirSource`, `StorageStatus::may_be_ephemeral` in `options.rs`/`api.rs`), changes this **[V]**. Order: explicit `data_dir` (final; if unusable, memory mode), then `BUGSEE_DATA_DIR` (also final), then a platform default, then `DataDirSource::TempFallback` = `/bugsee-/` (private, `0700`), each probed for writability; memory mode only when **all** candidates fail. The handler is installed when a directory was found (`options.native_crash_capture && persistent`), so `TempFallback` arms it. The platform default is `LOCALAPPDATA` on Windows, `$HOME/Library/Application Support` on Apple targets, and `$XDG_DATA_HOME` or `$HOME/.local/share` on every other Unix. **iOS** therefore gets the app container. **Android** app processes normally have neither `XDG_DATA_HOME` nor a useful `HOME` **[U]**, so a host that passes nothing lands on `TempFallback` under `std::env::temp_dir()`. What that directory is on Android is **unverified [U]**: Rust reads `$TMPDIR`, and whether the app's cache directory is exported through it (so the fallback is writable app storage) or it falls to `/data/local/tmp` (unwritable, so memory mode) must be checked on a device in Phase 1. `may_be_ephemeral()` reports `TempFallback`, because a cache directory can be purged by the OS or by the user, and a marker lost that way means one crash goes unreported, not that the SDK fails. 7. **The C ABI already arms the Android handler.** `bugsee_launch` builds `LaunchOptions::new(token)` and never clears `native_crash_capture`, whose default is `true` **[V]** (`options.rs`, `api.rs`). Android is not a stub target (`mod imp` includes `target_os = "android"`), so a successful C ABI launch on Android installs the unproven handler today. Making the C ABI default off would be a behaviour change for existing FFI hosts. -8. **The recovery classifier and the backend route.** `recovery.rs::build_report` computes `has_native = minidump.is_some() || crash_info.is_some()` **[V]**; with `has_native` and anything other than a fresh panic snapshot plus `signal=SIGABRT` it builds the native variant. So a `crash.minidump` is not an "extra file" attached to whatever report would have been built; its presence *selects* the native variant (see Phases 3 and 4). **The `crash.info` grammar recovery parses [V]** (`parse_crash_info`, `is_fresh`, `CORRELATION_WINDOW_MS`): one `key=value` pair per line, split at the first `=`, keys trimmed, unknown keys and lines without `=` ignored. `signal=` is **numeric** (`6`, not `SIGABRT`; anything unparseable becomes 0), `code=`, `address=` (or `addr=`; default `0x0`), `time=` (`CLOCK_REALTIME`, as the in-process handler writes it; unparseable becomes 0) and repeated `frame=` lines (optional `0x`). Correlation into one managed event requires all of: a fresh `panic.info`, `signal=6`, `time` **nonzero**, the panic snapshot's own timestamp nonzero, and `|time - panic.timestamp| <= 10_000` ms. A dump plus `signal=6` with `time` missing, zero or outside that window still takes `build_native`; with no `crash.info` at all it reports `signal.number` 0. The backend routes `sdk.type == 'rust'` reports through `crash/rust.py` and accepts `minidumpFile` **[V]** (worker `jobs/bundle.py`). Whether `rust.py` behaves for `platform_type` `android` or `ios` is untested here **[U]**: its docstring says "linux / windows / macos". +8. **The recovery classifier and the backend route.** `recovery.rs::build_report` computes `has_native = minidump.is_some() || crash_info.is_some()` **[V]**; with `has_native` and anything other than a fresh panic snapshot plus `signal=SIGABRT` it builds the native variant. So a `crash.minidump` is not an "extra file" attached to whatever report would have been built; its presence *selects* the native variant (see Phases 3 and 4). **The `crash.info` grammar recovery parses [V]** (`parse_crash_info`, `is_fresh`, `CORRELATION_WINDOW_MS`): one `key=value` pair per line, split at the first `=`, keys trimmed, unknown keys and lines without `=` ignored. `signal=` is **numeric** (`6`, not `SIGABRT`; anything unparseable becomes 0), `code=`, `address=` (or `addr=`; default `0x0`), `time=` (`CLOCK_REALTIME`, as the in-process handler writes it; unparseable becomes 0) and repeated `frame=` lines (optional `0x`). Correlation into one managed event requires all of: a fresh `panic.info`, `signal=6`, `time` **nonzero**, the panic snapshot's own timestamp nonzero, and `|time - panic.timestamp| <= 10_000` ms. A dump plus `signal=6` with `time` missing, zero or outside that window still takes `build_native`; with no `crash.info` at all it reports `signal.number` 0. The backend routes Rust-SDK reports through its Rust report path and accepts a minidump file **[V]**. Whether that path behaves for Android or iOS reports is untested here **[U]**; it was written for Linux, Windows and macOS. ### 2.4 The `CrashBackend` abstraction (PR #23, unmerged) @@ -70,23 +70,23 @@ What mobile needs that the contract does not yet carry: ### 3.1 Apple mobile (iOS, iPadOS, tvOS, visionOS, watchOS) - **No out-of-process monitor.** Apple mobile apps cannot `fork`/`exec` a helper **[U]** (the OOP design doc says the same about stores). Capture is in-process and must be async-signal-safe, or run on a dedicated Mach exception thread. -- **Mach exceptions versus BSD signals.** A Mach handler on its own thread is the only reliable way to handle stack overflow (the faulting thread has no stack for a signal frame; `sigaltstack` is unreliable on iOS **[U]**) and sees `EXC_BAD_ACCESS` before it becomes `SIGSEGV`; Swift traps arrive as `EXC_BREAKPOINT`/`SIGTRAP`. A debugger attached to the process takes the exception ports first, so crashes under Xcode never reach the reporter **[U]**. Bugsee's iOS SDK uses the **BSD signal handler by default, with Mach as an opt-in option**, and forces BSD on tvOS because `task_set_exception_ports` is prohibited there **[V]** (the SDK's own comment; the header fact itself is **[U]**). +- **Mach exceptions versus BSD signals.** A Mach handler on its own thread is the only reliable way to handle stack overflow (the faulting thread has no stack for a signal frame; `sigaltstack` is unreliable on iOS **[U]**) and sees `EXC_BAD_ACCESS` before it becomes `SIGSEGV`; Swift traps arrive as `EXC_BREAKPOINT`/`SIGTRAP`. A debugger attached to the process takes the exception ports first, so crashes under Xcode never reach the reporter **[U]**. A common configuration is the **BSD signal handler by default, with Mach as an opt-in option**, and BSD only on tvOS because `task_set_exception_ports` is prohibited there **[U]**. - **Uncatchable in-process:** `EXC_CRASH` from the kernel, jetsam/OOM terminations, watchdog kills and `SIGKILL`. These can only be inferred next launch (previous-run state, MetricKit **[U]**). -- **`NSException`** needs an explicit `NSSetUncaughtExceptionHandler` hook, and C++ exceptions need a `__cxa_throw` interposer **[V]** (the iOS SDK does both outside PLCrashReporter). A Rust-only process has neither; a Rust library inside an ObjC/Swift app will see them. +- **`NSException`** needs an explicit `NSSetUncaughtExceptionHandler` hook, and C++ exceptions need a `__cxa_throw` interposer **[U]** (iOS crash SDKs typically do both outside PLCrashReporter). A Rust-only process has neither; a Rust library inside an ObjC/Swift app will see them. - **App Store.** Private API use is the rejection risk. `mach_vm_*` functions are marked unavailable in the iOS SDK headers; calling them through a Rust `extern` bypasses the compile-time check but may still be flagged **[U]**. The in-repo code has **two** `mach_vm_read_overwrite` call sites **[V]**, both `cfg(target_vendor = "apple")` and so both compiled if `mod imp` is widened: `read_task_mem` (the frame-pointer walk) and `posix::unreadable` (used on every `SIGSEGV`/`SIGBUS` to decide whether the host recovered). The iOS-safe equivalent is `vm_read_overwrite` **[U]**. Replacing only the walker would still link the symbol. #4 also says to prefer public APIs. -- **Targets named by `target_os = "ios"`.** `cfg(target_os = "ios")` also matches Mac Catalyst (`aarch64-apple-ios-macabi`, `x86_64-apple-ios-macabi`) **[U]**, which Bugsee's iOS SDK compiles its crash layer out of **[V]** and which D12 excludes. Any widening must exclude `target_abi = "macabi"`, unless D12 is changed to include Catalyst. -- **Simulator.** Simulator processes are host-OS processes, so signals work as on macOS **[U]**. Bugsee's iOS SDK compiles its crash layer out on the Simulator and Mac Catalyst; the only recorded reason is a comment that PLCrashReporter is "unavailable on the simulator" **[V]**. Whether our own handler works there is an open empirical question (D6). +- **Targets named by `target_os = "ios"`.** `cfg(target_os = "ios")` also matches Mac Catalyst (`aarch64-apple-ios-macabi`, `x86_64-apple-ios-macabi`) **[U]**, which native crash layers commonly compile out **[U]** and which D12 excludes. Any widening must exclude `target_abi = "macabi"`, unless D12 is changed to include Catalyst. +- **Simulator.** Simulator processes are host-OS processes, so signals work as on macOS **[U]**. Existing iOS crash SDKs commonly compile their crash layer out on the Simulator and Mac Catalyst, on the grounds that PLCrashReporter is "unavailable on the simulator" **[U]**. Whether our own handler works there is an open empirical question (D6). - **Rust targets.** `aarch64-apple-ios`, `aarch64-apple-ios-sim` and `x86_64-apple-ios` are tier 2 **[U]**. #4 reports the tvOS/visionOS/watchOS targets as tier 2 and notes an earlier "tier 3" claim needs re-checking; they are out of scope for the first pass. -- **Coexistence.** Two reporters in one process is the normal case. Signal handlers chain; the single `NSUncaughtExceptionHandler` slot does not. Bugsee's iOS SDK refuses to run if its handler could not be installed over another reporter's **[V]**. +- **Coexistence.** Two reporters in one process is the normal case. Signal handlers chain; the single `NSUncaughtExceptionHandler` slot does not. A reasonable policy for an iOS reporter is to refuse to run if its handler could not be installed over another reporter's **[U]**. ### 3.2 Android - **ART owns `SIGSEGV`.** ART uses faults for implicit null checks, stack-overflow probing and GC barriers **[U]**, through its sigchain library, which runs ART's special handlers before user handlers and is meant to be the registration path for everyone **[U]**. Our chaining logic handles a host that recovers (section 2.2); the unknown is how our `sigaction` call interposes. - **bionic and `debuggerd`.** bionic installs `debuggerd`'s handler for fatal signals in every process **[U]**; it dumps a tombstone, resets to default and returns, so the fault re-executes and kills the process. That matches our "host declined, restored default" branch **[U]**. Tombstones and `ApplicationExitInfo` (API 30+) are the system's record and a safety net for ANR, OOM kills and native crashes we could not catch. Reading `ApplicationExitInfo` needs Java/JNI **[U]** and so a Kotlin/AAR shim. -- **`sigaltstack`.** Never replace ART's **[V]** (Bugsee's Android SDK rule): use the existing one with `SA_ONSTACK`, and do not install one on a thread we do not own. -- **Handler launch constraints (Crashpad), per Bugsee's Android SDK [V].** The handler must be started after extracting it from the APK; API 29+ starts it through the system linker, API 21-28 use a Java `app_process` path; an in-process interim handler covers the extraction window; an early-startup crash must finish export and upload on the launch path. These are the costs of "Crashpad on Android". -- **ABIs.** `arm64-v8a`, `armeabi-v7a`, `x86`, `x86_64` are what Bugsee's Android SDK builds **[V]**. The Rust equivalents (`aarch64`, `armv7`, `i686`, `x86_64` `*-linux-android`) are tier 2 **[U]**. 32-bit ABIs matter for 64-bit atomics (the Android SDK's comments call out torn reads **[V]**); the Rust handler uses `AtomicU64` generation counters, which must be tested on `armv7` **[U]**. -- **16 KB pages.** Bugsee's Android SDK links with `-z max-page-size=16384` **[V]**. #4 states the Play requirement as "from 2027-02-01" for API 35+ updates; I did not verify the date **[U]**. A Rust cdylib needs the same linker flags (or a recent NDK default **[U]**). The handler already reads the page size at runtime. +- **`sigaltstack`.** Never replace ART's **[U]**: use the existing one with `SA_ONSTACK`, and do not install one on a thread we do not own. +- **Handler launch constraints (Crashpad) [U].** The handler must be started after extracting it from the APK; API 29+ starts it through the system linker, API 21-28 use a Java `app_process` path; an in-process interim handler covers the extraction window; an early-startup crash must finish export and upload on the launch path. These are the costs of "Crashpad on Android". +- **ABIs.** `arm64-v8a`, `armeabi-v7a`, `x86`, `x86_64` are the usual Android ABIs **[U]**. The Rust equivalents (`aarch64`, `armv7`, `i686`, `x86_64` `*-linux-android`) are tier 2 **[U]**. 32-bit ABIs matter for 64-bit atomics (torn reads are the known hazard **[U]**); the Rust handler uses `AtomicU64` generation counters, which must be tested on `armv7` **[U]**. +- **16 KB pages.** Android native libraries need `-z max-page-size=16384` **[U]**. #4 states the Play requirement as "from 2027-02-01" for API 35+ updates; I did not verify the date **[U]**. A Rust cdylib needs the same linker flags (or a recent NDK default **[U]**). The handler already reads the page size at runtime. - **Where Rust runs.** Either a pure Rust app (NativeActivity / `android-activity`, Tauri, Bevy, Slint) where Rust owns the process, or a Rust library loaded by a Kotlin/Java host through JNI. They have different crash-reporter owners (section 4). ### 3.3 Engines compared @@ -95,30 +95,30 @@ What mobile needs that the contract does not yet carry: |---|---|---|---| | Platforms | Linux, Android, macOS; iOS by `cfg` widening | iOS, macOS, tvOS **[U]** | Android, desktop; iOS in-process **[U]** | | Process model | In-process | In-process | Out-of-process helper on Android **[V]** | -| Records | Marker, crashing-thread PCs, module map | All threads, registers, images with UUIDs, `NSException`; Mach or BSD **[U]** | Minidump: all threads, registers, modules, stacks **[V]**; the worker already stackwalks these | +| Records | Marker, crashing-thread PCs, module map | All threads, registers, images with UUIDs, `NSException`; Mach or BSD **[U]** | Minidump: all threads, registers, modules, stacks **[V]**; the backend already stackwalks these | | Stack overflow | Header only, frames best effort | Mach mode only **[U]** | Yes, separate process | | Heap corruption | Frames may be lost | Same exposure as in-process | Yes | | Packaging | None | C/ObjC/asm plus protobuf-c, buildable with the `cc` crate per #4 **[U]** | C++ tree plus a handler and a launch path **[V]** | | Licence surface | Ours | MIT plus protobuf-c **[U]** | Apache-2.0 plus bundled third-party code **[U]** | -| Maturity | New; `a1564ae` is its first coexistence pass | High **[U]**; Bugsee's iOS SDK already depends on a fork **[V]** | High **[U]**; Bugsee's Android SDK vendors it **[V]** | -| Unwinding | Frame pointers (Apple), `_Unwind_Backtrace` (Android) | Compact unwind / DWARF plus frame pointers **[U]** | `libunwindstack` in Bugsee's Android SDK **[V]** | +| Maturity | New; `a1564ae` is its first coexistence pass | High **[U]** | High **[U]** | +| Unwinding | Frame pointers (Apple), `_Unwind_Backtrace` (Android) | Compact unwind / DWARF plus frame pointers **[U]** | `libunwindstack` **[U]** | -Known weakness of our Apple walker: it starts from `(pc, fp)`, so a fault in a **frameless leaf** function skips the immediate caller (held in the link register). Bugsee's iOS SDK keeps a test scenario for exactly this **[V]**. +Known weakness of our Apple walker: it starts from `(pc, fp)`, so a fault in a **frameless leaf** function skips the immediate caller (held in the link register). The test matrix should include a scenario for exactly this. ### 3.4 Symbolication identity (what the backend expects) -All **[V]** from `crates/bugsee-native`, `recovery.rs` and the worker (`symbolfiles/*`, `crash/rust.py`, `crash/helpers/android_ndk.py`) unless marked. +All **[V]** from `crates/bugsee-native`, `recovery.rs` and the observed behaviour of the symbol upload and symbolication services, unless marked. -- **Android: GNU build-id.** Upload: `symbolfiles/elf.py` stores `code_id` per ELF object and **skips any object without a build-id**, with a warning that the library must be linked with `-Wl,--build-id`. Crash: `bugsee-native` reads `NT_GNU_BUILD_ID` from `PT_NOTE` and writes lowercase hex; the worker normalises it with the shared canonical function and rejects non-hex or all-zero ids. A `.so` without a build-id can never resolve. Whether packaging steps keep the note, and that the *unstripped* `.so` from `target//release/` is what must be uploaded, are **[U]**. -- **Apple: Mach-O `LC_UUID`.** Upload: dSYM `debug_id`, canonicalised by the appserver and worker (lowercase, no dashes). Crash: `bugsee-native::image_info` emits `LC_UUID` as lowercase hex; `tests/code_id_matches_symbolic.rs` cross-checks it against `symbolic`, the library the CLI and worker use. When Rust is a static library linked into the app, its frames are in the app's Mach-O, so the app's dSYM provides symbols. That is simpler than Android. +- **Android: GNU build-id.** Upload: the symbol service stores `code_id` per ELF object and **skips any object without a build-id**, with a warning that the library must be linked with `-Wl,--build-id`. Crash: `bugsee-native` reads `NT_GNU_BUILD_ID` from `PT_NOTE` and writes lowercase hex; the backend normalises it to a canonical form and rejects non-hex or all-zero ids. A `.so` without a build-id can never resolve. Whether packaging steps keep the note, and that the *unstripped* `.so` from `target//release/` is what must be uploaded, are **[U]**. +- **Apple: Mach-O `LC_UUID`.** Upload: dSYM `debug_id`, canonicalised by the backend (lowercase, no dashes). Crash: `bugsee-native::image_info` emits `LC_UUID` as lowercase hex; `tests/code_id_matches_symbolic.rs` cross-checks it against `symbolic`, the library the CLI and the symbolication service use. When Rust is a static library linked into the app, its frames are in the app's Mach-O, so the app's dSYM provides symbols. That is simpler than Android. - **`arch` and platform.** `aarch64` is reported as `arm64` to match `symbolic`; `source_platform()` is `std::env::consts::OS`, so `android` and `ios` pass through. -- **Frame model.** The thin variant sends `reladdr` per frame and `code_id` per module; the worker resolves them with SymCache primitives shared with the Apple path and appends `s.`-prefixed signatures to the client signature. Android NDK reports use a different path (minidump plus Breakpad symbols from ELF). Wiring the minidump route for `sdk.type == 'rust'` on `android` is untested **[U]**. -- **System modules.** PROGRESS.md notes Windows reports stay `missing_sym` because of unsymbolicatable system DLLs and that Android "almost certainly has the same shape" **[V]**. Mobile reports will carry `libc.so`, `libart.so` and system frameworks; this must be fixed on the worker for mobile issues to reach a clean symbolicated state. +- **Frame model.** The thin variant sends `reladdr` per frame and `code_id` per module; the ingestion pipeline resolves them against the uploaded symbols, as it does for Apple reports, and appends `s.`-prefixed signatures to the client signature. Android NDK reports use a different path (minidump plus Breakpad symbols from ELF). Wiring the minidump route for Rust-SDK reports on Android is untested **[U]**. +- **System modules.** PROGRESS.md notes Windows reports stay `missing_sym` because of unsymbolicatable system DLLs and that Android "almost certainly has the same shape" **[V]**. Mobile reports will carry `libc.so`, `libart.so` and system frameworks; this must be fixed in the backend for mobile issues to reach a clean symbolicated state. ### 3.5 Host coexistence - The existing rule (install over, chain to previous, delete the marker when the host recovers) already covers hosts that fault on purpose **[V]**. -- It does not answer **duplication**. With a different vendor's reporter present, a real crash reported by both is expected and harmless. With Bugsee's own native SDK in the same process it produces two issues for one crash. The Rust SDK cannot reliably detect the sibling SDK (symbol probing is fragile **[U]**), so the policy must be explicit configuration with a safe default (D4). +- It does not answer **duplication**. With a different vendor's reporter present, a real crash reported by both is expected and harmless. With another Bugsee native SDK in the same process it produces two issues for one crash. The Rust SDK cannot reliably detect the other SDK (symbol probing is fragile **[U]**), so the policy must be explicit configuration with a safe default (D4). - `NSUncaughtExceptionHandler` and `std::panic::set_hook` are single-slot. The panic hook is ours already; the `NSException` hook is not, and a PLCrashReporter backend would claim it (D5). --- @@ -140,16 +140,16 @@ Only scenario C demands engine parity (all threads, registers, ANR/OOM, `NSExcep ### 5.1 Apple mobile -**End state:** PLCrashReporter behind `CrashBackend` (`name() == "plcrashreporter"`), BSD signal handler, Mach exceptions off by default, tvOS BSD-only, `NSException` and C++ hooks as a later increment. This follows the agreed direction in #4 and matches the shipping iOS SDK's defaults **[V]**, which is also the evidence that it works on real devices. +**End state:** PLCrashReporter behind `CrashBackend` (`name() == "plcrashreporter"`), BSD signal handler, Mach exceptions off by default, tvOS BSD-only, `NSException` and C++ hooks as a later increment. This follows the agreed direction in #4 and matches common iOS crash-SDK defaults **[U]**. **Not first.** Do the cheap in-process widening first, as a baseline and fallback: 1. Widen `mod imp`'s `cfg` to include `target_os = "ios"` (device and simulator, but **excluding `target_abi = "macabi"`**, i.e. Mac Catalyst), replace `mach_vm_read_overwrite` with `vm_read_overwrite` at **both** call sites (`read_task_mem` and `posix::unreadable`) **[U]**, confirm with `nm` that no `mach_vm_*` symbol remains in an iOS build, and run the crash matrix on the iOS simulator. This gives end-to-end Apple mobile capture on the existing marker/module/recovery path with **no new dependencies**, and answers open decision 4 empirically. 2. Label it tier 2 / experimental: crashing thread only, frame-pointer unwinding (leaf-frame gap), no Mach, no `NSException`. Do not claim parity. 3. Then build the PLCrashReporter backend as its own `-sys` plus wrapper crate pair, selected by `default_backend()` on device targets, falling back to the in-process handler where PLCrashReporter is unavailable (simulator, if confirmed). -Why not skip step 1: it is a `cfg` change plus a test job, it de-risks the simulator and CI plumbing PLCrashReporter needs anyway, and it is the fallback. Why not stop there: leaf-frame loss, no all-thread stacks, no `NSException`, and no parity with what Bugsee's iOS SDK sends. +Why not skip step 1: it is a `cfg` change plus a test job, it de-risks the simulator and CI plumbing PLCrashReporter needs anyway, and it is the fallback. Why not stop there: leaf-frame loss, no all-thread stacks, no `NSException`, and no parity with what Bugsee's other Apple SDKs send. -**Report shape (D2).** A PLCrashReporter crash can become (a) our thin native variant plus additive threads and registers, or (b) the iOS SDK's crash document, so the existing Apple worker path does everything. (b) gives parity immediately but needs a worker routing change (the worker routes on `sdk.type == 'rust'`) and a large converter **[V]**. Recommend (a), because the Rust path already symbolicates `modules` plus `reladdr` and it keeps one report shape across desktop and mobile; revisit (b) if the worker owners prefer one Apple path. +**Report shape (D2).** A PLCrashReporter crash can become (a) our thin native variant plus additive threads and registers, or (b) the backend's existing Apple report variant, so the existing Apple path does everything. (b) gives parity immediately but needs a backend routing change and a large converter **[U]**. Recommend (a), because the Rust path already symbolicates `modules` plus `reladdr` and it keeps one report shape across desktop and mobile; revisit (b) if the backend owners prefer one Apple path. ### 5.2 Android @@ -160,7 +160,7 @@ Why not skip step 1: it is a `cfg` change plus a test job, it de-risks the simul - Build all four ABIs, link for 16 KB pages, and verify the build-id note survives packaging. - Data directory: depend on PR #24 and the Phase 0 work (section 2.3, item 6). -**Phase B (conditional):** Crashpad behind `CrashBackend` (`name() == "crashpad"`), only if Phase A shows a gap that matters (all-thread stacks, heap corruption, stack overflow, Java frames) **and** scenario C is a goal. The costs are the ones Bugsee's Android SDK already paid **[V]**. A prerequisite decision is whether to **share one Crashpad source tree** with the Android SDK instead of vendoring a second copy (D7). The OOP design doc recommends *against* Crashpad for desktop v1; if desktop picks `minidumper`, Android would be the only Crashpad user and "one engine everywhere" is not a benefit. +**Phase B (conditional):** Crashpad behind `CrashBackend` (`name() == "crashpad"`), only if Phase A shows a gap that matters (all-thread stacks, heap corruption, stack overflow, Java frames) **and** scenario C is a goal. The costs are the ones any Android Crashpad integration pays **[U]**. A prerequisite decision is whether to **share one Crashpad source tree** with other Bugsee Android components instead of vendoring a second copy (D7). The OOP design doc recommends *against* Crashpad for desktop v1; if desktop picks `minidumper`, Android would be the only Crashpad user and "one engine everywhere" is not a benefit. **Phase C (deferred):** `ApplicationExitInfo` merge and Java-frame capture. Both need Java code, so they belong in an AAR shipped with the Rust SDK, which is a packaging decision (D9). @@ -169,14 +169,14 @@ Why not skip step 1: it is a `cfg` change plus a test job, it de-risks the simul | Item | Why deferred | |---|---| | Hang/watchdog, ANR | Need main-thread liveness plumbing; the Rust SDK does not own the UI thread. | -| OOM/jetsam inference, MetricKit | Next-launch heuristics, a separate feature; Bugsee's iOS SDK keeps them outside PLCrashReporter **[V]**. | +| OOM/jetsam inference, MetricKit | Next-launch heuristics, a separate feature; they live outside PLCrashReporter **[U]**. | | `ApplicationExitInfo` | Needs a Java shim (D9). | | C++ `__cxa_throw` hook, ObjC `NSException` | Only matter for scenario C and ObjC/C++ hosts. | -| Mach exception backend | Opt-in in the shipping SDK **[V]**; stack-overflow reliability is its only strong benefit. | +| Mach exception backend | Opt-in in existing iOS crash SDKs **[U]**; stack-overflow reliability is its only strong benefit. | | tvOS, visionOS, watchOS device work | Target tiers and entitlements unverified; tvOS has no Mach exception ports **[V]**. | | Out-of-process on Apple mobile | Not possible without a helper process **[U]**. | | Out-of-process on Android | Same decision as Crashpad (Phase 4). | -| Instruction-window capture around the PC | An internal Bugsee plan scoped to the existing Android/iOS SDKs **[V]**; needs a safe-read API per platform. Revisit after parity. | +| Instruction-window capture around the PC | Scoped to a later phase **[U]**; needs a safe-read API per platform. Revisit after parity. | --- @@ -200,12 +200,12 @@ Phases are ordered by dependency and risk, not by calendar. Each ends in somethi 2. a Java `NullPointerException`, a Java `StackOverflowError` and an ART-handled implicit null check yield **no** marker and the SDK stays armed; 3. a library loaded after `launch` appears in `crash.modules`, or its frames are explicitly marked unresolved; 4. all four ABIs build in CI and at least `x86_64` and `arm64-v8a` run; - 5. an uploaded unstripped `.so` with a build-id symbolicates a native frame end to end against staging (the same round trip PROGRESS.md records for Windows); + 5. an uploaded unstripped `.so` with a build-id symbolicates a native frame end to end against a staging backend (the same round trip PROGRESS.md records for Windows); 6. a test reads each thread's `sigaltstack` before and after install and finds ART's unchanged. ### Phase 2 β€” Apple in-process on simulator and device - Widen the `cfg`, switch to `vm_read_overwrite`, and run the Apple matrix on `aarch64-apple-ios-sim` (and on a device manually). -- Port the *scenarios* of the iOS SDK's crash catalogue (abort, null, stack smash, ObjC exception, C++ throw, corrupt malloc, undefined/privileged instruction, read-only page write, OOM, deadlock, Swift traps) as an Apple test matrix **[V]**; copy the scenarios, not the files. +- Build an Apple test matrix of crash scenarios (abort, null, stack smash, ObjC exception, C++ throw, corrupt malloc, undefined/privileged instruction, read-only page write, OOM, deadlock, Swift traps) **[U]**. - **Acceptance:** 1. the simulator runs the signal-path matrix in CI; 2. each scenario is recorded as captured, partially captured or not capturable, in a table here or in PROGRESS.md; @@ -220,14 +220,14 @@ Phases are ordered by dependency and risk, not by calendar. Each ends in somethi - **Acceptance:** 0. an aborting-panic test that goes through `recovery::build_report` with this backend's output: a fresh `panic.info` plus a backend-produced `crash.info` (`signal=6`, nonzero `time` within 10 s of the panic timestamp) yields **one managed event** (no `signal` object, `exception_type` not `native`); the same input with a `time` outside the window yields the native variant, so the test pins both sides; 1. the Phase 2 matrix passes at least as well as the in-process handler on every scenario and strictly better on leaf-frame, all-thread and `NSException` cases; - 2. an app linking this crate and Bugsee's iOS SDK links, launches and does not duplicate the report under the chosen policy; + 2. an app linking this crate and another Bugsee native SDK links, launches and does not duplicate the report under the chosen policy; 3. binary size and build-time deltas are recorded; 4. a licence and advisory check (`cargo deny`) passes with the vendored notices; 5. the Rust staticlib's frames symbolicate from the app's dSYM in a staging round trip. ### Phase 4 β€” Crashpad go/no-go and, if go, backend (Android) - Decide from Phase 1 data and D1/D7. If go: vendor or share the source (D7), implement `CrashBackend`, carry host-supplied handler parameters through `BackendConfig` (D8), and write the dump to `crash.minidump`. A dump alone selects the native variant in `recovery.rs` (section 2.3, item 8), so the backend must also write a `crash.info` in the recovery grammar (section 2.3, item 8: numeric `signal=6`, nonzero `time=` within 10 s of `panic.info`), or recovery must change. Acceptance includes the same `build_report` aborting-panic test as Phase 3, run against a dump-bearing session (dump plus `crash.info` plus fresh `panic.info` yields one managed event). -- **Acceptance:** all Phase 1 criteria still hold on the Crashpad backend; stack-overflow and heap-corruption scenarios produce a dump with all threads; the API 21-28 and API 29+ launch paths each have an emulator or device run; an early-startup crash reaches the upload path before host code can crash again (open decision 6, measured); the worker symbolicates the dump for `sdk.type == 'rust'` on `android`. +- **Acceptance:** all Phase 1 criteria still hold on the Crashpad backend; stack-overflow and heap-corruption scenarios produce a dump with all threads; the API 21-28 and API 29+ launch paths each have an emulator or device run; an early-startup crash reaches the upload path before host code can crash again (open decision 6, measured); the backend symbolicates the dump for a Rust-SDK report on Android. ### Phase 5 β€” Optional `ApplicationExitInfo` safety net (AAR), hang/ANR, OOM inference, `NSException` and C++ hooks, Mach exceptions, tvOS/visionOS/watchOS, instruction windows. Each is gated by a measured customer need. @@ -242,17 +242,17 @@ Phases are ordered by dependency and risk, not by calendar. Each ends in somethi | On Android the no-directory fallback is `TempFallback` (possibly an OS-purgeable cache directory) or, if unwritable, memory mode. | Medium / medium | Phase 0: C ABI `data_dir` with the host passing `filesDir`; `may_be_ephemeral()` surfaced; a device run records what `temp_dir()` resolves to; no hard error. | | A dump-writing backend turns aborting panics into empty native crashes (`recovery.rs` classifier). | High if unaddressed / medium | Backend writes `crash.info` in the recovery grammar (numeric signal, nonzero ms `time`), or recovery changes; a `build_report` test in Phases 3 and 4. | | The C ABI already arms the unproven Android handler (`native_crash_capture` defaults to `true`). | High / high | Phase 1 gate; an explicit decision on the C ABI default (D4). | -| Duplicate issues with Bugsee's own native SDK or another reporter in the same process. | High in B and C / medium | Explicit mode, off when a host reporter is expected (D4); document. | +| Duplicate issues with another Bugsee native SDK or another reporter in the same process. | High in B and C / medium | Explicit mode, off when a host reporter is expected (D4); document. | | `NSUncaughtExceptionHandler` and signal ordering between PLCrashReporter and another reporter. | Medium / medium | Chain on install; a test with a second reporter present. | | App Store flags a private or prohibited symbol. | Low-medium / high | Phase 2 audit; public APIs only; Mach off. | -| Symbol collision between this crate's vendored PLCrashReporter and Bugsee's iOS SDK. | High if prefixes match / high | D3; a link test in Phase 3. | +| Symbol collision between this crate's vendored PLCrashReporter and another PLCrashReporter copy in the same app. | High if prefixes match / high | D3; a link test in Phase 3. | | Missing build-id (Android) or dSYM mismatch makes every mobile issue unsymbolicatable. | Medium / high | Build-id acceptance check in Phase 1; document uploading the unstripped artifact; reuse the existing `bugsee-cli` flow. | -| System modules keep issues `missing_sym` (known backend item). | High / medium | Track the worker fix as a Phase 1 dependency. | +| System modules keep issues `missing_sym` (known backend limitation). | High / medium | Track the backend fix as a Phase 1 dependency. | | Crashpad on Android pushes packaging (handler library, linker launch, APK extraction) onto every host. | High if chosen / medium | Phase 4 is conditional; ship as an optional feature. | | Vendored engine copies diverge from the SDKs' (bug fixes, notices). | Medium / medium | D7 single source of truth; re-vendor script with provenance. | | 32-bit ABIs untested (`u64` atomics, `cfg` branches). | Medium / medium | Phase 1 builds all four and runs two. | | Rust target tiers shift (tvOS/visionOS/watchOS). | Low / low | Re-check at the start of each phase; not in scope for Phase 2. | -| The worker's Rust route for `platform_type` `android`/`ios` is untested. | Medium / high | One staging round trip per platform in Phases 1 and 2, before any engine work. | +| The backend's Rust report path is untested for Android and iOS. | Medium / high | An end-to-end staging check per platform in Phases 1 and 2, before any engine work. | --- @@ -261,12 +261,12 @@ Phases are ordered by dependency and risk, not by calendar. Each ends in somethi | # | Decision | Options | Recommendation | |---|---|---|---| | **D1** | Who is the mobile consumer? (section 4) | A pure Rust apps; B libraries in native apps; C shared layer under Bugsee's SDKs; D under cross-platform wrappers | Serve A and B now; treat C as a later gate. Without C there is little case for Crashpad, and PLCrashReporter is a nice-to-have. | -| **D2** | Apple report shape | (a) thin variant extended additively with threads and registers; (b) emit the iOS SDK's crash document | (a), with worker owners' sign-off on the additive fields; revisit (b) if they prefer one Apple path. | -| **D3** | Symbol prefix of the vendored PLCrashReporter | Distinct from the iOS SDK's; or the iOS SDK consumes the Rust layer | Distinct, unless D1 resolves to C with the iOS SDK adopting this layer. | +| **D2** | Apple report shape | (a) thin variant extended additively with threads and registers; (b) emit the backend's existing Apple report variant | (a), with backend owners' sign-off on the additive fields; revisit (b) if they prefer one Apple path. | +| **D3** | Symbol prefix of the vendored PLCrashReporter | Distinct from any other bundled copy; or other Bugsee Apple SDKs consume the Rust layer | Distinct, unless D1 resolves to C with those SDKs adopting this layer. | | **D4** | Native-capture mode and its default on mobile | Keep `native_crash_capture` default `true` everywhere; default off for the C ABI (a behaviour change, since it is armed today); auto | Reuse the existing `native_crash_capture` option and expose it in the C ABI. Keep `true` for a Rust-API launch. Whether the C ABI flips to off on mobile is the maintainers' call; I lean to off for iOS/Android FFI hosts until Phase 1, accepting the change. Do not rely on auto-detecting other reporters **[U]**. | | **D5** | Ownership of `NSUncaughtExceptionHandler` / the C++ hook | Install only if free; install and chain; never | Install and chain, in Phase 5, with the contract stated in the `CrashBackend` docs. | -| **D6** | Is the iOS Simulator a supported crash-capture target? | Test our handler and record the result; follow the iOS SDK (disabled) | Test it in Phase 2. If it works, enable it for development and CI; if not, record why. | -| **D7** | Crashpad source of truth | Share one tree with the Android SDK; vendor a second copy; do not adopt | Decide after Phase 1; prefer sharing. Do not vendor a second copy without an owner for both. | +| **D6** | Is the iOS Simulator a supported crash-capture target? | Test our handler and record the result; follow common practice (disabled) | Test it in Phase 2. If it works, enable it for development and CI; if not, record why. | +| **D7** | Crashpad source of truth | Share one tree with other Bugsee Android components; vendor a second copy; do not adopt | Decide after Phase 1; prefer sharing. Do not vendor a second copy without an owner for both. | | **D8** | How backend-private parameters reach `BackendConfig` | Typed `#[non_exhaustive]` fields; a string map | Typed per-backend config structs carried by an enum, so misuse is a compile error. | | **D9** | Ship a Java/Kotlin AAR with the SDK? | Yes (for `ApplicationExitInfo`, Java stacks, `filesDir` discovery); no (native only) | **No for Phase 0**: the host-supplied C ABI path (Phase 0) is sufficient, with `TempFallback` as the no-directory behaviour. Revisit before Phase 5, when `ApplicationExitInfo` and Java stacks need Java code; an AAR could then also remove the need for the host to pass a path. It is a distribution commitment. | | **D10** | Early-crash recovery ("finish export and upload before host code can crash again") | Keep worker-thread recovery; add a synchronous launch-path step | Keep for A and B. Needed only for C and for Crashpad (Phase 4). | @@ -277,6 +277,6 @@ Phases are ordered by dependency and risk, not by calendar. Each ends in somethi ## 9. Verified and unverified -**Verified locally:** `crates/bugsee-native` (`Cargo.toml`, `src/lib.rs` Unix, Apple and Android sections, module snapshot code, stub), `crates/bugsee-core/src/recovery.rs`, `crates/bugsee-core/src/model/environment.rs`, `crates/bugsee/src/{options,api}.rs`, `crates/bugsee-ffi/{src/lib.rs,include/bugsee.h}`, `.github/workflows/ci.yml`, DESIGN.md, PROGRESS.md and issue #4; `bugsee-native-api` and the `CrashBackend` wiring from `origin/refactor/crash-backend`; `docs/design/out-of-process-monitor.md` from `origin/docs/issue-17-oop-monitor-design`; the Bugsee backend worker's ELF and dSYM identity code and the `sdk.type == 'rust'` route; the structure and comments of Bugsee's Android and iOS SDK crash code (read for behaviour; nothing copied). +**Verified locally:** `crates/bugsee-native` (`Cargo.toml`, `src/lib.rs` Unix, Apple and Android sections, module snapshot code, stub), `crates/bugsee-core/src/recovery.rs`, `crates/bugsee-core/src/model/environment.rs`, `crates/bugsee/src/{options,api}.rs`, `crates/bugsee-ffi/{src/lib.rs,include/bugsee.h}`, `.github/workflows/ci.yml`, DESIGN.md, PROGRESS.md and issue #4; `bugsee-native-api` and the `CrashBackend` wiring; `docs/design/out-of-process-monitor.md`; the observed identity handling for ELF build-ids and dSYM UUIDs. -**Not verified (everything marked [U]):** PLCrashReporter, Crashpad and Breakpad behaviour and licences; ART sigchain interposition, `debuggerd`, what `temp_dir()` resolves to on Android; Apple SDK header availability of `mach_vm_*` and `task_set_exception_ports`, App Store review rules; Rust target tiers; the Play Store 16 KB deadline; `ApplicationExitInfo` access from C; whether the worker's Rust path works for `android` and `ios`. +**Not verified (everything marked [U]):** PLCrashReporter, Crashpad and Breakpad behaviour and licences; ART sigchain interposition, `debuggerd`, what `temp_dir()` resolves to on Android; Apple SDK header availability of `mach_vm_*` and `task_set_exception_ports`, App Store review rules; Rust target tiers; the Play Store 16 KB deadline; `ApplicationExitInfo` access from C; whether the backend's Rust report path works for Android and iOS. diff --git a/docs/design/out-of-process-monitor.md b/docs/design/out-of-process-monitor.md index a66c99d..2f7a20e 100644 --- a/docs/design/out-of-process-monitor.md +++ b/docs/design/out-of-process-monitor.md @@ -25,7 +25,7 @@ In-process capture is already shipped on Linux, macOS, Windows and Android (**[V **Goals** - Reliable fatal-crash reports for stack overflow, heap corruption, and (Windows) fast-fail/`abort()` where the in-process path cannot. -- A crash artifact (minidump) that the server already symbolicates **[V]** (DESIGN Β§15: worker `crash/rust.py` handles the thin native variant "plus the generic minidump processor when a dump was harvested"; PDB support exists). +- A crash artifact (minidump) that the backend already symbolicates **[V]** (DESIGN Β§15: the backend handles the thin native variant plus its generic minidump processing when a dump was harvested; PDB support exists). - Fully opt-in; zero behaviour change and zero extra process for users who do not enable it. - Graceful degradation: any failure to start or reach the monitor falls back to the in-process backend and never fails `launch`. - **No change to the wire format, to the meaning of `crash.info`/`crash.minidump` (native crash), or to the report-precedence rules** (native > lone panic > abnormal exit). Next-launch recovery **does** change, in three bounded ways (section 5.4): a new `crash.exit` marker that only enriches the abnormal-exit report, a launch-time bounded wait on a monitor lock plus a `crash.done` completeness check, and (later) preference for a persisted crash-time context. Old SDK versions ignore the new files. @@ -70,7 +70,7 @@ Crates from EmbarkStudios (`crash-handler` **[V]** MIT OR Apache-2.0, `crash-con - Mitigation: wrap behind our own `OopEngine` trait inside `bugsee-monitor`; do not expose its types publicly. ### B. Crashpad as the out-of-process engine -Apache-2.0 **[U]**; handler is a separate executable (`crashpad_handler`) **[U]**; supports Windows, macOS, Linux, Android **[U]**; Windows has a `crashpad_wer.dll` WER module for fast-fail/unusual crashes **[U]**. Per #4, Bugsee's Android SDK already vendors Crashpad, so there is in-house experience and a possible single engine across Android + desktop. +Apache-2.0 **[U]**; handler is a separate executable (`crashpad_handler`) **[U]**; supports Windows, macOS, Linux, Android **[U]**; Windows has a `crashpad_wer.dll` WER module for fast-fail/unusual crashes **[U]**. Per #4, Crashpad is the planned Android engine, so there is a possible single engine across Android + desktop. - Pros: most battle-tested; WER module on Windows; same engine as Android (one place for fixes). - Cons: C++ build (GN/ninja or CMake wrapper) inside a Rust workspace; a mandatory separate executable to locate and ship (re-exec trick unavailable **[U]**); its own database and upload format (we would use it only to produce dumps, fighting its design); large third-party attack and licence surface; `-sys` + wrapper crate pair per #4's structure. - Not rejected: it is the right choice **if** maintainers decide one engine for Android and desktop (decision D2). @@ -292,7 +292,7 @@ Extend the existing subprocess harness (**[V]** `crates/bugsee-native/tests/cras | Recovering host | runtime that recovers SIGSEGV hundreds/sec | no dump per recovered fault (a1564ae regression guard) | | Contract | recovery fed a monitor-produced directory | thin native variant identical in shape to in-process | -Plus: a Windows real-hardware run for PDB round trip like PROGRESS.md did for in-process **[V]**; Docker + Yama matrix for Linux; a size/startup benchmark of re-exec mode; the full bundle fed to the worker ingest once that cross-repo E2E exists (DESIGN Β§13 planned). +Plus: a Windows real-hardware run for PDB round trip like PROGRESS.md did for in-process **[V]**; Docker + Yama matrix for Linux; a size/startup benchmark of re-exec mode; the full bundle fed to the ingestion pipeline once an end-to-end staging check exists (DESIGN Β§13 planned). --- diff --git a/docs/ffi-audit.md b/docs/ffi-audit.md index 317c801..bd49b50 100644 --- a/docs/ffi-audit.md +++ b/docs/ffi-audit.md @@ -156,7 +156,7 @@ Behaviour (verified in `crates/bugsee/src/api.rs`) vs. what the header says bundle), including the server's dedupe/blacklist semantics. A host should not have to reimplement that. - **Direction:** split the shipped `HttpTransport` into (a) the protocol layer - (session, issue, presigned PUT, `12003`/`12004`/`401`/`429` handling β€” stays + (session, issue, presigned PUT, duplicate/too-many-similar/`401`/`429` handling β€” stays in the core) and (b) a one-method HTTP primitive: `send(method, url, headers, body) -> {status, headers, body}`. The FFI exposes only (b) as a callback with a `user_data` pointer; the core keeps ownership of **queueing, ordering, From 2e1b44248d3c7322d8cf8894bcac1ebcd501af1e Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 14 Oct 2026 21:39:09 +0500 Subject: [PATCH 52/52] docs: drop numeric server error codes from notes and comments (#37) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comments and notes only; no behaviour change. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- DESIGN.md | 4 ++-- PROGRESS.md | 6 +++--- crates/bugsee-core/src/queue.rs | 4 ++-- crates/bugsee-core/src/runtime.rs | 2 +- crates/bugsee-core/src/transport.rs | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 985f0d5..bf354cd 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -297,9 +297,9 @@ That last case previously produced **no report whatsoever** β€” the SDK shut dow 2. `POST /v2/issues?app_token=…` β€” body = `app_token + access_token + request.json` β†’ `result.endpoint` (presigned URL). 3. `PUT ` β€” raw ZIP bytes, `Content-Length`, no auth header. Success β†’ delete bundle + cached URL. -**Server semantics:** `12003` SimilarCrashExists β†’ drop silently; `12004` TooManySimilar β†’ blacklist those `signatures` locally + drop; `401`/invalid β†’ re-register session + retry; `429`/`503`/network β†’ retry. +**Server semantics:** the "similar crash exists" verdict β†’ drop silently; the "too many similar" verdict β†’ blacklist those `signatures` locally + drop; `401`/invalid β†’ re-register session + retry; `429`/`503`/network β†’ retry. -**Offline queue + cross-launch retry:** bundles in `/queue/`; sync uploader thread (default; optional async `reqwest`) with backoff `min(30sΒ·2^n, 300s)`, caps export 15 / upload 60. Pending-report list + retry counters persist β†’ next launch re-drives. Crash reports flush synchronously first. A presigned endpoint obtained from `create_issue` is persisted (a `.endpoint` sidecar) when its PUT fails transiently, so a retry **resumes at the PUT** rather than re-creating the issue (which would mint a duplicate or trip server dedup `12003` and drop the bundle un-uploaded). The retry counter (and the abandon-at-cap deletion) advances **only** on backoff-paced attempts; a forced `flush()` re-attempts a queued report but never consumes its durable budget, so an offline flush can't delete a still-deliverable crash bundle. +**Offline queue + cross-launch retry:** bundles in `/queue/`; sync uploader thread (default; optional async `reqwest`) with backoff `min(30sΒ·2^n, 300s)`, caps export 15 / upload 60. Pending-report list + retry counters persist β†’ next launch re-drives. Crash reports flush synchronously first. A presigned endpoint obtained from `create_issue` is persisted (a `.endpoint` sidecar) when its PUT fails transiently, so a retry **resumes at the PUT** rather than re-creating the issue (which would mint a duplicate or trip server dedup and drop the bundle un-uploaded). The retry counter (and the abandon-at-cap deletion) advances **only** on backoff-paced attempts; a forced `flush()` re-attempts a queued report but never consumes its durable budget, so an offline flush can't delete a still-deliverable crash bundle. **APM:** separate lighter path `POST /v2/performance/transactions` (one bare transaction, batched ~30 s or realtime). diff --git a/PROGRESS.md b/PROGRESS.md index b0f39bf..cb135de 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -183,7 +183,7 @@ were pre-verified by inspection against `crash-handler` 0.6.3 + `libc`). 60-retry cap while offline β€” forced attempts are decoupled from the counter, so a still-deliverable crash bundle can't be deleted in ~1.3 s. Retry now **resumes at the presigned PUT** via a persisted `.endpoint` sidecar instead of re-POSTing - `create_issue` (no duplicate issues / no `12003`-drop of an un-uploaded bundle). + `create_issue` (no duplicate issues / no duplicate-verdict drop of an un-uploaded bundle). - **PII (high):** failed-request network entries scrub the URL out of reqwest's error `Display` (was leaking full credential-bearing URLs); header redaction is unified onto the normalized param matcher (`signature`/`csrf`/`api_key` no @@ -287,8 +287,8 @@ The defects, in the order they blocked things: application errors with `ok:false` and **HTTP 200**, but the transport only inspected the envelope for 4xx/5xx. The error degraded to "no endpoint in response", the report was dropped, and the server's explanation was thrown - away. The known codes `12003`/`12004` were reachable only on a 4xx too β€” so - `12004` skipped the signature blacklist entirely. + away. The known duplicate and too-many-similar verdicts were reachable only on a 4xx too β€” so + the latter skipped the signature blacklist entirely. 3. **Nothing could observe a dropped report** (`1b2a456`). `flush()` returns "the queue is empty", and the queue drains on success *and* on permanent failure alike β€” so it returned `true` while the server accepted nothing. diff --git a/crates/bugsee-core/src/queue.rs b/crates/bugsee-core/src/queue.rs index 84b2d47..a3feb86 100644 --- a/crates/bugsee-core/src/queue.rs +++ b/crates/bugsee-core/src/queue.rs @@ -190,7 +190,7 @@ pub fn next_attempt(report: &QueuedReport) -> i64 { /// The cached presigned upload endpoint for a report, if a prior attempt created /// the issue but the PUT failed transiently. A retry resumes at the PUT (step 3) /// instead of re-POSTing `create_issue` (which would mint a duplicate issue, or -/// trip server dedup `12003` and drop the bundle without ever uploading it). +/// trip server dedup and drop the bundle without ever uploading it). pub fn endpoint(report: &QueuedReport) -> Option { std::fs::read_to_string(report.endpoint_path()) .ok() @@ -272,7 +272,7 @@ fn load_blacklist(data_dir: &Path) -> BTreeSet { .unwrap_or_default() } -/// Persist crash signatures the server told us to stop sending (`12004`). +/// Persist crash signatures the server told us to stop sending (the too-many-similar verdict). /// /// Appends only the not-yet-present signatures, each on its own line, under /// `O_APPEND`. Appending (rather than read-modify-rewrite of the whole set) is diff --git a/crates/bugsee-core/src/runtime.rs b/crates/bugsee-core/src/runtime.rs index e37e986..b962f3d 100644 --- a/crates/bugsee-core/src/runtime.rs +++ b/crates/bugsee-core/src/runtime.rs @@ -2288,7 +2288,7 @@ fn deliver_queued( request_json, }; // Resume at the PUT if a prior attempt created the issue but the PUT failed - // (F14): re-POSTing create_issue mints a duplicate issue, or trips 12003 and + // (F14): re-POSTing create_issue mints a duplicate issue, or trips the server's dedup and // deletes the bundle unuploaded. let cached_endpoint = queue::endpoint(report); let mut endpoint_out: Option = None; diff --git a/crates/bugsee-core/src/transport.rs b/crates/bugsee-core/src/transport.rs index 7a62a34..1f9e136 100644 --- a/crates/bugsee-core/src/transport.rs +++ b/crates/bugsee-core/src/transport.rs @@ -55,7 +55,7 @@ pub trait Transport: Send + Sync { /// /// `cached_endpoint` is a presigned upload URL a prior attempt obtained but /// failed to PUT to; when set, delivery resumes at the PUT instead of re-creating -/// the issue (avoiding a duplicate issue / a `12003` drop of an un-uploaded +/// the issue (avoiding a duplicate issue / a duplicate-verdict drop of an un-uploaded /// bundle). On a *transient* PUT failure the endpoint that should be persisted /// for the next retry is written to `endpoint_out`; on any other outcome /// `endpoint_out` is left `None` (the caller then clears any stale cache). @@ -106,7 +106,7 @@ pub fn deliver( return match transport.upload_bundle(&endpoint, &report.zip) { Ok(()) => Ok(()), // Persist the endpoint so the retry resumes at the PUT rather - // than re-POSTing create_issue (duplicate issue / 12003 drop). + // than re-POSTing create_issue (duplicate issue / duplicate-verdict drop). Err(TransportError::Transient(e)) => { *endpoint_out = Some(endpoint); Err(TransportError::Transient(e))