diff --git a/Packages/WhoopStore/Sources/WhoopStore/OuraSidecarGenerations.swift b/Packages/WhoopStore/Sources/WhoopStore/OuraSidecarGenerations.swift new file mode 100644 index 0000000000..d7376d5446 --- /dev/null +++ b/Packages/WhoopStore/Sources/WhoopStore/OuraSidecarGenerations.swift @@ -0,0 +1,117 @@ +import Foundation + +/// Pure, deterministic selection rule for WHICH Oura diagnostics files a Test Centre export ships. +/// +/// The dumps (`Strand/BLE/Oura*Dump.swift`) keep a per-session generation ring: the live file +/// `oura--.jsonl` is the CURRENT session, and finished sessions roll into +/// `…jsonl.1` (newest retained) … `…jsonl.N` (oldest). Several files therefore map to one bundle +/// entry name, and the assembler must decide what to do about that. +/// +/// WHY THIS EXISTS — the rule it replaces was `LARGEST WINS`, and largest is the wrong criterion. +/// The wake drain flushes the whole night's bank in one burst, so on an ordinary morning the biggest +/// generation is whichever session happened to drain the most — routinely NOT the one holding the +/// night. Measured on `noop-master-iOS-v11.1.1-260905-0714.zip`: the shipped `oura-raw.jsonl` spanned +/// `2026-09-04 10:58:22 → 12:03:47 UTC` — the previous day's post-reinstall backfill, which was +/// simply the fattest file on disk. That morning's own 07:08 drain, the one carrying the night's +/// banked records (including the single `0x5c` of the day and the hypnogram), was in a different +/// generation and never left the device. Over the whole `Sleep Nights` corpus this cost **11 of 31 +/// nights**, which is the single biggest limit on the hypnogram ledger. +/// +/// THE RULE: do not pick at all — **concatenate the generations, oldest → newest**, and let the +/// bundle cap keep the tail. The cap already trims from the FRONT and keeps the most-recent bytes, +/// so merging in chronological order makes "the newest capture survives" a property of the two rules +/// composed, rather than a guess made blind at selection time. The morning drain is by construction +/// the newest data, so it can no longer be the thing that gets dropped. +/// +/// WHY NOT ship every generation as its own entry (the other candidate in the issue draft): each +/// extra entry competes for the same 20 MB cap under max-min fair allocation, so N generations of a +/// bulk sidecar would shrink every other stream's share — starving exactly the file this exists to +/// preserve. Merging costs one entry, keeps the bundle listing stable, and needs no new names. +/// +/// SAFE TO MERGE: generations are disjoint app sessions, and the raw capture deliberately keeps no +/// dedup high-water (a re-served record is itself evidence the ring re-sent it). Every line carries +/// its own `utc`/`iso`, and the offline reframer collapses duplicates by `(tag, ring-time)`, so a +/// concatenation reads exactly like the longer single capture it stands in for. +public enum OuraSidecarGenerations { + + /// Bundle entry name for a sidecar kind — the ring id is dropped, because bundle redaction scrubs + /// file CONTENT and never file NAMES. + public static func entryName(kind: String) -> String { "oura-\(kind).jsonl" } + + /// Classify a diagnostics filename as `(kind, generation)`, or nil when it is not one of `kinds`. + /// + /// `generation` is 0 for the live file and `n` for `…jsonl.n`, matching the dumps' own ring where + /// **1 is the NEWEST retained** session and higher indices are older. So a smaller generation is + /// always the more recent capture, and 0 (live) is the most recent of all — that single ordering + /// fact is what `mergePlan` sorts on. + /// + /// Rejects anything that is not exactly the ring's shape: no suffix at all beyond `.jsonl`, or a + /// dot followed by digits only. `oura-raw.txt`, `oura-raw-.jsonl.bak` and a bare + /// `oura-raw.jsonl` (no ring id) are all nil. + public static func classify(filename: String, kinds: [String]) -> (kind: String, generation: Int)? { + for kind in kinds where filename.hasPrefix("oura-\(kind)-") { + guard let range = filename.range(of: ".jsonl") else { continue } + let tail = filename[range.upperBound...] // "" for the live file, ".3" for a generation + if tail.isEmpty { return (kind, 0) } + guard tail.hasPrefix("."), tail.count > 1 else { continue } + let digits = tail.dropFirst() + guard digits.allSatisfy(\.isNumber), let n = Int(digits) else { continue } + return (kind, n) + } + return nil + } + + /// One bundle entry and the files that make it up, already in concatenation order. + public struct Plan: Equatable { + /// Normalized bundle entry name, e.g. `oura-raw.jsonl`. + public let entryName: String + /// Filenames to concatenate, **OLDEST → NEWEST**. Never empty. + public let files: [String] + public init(entryName: String, files: [String]) { + self.entryName = entryName + self.files = files + } + } + + /// Plan the merge for every sidecar kind present in `files`. + /// + /// Files are grouped by kind, ordered newest-first (live, then generation 1, 2, …), and taken + /// until their cumulative size reaches `ceilingBytes`; the file that crosses the ceiling is + /// INCLUDED (so the boundary never silently drops the run that spans it) and everything older is + /// left on the device. The kept run is then returned oldest → newest, ready to concatenate. + /// + /// `ceilingBytes` is a READ bound, not a budget: the bundle cap is the thing that decides what + /// finally ships, and it can never keep more than the whole cap for one entry — so reading beyond + /// that is pure wasted memory. Pass the bundle's cap. A non-positive ceiling still yields the + /// single newest file per kind, because shipping nothing would be worse than shipping the tail. + /// + /// Deterministic: the result does not depend on the order of `files`, and ties between two files + /// of the same kind and generation (impossible from one ring, possible if two rings wrote the same + /// kind) break on the filename so the bundle listing stays stable. + /// + /// Returns one `Plan` per kind, in `kinds` order, so the bundle listing is deterministic too. + public static func mergePlan(files: [(name: String, bytes: Int)], + kinds: [String], + ceilingBytes: Int) -> [Plan] { + var byKind: [String: [(generation: Int, name: String, bytes: Int)]] = [:] + for file in files { + guard let hit = classify(filename: file.name, kinds: kinds) else { continue } + byKind[hit.kind, default: []].append((hit.generation, file.name, max(0, file.bytes))) + } + return kinds.compactMap { kind -> Plan? in + guard let group = byKind[kind], !group.isEmpty else { return nil } + // Newest first: generation 0 (live), then 1 (newest retained), … N (oldest). + let newestFirst = group.sorted { + $0.generation != $1.generation ? $0.generation < $1.generation : $0.name < $1.name + } + var kept: [String] = [] + var accumulated = 0 + for file in newestFirst { + kept.append(file.name) + accumulated += file.bytes + if accumulated >= ceilingBytes { break } + } + return Plan(entryName: entryName(kind: kind), files: kept.reversed()) + } + } +} diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/OuraSidecarGenerationsTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/OuraSidecarGenerationsTests.swift new file mode 100644 index 0000000000..7ea6ff4727 --- /dev/null +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/OuraSidecarGenerationsTests.swift @@ -0,0 +1,140 @@ +import XCTest +@testable import WhoopStore + +/// Pins the export's generation-selection rule. These run under `swift test` with no app, no strap and +/// no CoreBluetooth, which matters because the caller (`TestBundleAssembler`) is app-target Swift that +/// no default CI job compiles — this is the only automated cover the rule has. +final class OuraSidecarGenerationsTests: XCTestCase { + + private let kinds = ["raw", "cva-ppg", "motion", "activity"] + + // MARK: - classify + + func testClassifyLiveFileIsGenerationZero() { + let hit = OuraSidecarGenerations.classify(filename: "oura-raw-AABB.jsonl", kinds: kinds) + XCTAssertEqual(hit?.kind, "raw") + XCTAssertEqual(hit?.generation, 0) + } + + func testClassifyRolledGenerations() { + for n in 1...8 { + let hit = OuraSidecarGenerations.classify(filename: "oura-raw-AABB.jsonl.\(n)", kinds: kinds) + XCTAssertEqual(hit?.kind, "raw") + XCTAssertEqual(hit?.generation, n, "generation \(n)") + } + // Multi-digit, so the ring is not silently capped at nine. + XCTAssertEqual(OuraSidecarGenerations.classify(filename: "oura-raw-AABB.jsonl.12", + kinds: kinds)?.generation, 12) + } + + func testClassifyRejectsNonRingShapes() { + // Not a JSONL sidecar, a non-numeric suffix, a trailing dot, and no ring id at all. + for bad in ["oura-raw.txt", "oura-raw-AABB.jsonl.bak", "oura-raw-AABB.jsonl.", + "oura-raw-AABB.jsonl.1a", "raw-capture.jsonl", "whoop.sqlite", "oura-raw.jsonl"] { + XCTAssertNil(OuraSidecarGenerations.classify(filename: bad, kinds: kinds), bad) + } + // A kind we do not collect. + XCTAssertNil(OuraSidecarGenerations.classify(filename: "oura-unknown-AABB.jsonl", kinds: kinds)) + } + + func testEntryNameDropsRingId() { + XCTAssertEqual(OuraSidecarGenerations.entryName(kind: "raw"), "oura-raw.jsonl") + } + + // MARK: - mergePlan ordering (the actual defect) + + /// THE REGRESSION TEST. Under the old largest-wins rule this exact input shipped `gen 2` alone — + /// the fattest file — and the newest capture never left the device. The plan must now END with the + /// live file, because the cap keeps the TAIL. + func testMergePlanEndsWithTheNewestFileEvenWhenAnOlderGenerationIsFarLarger() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-raw-AABB.jsonl", 100), // live: this morning's drain, small + ("oura-raw-AABB.jsonl.1", 500), + ("oura-raw-AABB.jsonl.2", 9_000)], // yesterday's backfill, the old winner + kinds: kinds, ceilingBytes: 1_000_000) + XCTAssertEqual(plan.count, 1) + XCTAssertEqual(plan[0].entryName, "oura-raw.jsonl") + XCTAssertEqual(plan[0].files, ["oura-raw-AABB.jsonl.2", + "oura-raw-AABB.jsonl.1", + "oura-raw-AABB.jsonl"]) + XCTAssertEqual(plan[0].files.last, "oura-raw-AABB.jsonl", "newest must be last") + } + + func testMergePlanIsIndependentOfInputOrder() { + let forward: [(name: String, bytes: Int)] = [("oura-raw-A.jsonl", 1), ("oura-raw-A.jsonl.1", 2), + ("oura-raw-A.jsonl.2", 3)] + let a = OuraSidecarGenerations.mergePlan(files: forward, kinds: kinds, ceilingBytes: 1_000) + let b = OuraSidecarGenerations.mergePlan(files: forward.reversed(), kinds: kinds, ceilingBytes: 1_000) + XCTAssertEqual(a, b) + } + + func testMergePlanOrdersEntriesByKindsNotByDiscovery() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-motion-A.jsonl", 1), ("oura-raw-A.jsonl", 1), ("oura-cva-ppg-A.jsonl", 1)], + kinds: kinds, ceilingBytes: 1_000) + XCTAssertEqual(plan.map(\.entryName), ["oura-raw.jsonl", "oura-cva-ppg.jsonl", "oura-motion.jsonl"]) + } + + func testMergePlanSkipsKindsWithNoFiles() { + let plan = OuraSidecarGenerations.mergePlan(files: [("oura-raw-A.jsonl", 1)], + kinds: kinds, ceilingBytes: 1_000) + XCTAssertEqual(plan.map(\.entryName), ["oura-raw.jsonl"]) + } + + // MARK: - mergePlan ceiling + + /// The ceiling is a READ bound. The file that crosses it is kept, so the run spanning the boundary + /// is never half-read; everything strictly older is left on the device. + func testMergePlanStopsAtCeilingButIncludesTheCrossingFile() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-raw-A.jsonl", 60), // newest + ("oura-raw-A.jsonl.1", 60), // crosses the 100 ceiling -> included + ("oura-raw-A.jsonl.2", 60)], // older -> dropped + kinds: kinds, ceilingBytes: 100) + XCTAssertEqual(plan[0].files, ["oura-raw-A.jsonl.1", "oura-raw-A.jsonl"]) + } + + /// A single file at or over the ceiling reads exactly as it did before this change — one file, the + /// newest — so the memory profile never regresses against largest-wins. + func testMergePlanKeepsOnlyTheNewestWhenItAlreadyFillsTheCeiling() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-raw-A.jsonl", 500), ("oura-raw-A.jsonl.1", 500)], + kinds: kinds, ceilingBytes: 100) + XCTAssertEqual(plan[0].files, ["oura-raw-A.jsonl"]) + } + + /// Shipping nothing would be strictly worse than shipping the newest tail, so a degenerate ceiling + /// still yields one file rather than an empty plan. + func testMergePlanWithNonPositiveCeilingStillShipsTheNewestFile() { + for ceiling in [0, -1] { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-raw-A.jsonl", 10), ("oura-raw-A.jsonl.1", 10)], + kinds: kinds, ceilingBytes: ceiling) + XCTAssertEqual(plan[0].files, ["oura-raw-A.jsonl"], "ceiling \(ceiling)") + } + } + + func testMergePlanNeverReturnsAnEmptyFileList() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-raw-A.jsonl", 0)], kinds: kinds, ceilingBytes: 1_000) + XCTAssertEqual(plan[0].files, ["oura-raw-A.jsonl"]) + } + + func testMergePlanIgnoresUnrelatedFiles() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("raw-capture.jsonl", 10), ("whoop.sqlite", 10), ("oura-raw-A.jsonl", 10)], + kinds: kinds, ceilingBytes: 1_000) + XCTAssertEqual(plan.count, 1) + XCTAssertEqual(plan[0].files, ["oura-raw-A.jsonl"]) + } + + /// Two rings that wrote the same kind collide on generation. The tie must break deterministically + /// rather than by directory order, or the bundle listing changes between exports. + func testMergePlanBreaksSameGenerationTiesOnFilename() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-raw-BBBB.jsonl", 10), ("oura-raw-AAAA.jsonl", 10)], + kinds: kinds, ceilingBytes: 1_000) + // Newest-first sorts A before B; reversed for concatenation, B lands last. + XCTAssertEqual(plan[0].files, ["oura-raw-BBBB.jsonl", "oura-raw-AAAA.jsonl"]) + } +} diff --git a/Strand/BLE/OuraRawDump.swift b/Strand/BLE/OuraRawDump.swift index 335b515b01..de9d135a04 100644 --- a/Strand/BLE/OuraRawDump.swift +++ b/Strand/BLE/OuraRawDump.swift @@ -18,18 +18,70 @@ import WhoopStore /// decoded sidecars there is NO dedup high-water: a re-served record is still evidence the ring re-sent it, /// and the offline reframer collapses duplicates by (tag, ring-time). /// +/// LIFETIME: the live file holds exactly ONE capture session. At the START of a session — the first time +/// this object resolves its file — the previous session is rolled into a numbered generation ring +/// (`.1` … `.N`, newest first) and never touched again while the session runs. One `OuraRawDump` is built +/// per `OuraLiveSource`, and a source survives BLE reconnects (`SourceCoordinator` returns early when the +/// active strap is unchanged), so a drain that disconnects and re-engages several times still lands whole +/// in one file. +/// +/// WHY NOT a size threshold, which is what this used to do: a threshold can fire MID-DRAIN, and on +/// 2026-08-09 it did — 25 MB rolled at 07:06:58 during the morning offload, carrying 32,005 records of +/// that drain and 71 of the night's 75 `0x5D` records into the `.1`. Lowering the threshold makes that +/// MORE frequent, not less; only rolling on a session boundary removes it. The corpus stays bounded +/// because a session is bounded: an overnight drain is ~32,000 records ≈ 4 MB. +/// +/// WHY A RING AND NOT ONE `.1`, which is what the first version of this did: **retaining a single +/// generation is not enough to survive an ordinary morning.** Measured 2026-08-10 — the app relaunched +/// twice after wake (07:45 and 08:17 local), so by the time the capture was pulled off the phone BOTH +/// retained slots held post-wake sessions of 87 KB and 44 KB, and the night's 4 MB was gone. Session +/// scoping fixed the mid-drain roll and then lost the night a different way: rotation ate a night every +/// ~5–6 days at random; one-deep session scoping ate it every morning the app restarted twice, which is +/// the normal morning. An iOS app is relaunched whenever the user opens it, so the number of session +/// boundaries between a night and its export is not something this class can predict — it can only keep +/// enough of them. +/// +/// Two rules keep the ring honest, and both exist because of that measurement: +/// * a session under `minRolledBytes` is DISCARDED rather than rolled, so a connect that captured +/// essentially nothing (the 44 KB one above) cannot evict a real overnight capture; +/// * the retained generations are pruned oldest-first to `maxRetainedBytes`, so the ring is bounded in +/// BYTES and not merely in file count (each generation may be up to `maxSessionBytes`). +/// +/// `TestBundleAssembler` ships the LARGEST generation for this kind, so the export carries the night even +/// when the live file is a fresh 30-second session — which is exactly the state a morning export is taken +/// in. Keeping generations on disk without that change would only help someone with a USB cable. +/// /// Location: `/OpenWhoop/Diagnostics/oura-raw-.jsonl` — beside the SQLite. final class OuraRawDump { private let deviceId: String private let log: (String) -> Void + private let directory: URL? private var fileURL: URL? private var resolveFailed = false private var announced = false + private var sessionBytes = 0 + private var ceilingHit = false + + /// Hard ceiling on ONE session's file. Never a rotation — a session that somehow reaches this stops + /// appending and says so once, because rolling mid-session is the exact failure this class was changed + /// to remove. Far above a real overnight drain (~4 MB), so in practice it never fires; it exists so a + /// pathological backlog cannot fill the disk. + static let maxSessionBytes = 25 * 1024 * 1024 + + /// How many completed sessions are kept beside the live file, as `.1` (newest) … `.N` (oldest). Eight + /// covers a morning of app relaunches with room to spare, and at a typical ~4 MB overnight session the + /// whole ring is ~32 MB — well inside `maxRetainedBytes`, which is what actually bounds it. + static let retainedGenerations = 8 + + /// A finished session smaller than this is deleted instead of rolled. It is not a judgement about which + /// captures matter — it is the one rule that stops a burst of trivial reconnect sessions from flushing + /// the ring. 16 KB is ~100 records: a connect that drained nothing. Deliberately far below the 44 KB + /// session that destroyed the 2026-08-10 capture, so the rule is a backstop and the ring does the work. + static let minRolledBytes = 16 * 1024 - /// Rotate the sidecar past this size (keeping one previous ".1"), so an always-on research corpus is - /// bounded to ~2× this on disk instead of growing forever. Matches `OuraMotionDump`/`OuraActivityDump` — - /// this file needs it MORE than either: full hex of every notification, and no dedup high-water. - private static let maxBytes = 25 * 1024 * 1024 + /// Byte budget for the retained generations (the live file is excluded — it has its own ceiling). + /// Pruned oldest-first, so a pathological run of large sessions cannot fill the disk. + static let maxRetainedBytes = 64 * 1024 * 1024 private static let iso: ISO8601DateFormatter = { let f = ISO8601DateFormatter() @@ -37,28 +89,18 @@ final class OuraRawDump { return f }() - init(deviceId: String, log: @escaping (String) -> Void) { + /// `directory` is injectable so the session-roll behaviour is testable against a temp dir; production + /// passes nil and gets `/OpenWhoop/Diagnostics`. + init(deviceId: String, log: @escaping (String) -> Void, directory: URL? = nil) { self.deviceId = deviceId self.log = log + self.directory = directory } /// Append one raw notification's bytes verbatim (hex-encoded), stamped with wall-clock arrival time. /// No-op on empty input. Best-effort: any file error is logged once and never disrupts the BLE path. func record(bytes: [UInt8]) { - guard !bytes.isEmpty, var url = resolveURL() else { return } - - // Bound the corpus (rotate to a single ".1", dropping the prior one) so an always-on research sidecar - // can't grow unbounded — same rotation as OuraMotionDump. Read the size via a fresh FileManager stat - // rather than URL.resourceValues, whose cache on the reused URL can return a stale small size. - let size = ((try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? Int) ?? 0 - if size > Self.maxBytes { - let old = url.deletingLastPathComponent().appendingPathComponent(url.lastPathComponent + ".1") - try? FileManager.default.removeItem(at: old) - try? FileManager.default.moveItem(at: url, to: old) - fileURL = nil - guard let fresh = resolveURL() else { return } - url = fresh - } + guard !bytes.isEmpty, let url = resolveURL() else { return } let now = Date() let line = OuraRawDumpLine.encode( @@ -66,11 +108,24 @@ final class OuraRawDump { iso: Self.iso.string(from: now), bytes: bytes) guard let data = (line + "\n").data(using: .utf8) else { return } + + // Ceiling check against a running counter, not a stat per record: the file starts EMPTY each + // session (resolveURL rolls it), so the counter is authoritative and costs no filesystem call. + guard sessionBytes + data.count <= Self.maxSessionBytes else { + if !ceilingHit { + ceilingHit = true + log("Oura: raw capture reached its \(Self.maxSessionBytes / (1024 * 1024)) MB session " + + "ceiling - no longer appending (the file is NOT rolled mid-session)") + } + return + } + do { let handle = try FileHandle(forWritingTo: url) defer { try? handle.close() } try handle.seekToEnd() handle.write(data) + sessionBytes += data.count } catch { log("Oura: raw dump write failed - \(error.localizedDescription)") return @@ -82,21 +137,41 @@ final class OuraRawDump { } } - /// Resolve (and create on first use) the sidecar file + its parent directory. Cached; a failure is - /// logged once and latched so we never spam the strap log on a read-only volume. + /// Resolve the sidecar file + its parent directory, ROLLING the previous session's file aside on the + /// first call so this session starts from empty. Cached, so the roll happens exactly once per object — + /// which is what makes it a session boundary rather than a size threshold. A failure is logged once and + /// latched so we never spam the strap log on a read-only volume. private func resolveURL() -> URL? { if let fileURL { return fileURL } if resolveFailed { return nil } do { - let base = try FileManager.default.url(for: .applicationSupportDirectory, in: .userDomainMask, - appropriateFor: nil, create: true) - let dir = base.appendingPathComponent("OpenWhoop/Diagnostics", isDirectory: true) + let dir: URL + if let directory { + dir = directory + } else { + let base = try FileManager.default.url(for: .applicationSupportDirectory, in: .userDomainMask, + appropriateFor: nil, create: true) + dir = base.appendingPathComponent("OpenWhoop/Diagnostics", isDirectory: true) + } try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) let safeId = deviceId.replacingOccurrences(of: "/", with: "_") let url = dir.appendingPathComponent("oura-raw-\(safeId).jsonl") + + // Session roll: shift the previous session into the generation ring and begin fresh, so the live + // file is always exactly this capture. A leftover that captured essentially nothing (including an + // empty one) is dropped rather than rolled — spending a generation on it is how the 2026-08-10 + // capture was lost. + let existing = Self.byteSize(url) + if existing >= Self.minRolledBytes { + rollGenerations(in: dir, base: url) + pruneRetained(in: dir, base: url) + } else if existing > 0 { + try? FileManager.default.removeItem(at: url) + } if !FileManager.default.fileExists(atPath: url.path) { FileManager.default.createFile(atPath: url.path, contents: nil) } + sessionBytes = 0 fileURL = url return url } catch { @@ -105,4 +180,54 @@ final class OuraRawDump { return nil } } + + /// The generation file for `index` (1 = newest retained session). + static func generationURL(base: URL, index: Int) -> URL { + base.deletingLastPathComponent() + .appendingPathComponent(base.lastPathComponent + ".\(index)") + } + + private static func byteSize(_ url: URL) -> Int { + ((try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? Int) ?? 0 + } + + /// Shift the ring up by one: the oldest generation is dropped, every other moves down a slot, and the + /// finished live file becomes `.1`. Best-effort throughout — a failed move must never break the BLE + /// path, and the worst outcome is one lost generation rather than a lost session. + private func rollGenerations(in dir: URL, base url: URL) { + let oldest = Self.generationURL(base: url, index: Self.retainedGenerations) + try? FileManager.default.removeItem(at: oldest) + var index = Self.retainedGenerations - 1 + while index >= 1 { + let from = Self.generationURL(base: url, index: index) + let to = Self.generationURL(base: url, index: index + 1) + if FileManager.default.fileExists(atPath: from.path) { + try? FileManager.default.removeItem(at: to) + try? FileManager.default.moveItem(at: from, to: to) + } + index -= 1 + } + try? FileManager.default.moveItem(at: url, to: Self.generationURL(base: url, index: 1)) + } + + /// Drop retained generations oldest-first until they fit `maxRetainedBytes`. Counts only the ring: the + /// live file has its own per-session ceiling and is not part of this budget. + private func pruneRetained(in dir: URL, base url: URL) { + var total = 0 + var kept: [URL] = [] + for index in 1...Self.retainedGenerations { + let gen = Self.generationURL(base: url, index: index) + let size = Self.byteSize(gen) + guard size > 0 else { continue } + total += size + kept.append(gen) + } + guard total > Self.maxRetainedBytes else { return } + for gen in kept.reversed() { // oldest slot first + guard total > Self.maxRetainedBytes else { break } + total -= Self.byteSize(gen) + try? FileManager.default.removeItem(at: gen) + } + log("Oura: raw capture ring pruned to its \(Self.maxRetainedBytes / (1024 * 1024)) MB budget") + } } diff --git a/Strand/System/TestBundleAssembler.swift b/Strand/System/TestBundleAssembler.swift index d1a949cf00..e10719a41a 100644 --- a/Strand/System/TestBundleAssembler.swift +++ b/Strand/System/TestBundleAssembler.swift @@ -1,5 +1,6 @@ import Foundation import StrandAnalytics +import WhoopStore /// Assembles the Test Centre export bundle: gathers report.txt, meta.json, raw-capture and last-crash, /// runs the redaction pass over EVERY file, applies the 20 MB cap, and hands the entries to @@ -14,14 +15,25 @@ enum TestBundleAssembler { /// The redaction stamp written into meta.json so a maintainer knows the whole-bundle scrub ran. static let redactionVersion = "v2" + /// The bundle's hard byte cap (spec section 5.4; under GitHub's 25 MB attachment limit). Shared by + /// `capEntries` (what finally ships) and `ouraDiagnosticEntries` (how much it is worth reading), so the + /// read ceiling can never drift above the budget it feeds. + static let defaultCapBytes = 20 * 1024 * 1024 + + /// The Oura Diagnostics-dir sidecar kinds the bundle attaches — the SINGLE source of truth + /// `normalizedOuraEntryName`/`ouraDiagnosticEntries`/`trimmableNames`/`ouraSidecarNames` all derive + /// from, so a new dump writer (`Strand/BLE/Oura*Dump.swift`) needs exactly one line added here to be + /// picked up everywhere instead of silently missing the export bundle. (Landed after `cva-ppg` and + /// `real-steps` shipped their writers but not their kind entry — both wrote real files on-device that + /// the bundler's hardcoded 3-kind list then dropped from every export.) + static let ouraSidecarKinds: [String] = ["raw", "ibihr", "activity", "cva-ppg", "motion", "real-steps"] + /// The bundle files that may be trimmed to fit the cap (newest-tail kept). The strap-log tail and /// meta.json are already bounded, so only these raw research streams can blow the budget: the WHOOP - /// frame capture plus the Oura ring's Tier-B JSONL sidecars (raw notifications / IBI-HR / activity MET). - /// Everything NOT listed here is kept whole and its bytes are reserved before the trimmable group is - /// given the remainder. These are the NORMALIZED entry names (the ring id is dropped from the filename). - static let trimmableNames: Set = [ - "raw-capture.jsonl", "oura-raw.jsonl", "oura-ibihr.jsonl", "oura-activity.jsonl", - ] + /// frame capture plus the Oura ring's Tier-B JSONL sidecars (raw notifications / IBI-HR / activity MET / + /// CVA-PPG / motion / real-steps). Everything NOT listed here is kept whole and its bytes are reserved + /// before the trimmable group is given the remainder. NORMALIZED entry names (ring id dropped). + static let trimmableNames: Set = Set(["raw-capture.jsonl"] + ouraSidecarKinds.map { "oura-\($0).jsonl" }) /// #572 follow-up: the Oura Tier-B sidecars carry the ring id in a `"deviceId"` JSON field. The /// whole-bundle UUID scrub (`LiveState.redactPii`) masks it only when it is a CANONICAL dashed @@ -30,9 +42,7 @@ enum TestBundleAssembler { /// covered, and (unlike broadening the UUID regex to dashless hex) it CANNOT touch the raw `hex` capture /// field, which a greedier rule would shred. Scoped to the sidecars so a non-PII logical `deviceId` /// (e.g. "my-whoop") elsewhere in the bundle stays readable. NORMALIZED names (ring id already dropped). - static let ouraSidecarNames: Set = [ - "oura-raw.jsonl", "oura-ibihr.jsonl", "oura-activity.jsonl", - ] + static let ouraSidecarNames: Set = Set(ouraSidecarKinds.map { "oura-\($0).jsonl" }) /// `oura-raw.jsonl` is the ONE sidecar whose payload is nothing but hex (`OuraRawDumpLine.encode`) — /// `oura-ibihr.jsonl`/`oura-activity.jsonl` encode DECODED numeric fields, so `LiveState.redactHexDump`'s @@ -76,35 +86,83 @@ enum TestBundleAssembler { /// Hard cap the bundle at `capBytes` (20 MB default, under GitHub's 25 MB; spec section 5.4). The /// strap-log tail and meta.json are already bounded, so only the `trimmableNames` research streams can - /// exceed. We reserve the whole size of every non-trimmable file, then split the remaining budget across - /// the trimmable files IN PROPORTION to their size, keeping the MOST-RECENT tail of each (newest data is - /// the most diagnostic) and trimming from the front. With a single trimmable file this is identical to - /// the prior raw-capture-only behaviour (its share is the whole remainder). Returns the capped entries - /// plus whether any truncation happened, which the caller writes to meta.truncated. + /// exceed. We reserve the whole size of every non-trimmable file, then share the remaining budget across + /// the trimmable files, keeping the MOST-RECENT tail of each (newest data is the most diagnostic) and + /// trimming from the front. Returns the capped entries plus whether any truncation happened, which the + /// caller writes to meta.truncated. + /// + /// ALLOCATION: **max-min fair** (water-filling), NOT proportional-to-size. Each stream is offered an + /// equal share of what is left; a stream smaller than its share is kept WHOLE and its unused remainder + /// flows to the streams that are still over. See `fairAllowances` for why, and for the measured export + /// that motivated it. With a single trimmable file this is still identical to the original + /// raw-capture-only behaviour: its share is the whole remainder. static func capEntries(_ entries: [FileExport.BundleEntry], - capBytes: Int = 20 * 1024 * 1024) -> (entries: [FileExport.BundleEntry], truncated: Bool) { + capBytes: Int = defaultCapBytes) -> (entries: [FileExport.BundleEntry], truncated: Bool) { let total = entries.reduce(0) { $0 + $1.data.count } guard total > capBytes else { return (entries, false) } // Reserve the non-trimmable files (kept whole), then share the remainder across the trimmable ones. let nonTrimmable = entries.filter { !trimmableNames.contains($0.name) }.reduce(0) { $0 + $1.data.count } let budget = max(0, capBytes - nonTrimmable) - let trimmableTotal = entries.filter { trimmableNames.contains($0.name) }.reduce(0) { $0 + $1.data.count } + let allowance = fairAllowances( + sizes: entries.filter { trimmableNames.contains($0.name) }.map { ($0.name, $0.data.count) }, + budget: budget) var truncated = false let capped = entries.map { entry -> FileExport.BundleEntry in guard trimmableNames.contains(entry.name) else { return entry } - // Each trimmable file gets a byte share proportional to its size (integer floor keeps the sum - // under budget, so the bundle never breaches the cap). Kept whole if already within its share. - let share = trimmableTotal > 0 - ? Int(Double(budget) * Double(entry.data.count) / Double(trimmableTotal)) - : 0 + let share = allowance[entry.name] ?? 0 guard entry.data.count > share else { return entry } truncated = true - // Keep the tail (most recent): the last `share` bytes. - return FileExport.BundleEntry(name: entry.name, data: Data(entry.data.suffix(share))) + // Keep the tail (most recent): the last `share` bytes, then snap forward to the next full + // JSONL line. A raw byte-count tail can (and in production did - a real export shipped + // oura-cva-ppg.jsonl/oura-real-steps.jsonl/oura-motion.jsonl each with a corrupted first + // line) land mid-record; every trimmable name is newline-delimited JSONL, so dropping the + // partial line at the front keeps every kept line honest. + return FileExport.BundleEntry(name: entry.name, + data: Self.trimToLineBoundary(entry.data.suffix(share))) } return (capped, truncated) } + /// Split `budget` bytes across the named streams **max-min fairly**: walk them smallest-first, offering + /// each an equal share of whatever budget remains, and let a stream that fits under its share take only + /// what it needs so the surplus rolls forward to the larger ones. Integer floor keeps the sum at or under + /// `budget`, so the bundle can never breach the cap. Pure; `sizes` order does not affect the result. + /// + /// WHY NOT PROPORTIONAL-TO-SIZE (the original rule): it hands the budget to the BULKIEST stream, and for + /// these sidecars bulk is close to inversely correlated with diagnostic value. Measured on a real export + /// (`noop-master-iOS-v9.3.1-260809-0716.zip`, six trimmable sidecars over a 20,863,648-byte budget): + /// `oura-spo2.jsonl` — a per-sample dump whose values are also in the SQLite the same bundle ships — + /// took **53.9 %**, while `oura-raw.jsonl` got **1.9 %** (388,064 bytes: 8 min 43 s of an 8.4 h night). + /// The raw sidecar is the UNDECODED wire bytes, the one file in the bundle from which a protocol fact can + /// be re-derived independently of our own decoder, so starving it costs a night of verification that + /// nothing else can supply. Under this rule the ~3.47 MB fair share keeps it whole and the surplus goes + /// where it is merely bulk. Same 20 MB bundle, no editorial ranking of the streams. + static func fairAllowances(sizes: [(name: String, bytes: Int)], budget: Int) -> [String: Int] { + // Smallest-first, name-tiebroken so the result is deterministic regardless of input order. + let ordered = sizes.sorted { $0.bytes != $1.bytes ? $0.bytes < $1.bytes : $0.name < $1.name } + var allowance: [String: Int] = [:] + var left = max(0, budget) + var unserved = ordered.count + for stream in ordered { + let share = unserved > 0 ? left / unserved : 0 + let give = min(max(0, stream.bytes), share) + allowance[stream.name] = give + left -= give + unserved -= 1 + } + return allowance + } + + /// Drop bytes up to and including the first newline in `data`, so a raw byte-count tail (from + /// `capEntries`'s fair-share trim) never starts mid-line for a newline-delimited JSONL sidecar. + /// Returns `data` unchanged when it contains no newline (a single very-long line, or already empty) + /// - never worse than the untrimmed behaviour, just not improved. Pure/testable. + static func trimToLineBoundary(_ data: Data) -> Data { + guard let newline = data.firstIndex(of: 0x0A) else { return data } + let start = data.index(after: newline) + return data.subdata(in: start..-.jsonl` → `oura-.jsonl`, which - /// (a) keeps the ring id out of the bundle's filenames (redaction scrubs content, never names) and - /// (b) lands on the `trimmableNames` set so the cap can trim it. Pure/string-only for unit testing. + /// file is not one of the Oura sidecars in `ouraSidecarKinds`. `oura--.jsonl` → + /// `oura-.jsonl`, which (a) keeps the ring id out of the bundle's filenames (redaction scrubs + /// content, never names) and (b) lands on the `trimmableNames` set so the cap can trim it. + /// + /// A ROLLED GENERATION (`oura--.jsonl.`, written by `OuraRawDump`'s session ring) maps + /// to the SAME normalized name on purpose — `ouraDiagnosticEntries` merges the generations behind that + /// one name rather than choosing between them. Before generations were recognised at all, the `.jsonl` + /// suffix test rejected every one of them and the bundle could only ever ship the live file — which on + /// 2026-08-10 was a 30-second morning session while the night's 4 MB sat in a generation the export + /// never looked at. + /// + /// Thin wrapper over `WhoopStore.OuraSidecarGenerations`, where the rule is pure and unit-tested under + /// `swift test`; this file is app-target Swift that no default CI job compiles. static func normalizedOuraEntryName(forFile filename: String) -> String? { - for kind in ["raw", "ibihr", "activity"] { - if filename.hasPrefix("oura-\(kind)-"), filename.hasSuffix(".jsonl") { - return "oura-\(kind).jsonl" - } - } - return nil + guard let hit = OuraSidecarGenerations.classify(filename: filename, kinds: ouraSidecarKinds) + else { return nil } + return OuraSidecarGenerations.entryName(kind: hit.kind) } /// Gather the Oura ring's Tier-B JSONL sidecars as bundle entries, normalized names and RAW bytes (the /// caller redacts + caps). Enumerates the Diagnostics dir rather than reconstructing per-ring filenames, - /// so it needs no active-ring id and picks up whatever was captured. If two rings produced the same kind - /// (rare), the entry names would collide; we keep the LARGEST file per normalized name (the fuller - /// capture is the more useful one) so the bundle never carries duplicate names. - static func ouraDiagnosticEntries() -> [FileExport.BundleEntry] { - guard let dir = ouraDiagnosticsDir(), - let files = try? FileManager.default.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil) + /// so it needs no active-ring id and picks up whatever was captured. + /// + /// Several files map to one entry name — the live session plus its rolled generations, and (rarely) two + /// rings that wrote the same kind. We **concatenate them oldest → newest** rather than choosing one, so + /// that the cap's existing keep-the-tail trim decides what ships. `OuraSidecarGenerations.mergePlan` + /// holds that rule, pure and unit-tested; see its doc comment for the measured export that motivated it. + /// + /// ⚠️ THIS REPLACES `largest-wins`, which was not a buggy pick but the wrong criterion: the wake drain + /// flushes the night's whole bank at once, so the biggest generation is whichever session drained most + /// — routinely not the one holding the night. It cost 11 of 31 nights in the `Sleep Nights` corpus. + /// + /// Reads are bounded by `capBytes`: an entry can never keep more than the whole bundle cap, so pulling + /// older generations past that point would be wasted memory. A kind whose newest file already fills the + /// cap therefore reads exactly one file, as it did before. + /// + /// `directory` is injectable so the merge is testable against a temp dir laid out like a real ring; + /// production passes nil and gets `/OpenWhoop/Diagnostics`. + static func ouraDiagnosticEntries(capBytes: Int = defaultCapBytes, + directory: URL? = nil) -> [FileExport.BundleEntry] { + guard let dir = directory ?? ouraDiagnosticsDir(), + let files = try? FileManager.default.contentsOfDirectory( + at: dir, includingPropertiesForKeys: [.fileSizeKey]) else { return [] } - var byName: [String: FileExport.BundleEntry] = [:] - for url in files.sorted(by: { $0.lastPathComponent < $1.lastPathComponent }) { - guard let name = normalizedOuraEntryName(forFile: url.lastPathComponent), - let entry = fileEntry(at: url, name: name) else { continue } - if let existing = byName[name], existing.data.count >= entry.data.count { continue } - byName[name] = entry + let sizes: [(name: String, bytes: Int)] = files.map { url in + // Size from the directory entry, so planning never reads a file it will not ship. + let bytes = (try? url.resourceValues(forKeys: [.fileSizeKey]))?.fileSize ?? 0 + return (url.lastPathComponent, bytes) + } + let plans = OuraSidecarGenerations.mergePlan(files: sizes, kinds: ouraSidecarKinds, + ceilingBytes: capBytes) + return plans.compactMap { plan in + var merged = Data() + for filename in plan.files { + guard let part = try? Data(contentsOf: dir.appendingPathComponent(filename)), + !part.isEmpty else { continue } + // Every sidecar is newline-delimited JSONL. A file whose last write did not end in a + // newline would otherwise splice its final record onto the next generation's first one, + // producing a line that parses as neither. Cheap to guarantee, silent to get wrong. + if let last = merged.last, last != 0x0A { merged.append(0x0A) } + merged.append(part) + } + guard !merged.isEmpty else { return nil } + return FileExport.BundleEntry(name: plan.entryName, data: merged) } - // Stable order (raw, ibihr, activity) so the bundle listing is deterministic. - return ["oura-raw.jsonl", "oura-ibihr.jsonl", "oura-activity.jsonl"].compactMap { byName[$0] } } /// The on-disk path a crash report would live at, when a crash handler is wired. There is no producer diff --git a/StrandTests/OuraRawDumpSessionTests.swift b/StrandTests/OuraRawDumpSessionTests.swift new file mode 100644 index 0000000000..49eb91f53d --- /dev/null +++ b/StrandTests/OuraRawDumpSessionTests.swift @@ -0,0 +1,167 @@ +import XCTest +@testable import Strand + +/// The raw Oura sidecar is scoped to ONE capture session (option 1 of the 2026-08-09 sidecar decision), +/// and completed sessions are kept in a numbered generation ring (the 2026-08-10 follow-up). +/// +/// These pin two properties, each bought with a lost night of raw evidence: +/// * the live file is never rolled part-way through a session — a 25 MB threshold fired mid-drain at +/// 07:06:58 on 2026-08-09 and carried 71 of the night's 75 `0x5D` records out of the export; +/// * a session boundary does not evict a real capture — one-deep retention lost the 2026-08-10 night +/// because the app relaunched twice after wake (07:45, 08:17), filling both slots with sessions of +/// 87 KB and 44 KB. +final class OuraRawDumpSessionTests: XCTestCase { + private var dir: URL! + + override func setUpWithError() throws { + dir = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("oura-raw-tests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + try? FileManager.default.removeItem(at: dir) + } + + private func live() -> URL { dir.appendingPathComponent("oura-raw-ring1.jsonl") } + private func previous() -> URL { generation(1) } + private func generation(_ index: Int) -> URL { + dir.appendingPathComponent("oura-raw-ring1.jsonl.\(index)") + } + private func lines(_ url: URL) -> [String] { + guard let s = try? String(contentsOf: url, encoding: .utf8) else { return [] } + return s.split(separator: "\n").map(String.init) + } + + private func size(_ url: URL) -> Int { + ((try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? Int) ?? 0 + } + + /// A session big enough to be worth a generation slot (>= `minRolledBytes`), tagged so it can be + /// identified after rolling. `repeat` is load-bearing: a fresh `OuraRawDump` has not resolved its file + /// yet, so before the FIRST record the live file still holds the previous session — a leading size + /// check would see it already "full" and write nothing at all. + private func writeSubstantialSession(tag: UInt8) { + let dump = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + let payload = [tag] + [UInt8](repeating: 0x00, count: 512) + repeat { dump.record(bytes: payload) } while size(live()) < OuraRawDump.minRolledBytes + } + + /// The `hex` field of a file's first record. Read the FIELD, never the whole line: the line also carries + /// a unix `utc` and an ISO stamp, and a substring search for a tag like "22" hits those constantly. + private func firstHex(_ url: URL) -> String? { + guard let line = lines(url).first, + let range = line.range(of: "\"hex\":\"") else { return nil } + return line[range.upperBound...].prefix(while: { $0 != "\"" }).description + } + + func testOneSessionKeepsEveryRecordInTheLiveFile() { + let dump = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + for i in 0..<500 { dump.record(bytes: [0x5D, UInt8(i % 256), 0x01, 0x02]) } + // The whole session is in ONE file, and nothing was rolled aside part-way through. + XCTAssertEqual(lines(live()).count, 500) + XCTAssertFalse(FileManager.default.fileExists(atPath: previous().path)) + } + + func testANewSessionRollsThePreviousOneAsideAndStartsEmpty() { + writeSubstantialSession(tag: 0xAA) + let rolledLineCount = lines(live()).count + XCTAssertGreaterThan(rolledLineCount, 0) + + // A new object is a new session (one is built per OuraLiveSource). + let second = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + second.record(bytes: [0xBB]) + + XCTAssertEqual(lines(live()).count, 1, "the live file is THIS session only") + XCTAssertEqual(lines(previous()).count, rolledLineCount, "the previous session is retained as .1") + XCTAssertTrue(lines(live())[0].contains("bb"), "the live file holds the new session's bytes") + XCTAssertEqual(firstHex(previous())?.prefix(2), "aa") + } + + /// THE 2026-08-10 REGRESSION TEST. Two ordinary post-wake app relaunches must not be able to reach back + /// and destroy the night: with one-deep retention this is exactly what happened. + func testAnOvernightSessionSurvivesTwoRestartsAfterWake() { + writeSubstantialSession(tag: 0x11) // "the night" + writeSubstantialSession(tag: 0x22) // 07:45 relaunch + writeSubstantialSession(tag: 0x33) // 08:17 relaunch + let export = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + export.record(bytes: [0xFF]) // the session the export is taken during + + XCTAssertEqual(firstHex(live()), "ff") + XCTAssertEqual(firstHex(generation(1))?.prefix(2), "33") + XCTAssertEqual(firstHex(generation(2))?.prefix(2), "22") + XCTAssertEqual(firstHex(generation(3))?.prefix(2), "11", "the night is still on disk") + } + + func testTheGenerationRingIsBoundedAtRetainedGenerations() { + for _ in 0..<(OuraRawDump.retainedGenerations + 4) { writeSubstantialSession(tag: 0x55) } + let files = (try? FileManager.default.contentsOfDirectory(atPath: dir.path)) ?? [] + let ours = files.filter { $0.hasPrefix("oura-raw-ring1.jsonl") } + XCTAssertEqual(ours.count, OuraRawDump.retainedGenerations + 1, + "the live file plus exactly \(OuraRawDump.retainedGenerations) generations") + XCTAssertFalse(FileManager.default.fileExists( + atPath: generation(OuraRawDump.retainedGenerations + 1).path)) + } + + /// The rule that makes the ring worth having: a connect that captured essentially nothing is DELETED, + /// not rolled, so a burst of trivial reconnect sessions cannot flush a real capture out of the ring. + func testATrivialSessionIsDiscardedRatherThanSpendingAGeneration() { + writeSubstantialSession(tag: 0xAA) // the capture worth keeping + + for _ in 0..<4 { // four near-empty sessions + let tiny = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + tiny.record(bytes: [0xBB]) + } + let final = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + final.record(bytes: [0xCC]) + + XCTAssertEqual(firstHex(live()), "cc") + XCTAssertEqual(firstHex(previous())?.prefix(2), "aa", + "the real session is still .1 — no trivial session took a slot") + XCTAssertFalse(FileManager.default.fileExists(atPath: generation(2).path)) + } + + func testAnEmptyLeftoverIsReusedRatherThanSpendingARetainedGeneration() { + // Build the state a killed session leaves behind: `.1` holds a REAL session, and the live file was + // created but never written to (0 bytes). Rolling that empty file would push the real capture down + // the ring for nothing. + writeSubstantialSession(tag: 0xAA) + let sessionB = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + sessionB.record(bytes: [0xBB]) // rolls A aside + XCTAssertEqual(firstHex(previous())?.prefix(2), "aa") + FileManager.default.createFile(atPath: live().path, contents: Data()) // B left nothing behind + + let sessionC = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + sessionC.record(bytes: [0xCC]) + + XCTAssertEqual(lines(live()).count, 1) + XCTAssertTrue(lines(live())[0].contains("cc")) + XCTAssertEqual(firstHex(previous())?.prefix(2), "aa", + "the real previous session survived an empty leftover") + } + + func testASessionThatReceivesNothingTouchesNoFilesAtAll() { + let sessionA = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + sessionA.record(bytes: [0xAA]) + // A source that comes up and never gets a record never resolves, so it cannot roll anything. + let silent = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + silent.record(bytes: []) // empty input is a no-op + XCTAssertEqual(lines(live()).count, 1) + XCTAssertTrue(lines(live())[0].contains("aa")) + XCTAssertFalse(FileManager.default.fileExists(atPath: previous().path)) + } + + func testCeilingStopsAppendingAndNeverRollsMidSession() { + let dump = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + // One record is far under the ceiling, so drive the counter with a payload sized to cross it. + let big = [UInt8](repeating: 0xEE, count: 64 * 1024) + for _ in 0..<(OuraRawDump.maxSessionBytes / (128 * 1024) + 8) { dump.record(bytes: big) } + + let size = (try? FileManager.default.attributesOfItem(atPath: live().path))?[.size] as? Int ?? 0 + XCTAssertLessThanOrEqual(size, OuraRawDump.maxSessionBytes, "the ceiling bounds the session") + XCTAssertGreaterThan(size, 0) + // The critical property: hitting the ceiling must NOT roll the file, or we recreate the 2026-08-09 + // bug at a different threshold. + XCTAssertFalse(FileManager.default.fileExists(atPath: previous().path)) + } +} diff --git a/StrandTests/TestBundleAssemblerTests.swift b/StrandTests/TestBundleAssemblerTests.swift index 36cf9e365f..90c97cfe45 100644 --- a/StrandTests/TestBundleAssemblerTests.swift +++ b/StrandTests/TestBundleAssemblerTests.swift @@ -46,6 +46,36 @@ final class TestBundleAssemblerTests: XCTestCase { XCTAssertEqual(raw.data.last, Data(oversized.utf8).last) } + func testCapSnapsTrimmedTailToLineBoundary() { + // A raw byte-count tail can (and in production did) land mid-record: a real export shipped + // oura-cva-ppg.jsonl/oura-real-steps.jsonl/oura-motion.jsonl each with a corrupted first line. + // Every trimmable name is newline-delimited JSONL, so the trimmed tail must always start at a + // clean line boundary. + let small = FileExport.BundleEntry(name: "report.txt", data: Data("small".utf8)) + let lines = (1...2000).map { "{\"n\":\($0),\"pad\":\"\(String(repeating: "x", count: 50))\"}" } + let raw = FileExport.BundleEntry(name: "oura-raw.jsonl", data: Data(lines.joined(separator: "\n").utf8)) + // Force a mid-file trim at a cap unlikely to land exactly on a newline. + let cap = raw.data.count / 2 + let (capped, truncated) = TestBundleAssembler.capEntries([small, raw], capBytes: cap) + XCTAssertTrue(truncated) + let cappedRaw = capped.first { $0.name == "oura-raw.jsonl" }! + let text = String(data: cappedRaw.data, encoding: .utf8)! + XCTAssertFalse(text.isEmpty) + XCTAssertTrue(text.hasPrefix("{"), "trimmed tail must start at a clean line boundary, got: \(text.prefix(30))") + // Every kept line must still be valid JSON (no partial record survived). + for line in text.split(separator: "\n") { + XCTAssertNotNil(try? JSONSerialization.jsonObject(with: Data(line.utf8)), + "corrupted line survived trimming: \(line.prefix(30))") + } + } + + func testTrimToLineBoundaryHandlesNoNewline() { + // A single-line (or already-empty) entry has nothing to snap to - returned unchanged, never worse. + let noNewline = Data("no newline here".utf8) + XCTAssertEqual(TestBundleAssembler.trimToLineBoundary(noNewline), noNewline) + XCTAssertEqual(TestBundleAssembler.trimToLineBoundary(Data()), Data()) + } + func testCapLeavesUndersizedBundleUntouched() { let entries = [FileExport.BundleEntry(name: "report.txt", data: Data("tiny".utf8))] let (capped, truncated) = TestBundleAssembler.capEntries(entries, capBytes: 20 * 1024 * 1024) @@ -56,30 +86,61 @@ final class TestBundleAssemblerTests: XCTestCase { // MARK: - Oura diagnostics attachment (#Test-Centre Oura sidecars) func testNormalizedOuraEntryNameDropsRingId() { - // The three sidecars normalize to id-free names; the ring UUID is gone from the filename. + // All six sidecars normalize to id-free names; the ring UUID is gone from the filename. XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( forFile: "oura-raw-5C4C0BF8-2DF6-1B3A-18D0-3DF0B3590148.jsonl"), "oura-raw.jsonl") XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( forFile: "oura-ibihr-5C4C0BF8-2DF6-1B3A-18D0-3DF0B3590148.jsonl"), "oura-ibihr.jsonl") XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( forFile: "oura-activity-oura-5C4C0BF8.jsonl"), "oura-activity.jsonl") + // #Test-Centre follow-up: cva-ppg/motion/real-steps shipped writers (Strand/BLE/Oura*Dump.swift) + // before the bundler knew their kind — regression coverage for that gap. + XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-cva-ppg-oura-2H3B2405003655.jsonl"), "oura-cva-ppg.jsonl") + XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-motion-oura-2H3B2405003655.jsonl"), "oura-motion.jsonl") + XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-real-steps-oura-2H3B2405003655.jsonl"), "oura-real-steps.jsonl") // Non-sidecar files are ignored. XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName(forFile: "raw-capture.jsonl")) XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName(forFile: "whoop.sqlite")) XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName(forFile: "oura-raw.txt")) } + /// A ROLLED GENERATION must normalize to the same entry name as the live file, so + /// `ouraDiagnosticEntries` can MERGE the generations behind that one name (it used to choose the + /// largest between them, which is why a morning export kept shipping the wrong session). + /// Before this, `hasSuffix(".jsonl")` rejected every `.jsonl.` and a morning export could only ever + /// carry the live file — a 30-second session, while the night sat in a generation nothing looked at + /// (measured 2026-08-10). + func testRolledGenerationsNormalizeToTheSameEntryName() { + for suffix in [".jsonl.1", ".jsonl.2", ".jsonl.8", ".jsonl.12"] { + XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-raw-oura-2H3B2405003655" + suffix), "oura-raw.jsonl", suffix) + } + XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-cva-ppg-oura-2H3B2405003655.jsonl.3"), "oura-cva-ppg.jsonl") + // A non-numeric or empty tail is NOT a generation — those are someone else's files, not ours. + XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-raw-oura-2H3B2405003655.jsonl.bak")) + XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-raw-oura-2H3B2405003655.jsonl.")) + XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-raw-oura-2H3B2405003655.jsonl.1.gz")) + } + func testNormalizedOuraNamesAreAllTrimmable() { // Every normalized sidecar name must be in the cap's trimmable set, else a big night's dump could // blow the 20 MB cap instead of being tail-trimmed. - for kind in ["raw", "ibihr", "activity"] { + for kind in TestBundleAssembler.ouraSidecarKinds { XCTAssertTrue(TestBundleAssembler.trimmableNames.contains("oura-\(kind).jsonl")) + XCTAssertTrue(TestBundleAssembler.ouraSidecarNames.contains("oura-\(kind).jsonl")) } } func testCapSharesBudgetAcrossOuraAndRawCaptureKeepingTails() { - // report.txt kept whole; raw-capture + oura-ibihr together blow the cap → BOTH trimmed to a - // proportional tail, bundle stays under cap, and each keeps its most-recent byte. + // report.txt kept whole; raw-capture + oura-ibihr together blow the cap → BOTH over their fair + // share, so both are trimmed, the bundle stays under cap, and each keeps its most-recent byte. let small = FileExport.BundleEntry(name: "report.txt", data: Data("small".utf8)) let raw = FileExport.BundleEntry(name: "raw-capture.jsonl", data: Data((String(repeating: "r", count: 30 * 1024 * 1024) + "R").utf8)) @@ -96,8 +157,57 @@ final class TestBundleAssemblerTests: XCTestCase { XCTAssertLessThan(cappedIbi.data.count, ibi.data.count) // trimmed XCTAssertEqual(cappedRaw.data.last, raw.data.last) // newest kept XCTAssertEqual(cappedIbi.data.last, ibi.data.last) - // Bigger stream gets the bigger share (raw was 3× ibi). - XCTAssertGreaterThan(cappedRaw.data.count, cappedIbi.data.count) + // Both were over the fair share, so they land on it — within a byte of each other, NOT split 3:1 + // by size the way the old proportional rule would have. + XCTAssertLessThanOrEqual(abs(cappedRaw.data.count - cappedIbi.data.count), 1) + } + + /// The bug this allocation exists to prevent (measured on `noop-master-iOS-v9.3.1-260809-0716.zip`): + /// proportional-to-size handed 53.9 % of the budget to a bulk sidecar dump and left the raw wire capture + /// — the only sidecar a protocol fact can be re-derived from — with 1.9 %, i.e. 8 min of an 8.4 h night. + /// Max-min fair keeps a modest stream WHOLE and takes the bytes off the stream that is merely bulky. + /// (The bulky neighbour is modelled here as the CVA-PPG dump — the measured case was a SpO2 research + /// sidecar that this branch does not carry; the allocation rule is kind-agnostic.) + func testSmallHighValueSidecarSurvivesABulkyNeighbour() { + let report = FileExport.BundleEntry(name: "report.txt", data: Data("small".utf8)) + // Mirrors the real shape: one 40 MB bulk dump beside a 2 MB wire capture, 20 MB of room. + let bulk = FileExport.BundleEntry(name: "oura-cva-ppg.jsonl", + data: Data(String(repeating: "s\n", count: 20 * 1024 * 1024).utf8)) + let raw = FileExport.BundleEntry(name: "oura-raw.jsonl", + data: Data(String(repeating: "r\n", count: 1024 * 1024).utf8)) + let cap = 20 * 1024 * 1024 + let (capped, truncated) = TestBundleAssembler.capEntries([report, bulk, raw], capBytes: cap) + XCTAssertTrue(truncated) + XCTAssertLessThanOrEqual(capped.reduce(0) { $0 + $1.data.count }, cap) + // The wire capture is under its ~10 MB fair share, so it ships WHOLE — not scaled down to ~9 % of + // the budget the way size-proportional splitting would have left it. + XCTAssertEqual(capped.first { $0.name == "oura-raw.jsonl" }?.data.count, raw.data.count) + // Its surplus rolls forward: the bulk dump gets everything the raw capture did not need. + let cappedBulk = capped.first { $0.name == "oura-cva-ppg.jsonl" }! + XCTAssertLessThan(cappedBulk.data.count, bulk.data.count) + XCTAssertGreaterThan(cappedBulk.data.count, cap - raw.data.count - 1024) + } + + func testFairAllowancesIsWaterFillingAndNeverBreachesBudget() { + // Classic water-filling: 1 fits whole, 10 takes its share of what is left, 30 takes the rest. + let a = TestBundleAssembler.fairAllowances( + sizes: [("big", 30), ("small", 1), ("mid", 10)], budget: 20) + XCTAssertEqual(a["small"], 1) // whole, under any share + XCTAssertEqual(a["mid"], 9) // (20-1)/2 + XCTAssertEqual(a["big"], 10) // the remainder + XCTAssertEqual(a.values.reduce(0, +), 20) // budget fully used + // Input order must not change the answer. + let b = TestBundleAssembler.fairAllowances( + sizes: [("small", 1), ("mid", 10), ("big", 30)], budget: 20) + XCTAssertEqual(a, b) + // Everything fits: nobody is trimmed. + XCTAssertEqual(TestBundleAssembler.fairAllowances(sizes: [("x", 3), ("y", 4)], budget: 100), + ["x": 3, "y": 4]) + // Degenerate budgets stay in range rather than going negative. + XCTAssertEqual(TestBundleAssembler.fairAllowances(sizes: [("x", 5)], budget: 0), ["x": 0]) + XCTAssertEqual(TestBundleAssembler.fairAllowances(sizes: [], budget: 10), [:]) + // A single trimmable stream still gets the WHOLE remainder — the original behaviour, unchanged. + XCTAssertEqual(TestBundleAssembler.fairAllowances(sizes: [("only", 999)], budget: 42), ["only": 42]) } /// The redaction contract for the Oura sidecars (PR review, ryanbr): the ONLY PII in a raw line is the @@ -181,4 +291,82 @@ final class TestBundleAssemblerTests: XCTestCase { [FileExport.BundleEntry(name: "report.txt", data: Data(line.utf8))]).first!.data, encoding: .utf8)! XCTAssertTrue(out.contains("\"deviceId\":\"my-whoop\""), "a non-sidecar logical deviceId must be left readable") } + + // MARK: - generation merge (the capture-coverage defect) + + /// END-TO-END over real files on disk, reproducing the layout that broke the export on + /// 2026-09-05: the live file holds the morning drain (small), while a rolled generation holds the + /// previous day's post-reinstall backfill (large). Under the old largest-wins rule the bundle + /// shipped the BACKFILL and the night's own drain never left the device — the shipped + /// `oura-raw.jsonl` spanned `2026-09-04 10:58 → 12:03 UTC` against a night that ended that + /// morning. The merged entry must now contain BOTH, with the morning drain LAST so the cap's + /// keep-the-tail trim can never be the thing that drops it. + func testOuraDiagnosticEntriesMergesGenerationsNewestLast() throws { + let dir = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("oura-gen-merge-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: dir) } + + let morning = "{\"utc\":1788627494,\"iso\":\"2026-09-05T07:08:14Z\",\"hex\":\"5c08\"}\n" + let backfill = String(repeating: "{\"utc\":1788519502,\"iso\":\"2026-09-04T10:58:22Z\"}\n", + count: 50) + try morning.write(to: dir.appendingPathComponent("oura-raw-RING1.jsonl"), + atomically: true, encoding: .utf8) + try backfill.write(to: dir.appendingPathComponent("oura-raw-RING1.jsonl.1"), + atomically: true, encoding: .utf8) + + let entries = TestBundleAssembler.ouraDiagnosticEntries(directory: dir) + XCTAssertEqual(entries.map(\.name), ["oura-raw.jsonl"], "one entry, ring id dropped") + let text = try XCTUnwrap(String(data: entries[0].data, encoding: .utf8)) + XCTAssertTrue(text.contains("2026-09-04T10:58:22Z"), "older generation must still be present") + XCTAssertTrue(text.hasSuffix(morning), "the newest capture must be LAST — the cap keeps the tail") + XCTAssertLessThan(try XCTUnwrap(text.range(of: "2026-09-04T10:58:22Z")).lowerBound, + try XCTUnwrap(text.range(of: "2026-09-05T07:08:14Z")).lowerBound, + "chronological order") + } + + /// A generation whose last write did not end in a newline must not splice its final record onto + /// the next generation's first one — that would produce a line parsing as neither. + func testOuraDiagnosticEntriesInsertsANewlineBetweenGenerations() throws { + let dir = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("oura-gen-nl-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: dir) } + + try "{\"a\":1}".write(to: dir.appendingPathComponent("oura-raw-R.jsonl.1"), // NO trailing newline + atomically: true, encoding: .utf8) + try "{\"b\":2}\n".write(to: dir.appendingPathComponent("oura-raw-R.jsonl"), + atomically: true, encoding: .utf8) + + let entries = TestBundleAssembler.ouraDiagnosticEntries(directory: dir) + let text = try XCTUnwrap(String(data: entries[0].data, encoding: .utf8)) + XCTAssertEqual(text, "{\"a\":1}\n{\"b\":2}\n") + XCTAssertEqual(text.split(separator: "\n").count, 2, "two whole lines, nothing spliced") + } + + /// Every merged line must survive the redact + cap pass the bundle actually applies, and the + /// newest record must be the one that survives a cap far smaller than the merged size. + func testMergedSidecarKeepsTheNewestRecordsUnderTheCap() throws { + let dir = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("oura-gen-cap-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: dir) } + + try String(repeating: "{\"old\":1}\n", count: 4000) + .write(to: dir.appendingPathComponent("oura-raw-R.jsonl.1"), atomically: true, encoding: .utf8) + try String(repeating: "{\"new\":1}\n", count: 100) + .write(to: dir.appendingPathComponent("oura-raw-R.jsonl"), atomically: true, encoding: .utf8) + + let merged = TestBundleAssembler.ouraDiagnosticEntries(directory: dir) + let (capped, truncated) = TestBundleAssembler.capEntries(merged, capBytes: 2_000) + XCTAssertTrue(truncated) + let text = try XCTUnwrap(String(data: capped[0].data, encoding: .utf8)) + XCTAssertTrue(text.contains("{\"new\":1}"), "the newest records must survive the cap") + // Every kept line must be whole — the front trim snaps to a line boundary, so the partial + // record the raw byte-count tail would otherwise start on is dropped. + for line in text.split(separator: "\n") { + XCTAssertTrue(line.hasPrefix("{") && line.hasSuffix("}"), "no partial line: \(line)") + } + } + }