From 019968786392d2d3577ec8f141e03b11b15da4ef Mon Sep 17 00:00:00 2001 From: Pipiche Date: Thu, 30 Jul 2026 21:12:28 +0200 Subject: [PATCH 1/7] fix(oura): stop the Test Centre export dropping the cva-ppg/motion/real-steps sidecars TestBundleAssembler.normalizedOuraEntryName() hardcoded the Oura Diagnostics-dir sidecar kinds it recognizes to the original three (raw/ibihr/activity). The cva-ppg, motion, and real-steps dump writers shipped afterward and were writing real files on-device the whole time, but the export bundler silently dropped all three from every Test Centre bundle since it didn't know their kind - confirmed against two real exports where the strap log showed the writers firing but the files never made it into the bundle. Collapsed the 4 places that independently listed sidecar kinds down to one source of truth (ouraSidecarKinds), so a future dump writer needs exactly one line added to be picked up by normalizedOuraEntryName/ouraDiagnosticEntries/ trimmableNames/ouraSidecarNames together instead of silently missing export. (The original commit also carried a hardcoded test-branch build marker for the strap-log header, explicitly flagged "must never ship". It is dropped here along with its two call sites, and the follow-up commit that existed only to remove it is dropped with it.) Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013EHFxGQzockMYJqDXYB3S1 --- Strand/System/TestBundleAssembler.swift | 35 ++++++++++++---------- StrandTests/TestBundleAssemblerTests.swift | 13 ++++++-- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/Strand/System/TestBundleAssembler.swift b/Strand/System/TestBundleAssembler.swift index d1a949cf00..ce2e7a7155 100644 --- a/Strand/System/TestBundleAssembler.swift +++ b/Strand/System/TestBundleAssembler.swift @@ -14,14 +14,20 @@ enum TestBundleAssembler { /// The redaction stamp written into meta.json so a maintainer knows the whole-bundle scrub ran. static let redactionVersion = "v2" + /// The Oura Diagnostics-dir sidecar kinds the bundle attaches — the SINGLE source of truth + /// `normalizedOuraEntryName`/`ouraDiagnosticEntries`/`trimmableNames`/`ouraSidecarNames` all derive + /// from, so a new dump writer (`Strand/BLE/Oura*Dump.swift`) needs exactly one line added here to be + /// picked up everywhere instead of silently missing the export bundle. (Landed after `cva-ppg` and + /// `real-steps` shipped their writers but not their kind entry — both wrote real files on-device that + /// the bundler's hardcoded 3-kind list then dropped from every export.) + static let ouraSidecarKinds: [String] = ["raw", "ibihr", "activity", "cva-ppg", "motion", "real-steps"] + /// The bundle files that may be trimmed to fit the cap (newest-tail kept). The strap-log tail and /// meta.json are already bounded, so only these raw research streams can blow the budget: the WHOOP - /// frame capture plus the Oura ring's Tier-B JSONL sidecars (raw notifications / IBI-HR / activity MET). - /// Everything NOT listed here is kept whole and its bytes are reserved before the trimmable group is - /// given the remainder. These are the NORMALIZED entry names (the ring id is dropped from the filename). - static let trimmableNames: Set = [ - "raw-capture.jsonl", "oura-raw.jsonl", "oura-ibihr.jsonl", "oura-activity.jsonl", - ] + /// frame capture plus the Oura ring's Tier-B JSONL sidecars (raw notifications / IBI-HR / activity MET / + /// CVA-PPG / motion / real-steps). Everything NOT listed here is kept whole and its bytes are reserved + /// before the trimmable group is given the remainder. NORMALIZED entry names (ring id dropped). + static let trimmableNames: Set = Set(["raw-capture.jsonl"] + ouraSidecarKinds.map { "oura-\($0).jsonl" }) /// #572 follow-up: the Oura Tier-B sidecars carry the ring id in a `"deviceId"` JSON field. The /// whole-bundle UUID scrub (`LiveState.redactPii`) masks it only when it is a CANONICAL dashed @@ -30,9 +36,7 @@ enum TestBundleAssembler { /// covered, and (unlike broadening the UUID regex to dashless hex) it CANNOT touch the raw `hex` capture /// field, which a greedier rule would shred. Scoped to the sidecars so a non-PII logical `deviceId` /// (e.g. "my-whoop") elsewhere in the bundle stays readable. NORMALIZED names (ring id already dropped). - static let ouraSidecarNames: Set = [ - "oura-raw.jsonl", "oura-ibihr.jsonl", "oura-activity.jsonl", - ] + static let ouraSidecarNames: Set = Set(ouraSidecarKinds.map { "oura-\($0).jsonl" }) /// `oura-raw.jsonl` is the ONE sidecar whose payload is nothing but hex (`OuraRawDumpLine.encode`) — /// `oura-ibihr.jsonl`/`oura-activity.jsonl` encode DECODED numeric fields, so `LiveState.redactHexDump`'s @@ -320,11 +324,12 @@ enum TestBundleAssembler { } /// Map a diagnostics filename to a NORMALIZED bundle entry name that drops the ring id, or nil when the - /// file is not one of the three Oura sidecars. `oura--.jsonl` → `oura-.jsonl`, which - /// (a) keeps the ring id out of the bundle's filenames (redaction scrubs content, never names) and - /// (b) lands on the `trimmableNames` set so the cap can trim it. Pure/string-only for unit testing. + /// file is not one of the Oura sidecars in `ouraSidecarKinds`. `oura--.jsonl` → + /// `oura-.jsonl`, which (a) keeps the ring id out of the bundle's filenames (redaction scrubs + /// content, never names) and (b) lands on the `trimmableNames` set so the cap can trim it. + /// Pure/string-only for unit testing. static func normalizedOuraEntryName(forFile filename: String) -> String? { - for kind in ["raw", "ibihr", "activity"] { + for kind in ouraSidecarKinds { if filename.hasPrefix("oura-\(kind)-"), filename.hasSuffix(".jsonl") { return "oura-\(kind).jsonl" } @@ -348,8 +353,8 @@ enum TestBundleAssembler { if let existing = byName[name], existing.data.count >= entry.data.count { continue } byName[name] = entry } - // Stable order (raw, ibihr, activity) so the bundle listing is deterministic. - return ["oura-raw.jsonl", "oura-ibihr.jsonl", "oura-activity.jsonl"].compactMap { byName[$0] } + // Stable order (ouraSidecarKinds order) so the bundle listing is deterministic. + return ouraSidecarKinds.map { "oura-\($0).jsonl" }.compactMap { byName[$0] } } /// The on-disk path a crash report would live at, when a crash handler is wired. There is no producer diff --git a/StrandTests/TestBundleAssemblerTests.swift b/StrandTests/TestBundleAssemblerTests.swift index 36cf9e365f..7395a933e0 100644 --- a/StrandTests/TestBundleAssemblerTests.swift +++ b/StrandTests/TestBundleAssemblerTests.swift @@ -56,13 +56,21 @@ final class TestBundleAssemblerTests: XCTestCase { // MARK: - Oura diagnostics attachment (#Test-Centre Oura sidecars) func testNormalizedOuraEntryNameDropsRingId() { - // The three sidecars normalize to id-free names; the ring UUID is gone from the filename. + // All six sidecars normalize to id-free names; the ring UUID is gone from the filename. XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( forFile: "oura-raw-5C4C0BF8-2DF6-1B3A-18D0-3DF0B3590148.jsonl"), "oura-raw.jsonl") XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( forFile: "oura-ibihr-5C4C0BF8-2DF6-1B3A-18D0-3DF0B3590148.jsonl"), "oura-ibihr.jsonl") XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( forFile: "oura-activity-oura-5C4C0BF8.jsonl"), "oura-activity.jsonl") + // #Test-Centre follow-up: cva-ppg/motion/real-steps shipped writers (Strand/BLE/Oura*Dump.swift) + // before the bundler knew their kind — regression coverage for that gap. + XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-cva-ppg-oura-2H3B2405003655.jsonl"), "oura-cva-ppg.jsonl") + XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-motion-oura-2H3B2405003655.jsonl"), "oura-motion.jsonl") + XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-real-steps-oura-2H3B2405003655.jsonl"), "oura-real-steps.jsonl") // Non-sidecar files are ignored. XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName(forFile: "raw-capture.jsonl")) XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName(forFile: "whoop.sqlite")) @@ -72,8 +80,9 @@ final class TestBundleAssemblerTests: XCTestCase { func testNormalizedOuraNamesAreAllTrimmable() { // Every normalized sidecar name must be in the cap's trimmable set, else a big night's dump could // blow the 20 MB cap instead of being tail-trimmed. - for kind in ["raw", "ibihr", "activity"] { + for kind in TestBundleAssembler.ouraSidecarKinds { XCTAssertTrue(TestBundleAssembler.trimmableNames.contains("oura-\(kind).jsonl")) + XCTAssertTrue(TestBundleAssembler.ouraSidecarNames.contains("oura-\(kind).jsonl")) } } From ed8b898d6667892f80aca2cc1b6b7a60f53f1656 Mon Sep 17 00:00:00 2001 From: Pipiche Date: Fri, 31 Jul 2026 12:19:08 +0200 Subject: [PATCH 2/7] fix(oura): snap Test Centre export trimming to a JSONL line boundary TestBundleAssembler.capEntries trimmed oversized bundle files to a raw byte count (keeping the most-recent tail), which is safe for a single freeform text file but not for the newline-delimited JSONL sidecars: a byte-count cut can land mid-record. Confirmed in production - a real overnight export (2026-07-30/31) shipped oura-cva-ppg.jsonl, oura-real-steps.jsonl, and oura-motion.jsonl each with a corrupted first line once the sidecars' combined size crossed the 20 MB cap for the first time. Adds trimToLineBoundary: after the proportional byte-count trim, drop bytes up to and including the first newline, so every kept line is a complete record. Every trimmable name is newline-delimited JSONL, so this is safe across the board; a single very-long line (no newline found) is returned unchanged, never worse than before. Tested: new testCapSnapsTrimmedTailToLineBoundary (every kept line parses as valid JSON after a forced mid-file trim) and testTrimToLineBoundaryHandlesNoNewline; existing cap tests unaffected (their fixtures have no embedded newlines, so the snap is a no-op for them). macOS Strand target builds; TestBundleAssemblerTests 13/13 pass. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_013EHFxGQzockMYJqDXYB3S1 --- Strand/System/TestBundleAssembler.swift | 19 ++++++++++++-- StrandTests/TestBundleAssemblerTests.swift | 30 ++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/Strand/System/TestBundleAssembler.swift b/Strand/System/TestBundleAssembler.swift index ce2e7a7155..fa0d68ef38 100644 --- a/Strand/System/TestBundleAssembler.swift +++ b/Strand/System/TestBundleAssembler.swift @@ -103,12 +103,27 @@ enum TestBundleAssembler { : 0 guard entry.data.count > share else { return entry } truncated = true - // Keep the tail (most recent): the last `share` bytes. - return FileExport.BundleEntry(name: entry.name, data: Data(entry.data.suffix(share))) + // Keep the tail (most recent): the last `share` bytes, then snap forward to the next full + // JSONL line. A raw byte-count tail can (and in production did - a real export shipped + // oura-cva-ppg.jsonl/oura-real-steps.jsonl/oura-motion.jsonl each with a corrupted first + // line) land mid-record; every trimmable name is newline-delimited JSONL, so dropping the + // partial line at the front keeps every kept line honest. + return FileExport.BundleEntry(name: entry.name, + data: Self.trimToLineBoundary(entry.data.suffix(share))) } return (capped, truncated) } + /// Drop bytes up to and including the first newline in `data`, so a raw byte-count tail (from + /// `capEntries`'s proportional trim) never starts mid-line for a newline-delimited JSONL sidecar. + /// Returns `data` unchanged when it contains no newline (a single very-long line, or already empty) + /// - never worse than the untrimmed behaviour, just not improved. Pure/testable. + static func trimToLineBoundary(_ data: Data) -> Data { + guard let newline = data.firstIndex(of: 0x0A) else { return data } + let start = data.index(after: newline) + return data.subdata(in: start.. Date: Sun, 9 Aug 2026 15:15:10 +0200 Subject: [PATCH 3/7] fix(diagnostics): share the export cap max-min fairly, not in proportion to size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `TestBundleAssembler.capEntries` held the bundle to 20 MB by splitting the budget across `trimmableNames` IN PROPORTION to each file's size. That hands the budget to the bulkiest stream, and for the Oura sidecars bulk is close to inversely correlated with diagnostic value. Measured on a real export (`noop-master-iOS-v9.3.1-260809-0716.zip`, six trimmable sidecars over a 20,863,648-byte budget): oura-spo2.jsonl 11,242,865 53.9 % oura-real-steps.jsonl 3,782,590 18.1 % oura-motion.jsonl 2,827,017 13.5 % oura-cva-ppg.jsonl 2,417,008 11.6 % oura-raw.jsonl 388,064 1.9 % oura-activity.jsonl 206,921 1.0 % Trimmable total 20,864,465 against the computed budget of 20,863,648 — agreeing to 817 bytes of integer-floor rounding, so this is the cap firing exactly as written. `oura-raw.jsonl` is the UNDECODED wire bytes: the one file in the bundle from which a protocol fact can be re-derived independently of NOOP's own decoder, which is the entire reason `OuraRawDump` exists. It shipped 3,123 records — 8 min 43 s of an 8.4 h night whose full history drain had just completed. The verifiers that read the raw sidecar ON PURPOSE (so a decode bug cannot launder itself into the number checking it) could not run at all that night, while 11 MB of SpO2 samples, also present in the SQLite the same bundle ships, rode along whole. Replaced with max-min fair (water-filling) allocation: walk the streams smallest-first offering each an equal share of what is left, so a stream under its share is kept WHOLE and its surplus rolls forward to the ones still over. It makes no editorial claim about which sidecar matters — the small file wins because it is small, not because it is favoured. Integer floor keeps the sum at or under budget, so the cap can never breach. Because each shipped size IS its old proportional share, the on-disk sizes are the shipped sizes times one unknown scale k, and the outcome follows exactly: with six streams the fair share is 3.48 MB, so `oura-raw.jsonl` ships WHOLE for any k up to ~9 (a sidecar directory under ~190 MB) and still gets 9x its current bytes beyond that. A single trimmable stream still receives the whole remainder, so the original raw-capture-only behaviour is unchanged. `fairAllowances` is pure and separately tested. One existing test pinned the old rule (bigger stream gets the bigger share) and now pins the new one; two tests are added — the water-filling contract including determinism and degenerate budgets, and the real bug shape, a modest wire capture beside a bulky neighbour. Verification: StrandTests 1083/0 (1 skipped) including all 15 TestBundleAssembler tests; `Strand` macOS and `NOOPiOS` iOS both built locally, which no default CI covers. Apple-only — `TestBundleAssembler` has no Android twin, so there is no parity half. Diagnostics only: no BLE writes, no schema change, no analytics, nothing user-facing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01E7i8DiChrWDDbppKpywaSA --- Strand/System/TestBundleAssembler.swift | 57 +++++++++++++++++----- StrandTests/TestBundleAssemblerTests.swift | 55 +++++++++++++++++++-- 2 files changed, 96 insertions(+), 16 deletions(-) diff --git a/Strand/System/TestBundleAssembler.swift b/Strand/System/TestBundleAssembler.swift index fa0d68ef38..d8f88d5c3f 100644 --- a/Strand/System/TestBundleAssembler.swift +++ b/Strand/System/TestBundleAssembler.swift @@ -80,11 +80,16 @@ enum TestBundleAssembler { /// Hard cap the bundle at `capBytes` (20 MB default, under GitHub's 25 MB; spec section 5.4). The /// strap-log tail and meta.json are already bounded, so only the `trimmableNames` research streams can - /// exceed. We reserve the whole size of every non-trimmable file, then split the remaining budget across - /// the trimmable files IN PROPORTION to their size, keeping the MOST-RECENT tail of each (newest data is - /// the most diagnostic) and trimming from the front. With a single trimmable file this is identical to - /// the prior raw-capture-only behaviour (its share is the whole remainder). Returns the capped entries - /// plus whether any truncation happened, which the caller writes to meta.truncated. + /// exceed. We reserve the whole size of every non-trimmable file, then share the remaining budget across + /// the trimmable files, keeping the MOST-RECENT tail of each (newest data is the most diagnostic) and + /// trimming from the front. Returns the capped entries plus whether any truncation happened, which the + /// caller writes to meta.truncated. + /// + /// ALLOCATION: **max-min fair** (water-filling), NOT proportional-to-size. Each stream is offered an + /// equal share of what is left; a stream smaller than its share is kept WHOLE and its unused remainder + /// flows to the streams that are still over. See `fairAllowances` for why, and for the measured export + /// that motivated it. With a single trimmable file this is still identical to the original + /// raw-capture-only behaviour: its share is the whole remainder. static func capEntries(_ entries: [FileExport.BundleEntry], capBytes: Int = 20 * 1024 * 1024) -> (entries: [FileExport.BundleEntry], truncated: Bool) { let total = entries.reduce(0) { $0 + $1.data.count } @@ -92,15 +97,13 @@ enum TestBundleAssembler { // Reserve the non-trimmable files (kept whole), then share the remainder across the trimmable ones. let nonTrimmable = entries.filter { !trimmableNames.contains($0.name) }.reduce(0) { $0 + $1.data.count } let budget = max(0, capBytes - nonTrimmable) - let trimmableTotal = entries.filter { trimmableNames.contains($0.name) }.reduce(0) { $0 + $1.data.count } + let allowance = fairAllowances( + sizes: entries.filter { trimmableNames.contains($0.name) }.map { ($0.name, $0.data.count) }, + budget: budget) var truncated = false let capped = entries.map { entry -> FileExport.BundleEntry in guard trimmableNames.contains(entry.name) else { return entry } - // Each trimmable file gets a byte share proportional to its size (integer floor keeps the sum - // under budget, so the bundle never breaches the cap). Kept whole if already within its share. - let share = trimmableTotal > 0 - ? Int(Double(budget) * Double(entry.data.count) / Double(trimmableTotal)) - : 0 + let share = allowance[entry.name] ?? 0 guard entry.data.count > share else { return entry } truncated = true // Keep the tail (most recent): the last `share` bytes, then snap forward to the next full @@ -114,8 +117,38 @@ enum TestBundleAssembler { return (capped, truncated) } + /// Split `budget` bytes across the named streams **max-min fairly**: walk them smallest-first, offering + /// each an equal share of whatever budget remains, and let a stream that fits under its share take only + /// what it needs so the surplus rolls forward to the larger ones. Integer floor keeps the sum at or under + /// `budget`, so the bundle can never breach the cap. Pure; `sizes` order does not affect the result. + /// + /// WHY NOT PROPORTIONAL-TO-SIZE (the original rule): it hands the budget to the BULKIEST stream, and for + /// these sidecars bulk is close to inversely correlated with diagnostic value. Measured on a real export + /// (`noop-master-iOS-v9.3.1-260809-0716.zip`, six trimmable sidecars over a 20,863,648-byte budget): + /// `oura-spo2.jsonl` — a per-sample dump whose values are also in the SQLite the same bundle ships — + /// took **53.9 %**, while `oura-raw.jsonl` got **1.9 %** (388,064 bytes: 8 min 43 s of an 8.4 h night). + /// The raw sidecar is the UNDECODED wire bytes, the one file in the bundle from which a protocol fact can + /// be re-derived independently of our own decoder, so starving it costs a night of verification that + /// nothing else can supply. Under this rule the ~3.47 MB fair share keeps it whole and the surplus goes + /// where it is merely bulk. Same 20 MB bundle, no editorial ranking of the streams. + static func fairAllowances(sizes: [(name: String, bytes: Int)], budget: Int) -> [String: Int] { + // Smallest-first, name-tiebroken so the result is deterministic regardless of input order. + let ordered = sizes.sorted { $0.bytes != $1.bytes ? $0.bytes < $1.bytes : $0.name < $1.name } + var allowance: [String: Int] = [:] + var left = max(0, budget) + var unserved = ordered.count + for stream in ordered { + let share = unserved > 0 ? left / unserved : 0 + let give = min(max(0, stream.bytes), share) + allowance[stream.name] = give + left -= give + unserved -= 1 + } + return allowance + } + /// Drop bytes up to and including the first newline in `data`, so a raw byte-count tail (from - /// `capEntries`'s proportional trim) never starts mid-line for a newline-delimited JSONL sidecar. + /// `capEntries`'s fair-share trim) never starts mid-line for a newline-delimited JSONL sidecar. /// Returns `data` unchanged when it contains no newline (a single very-long line, or already empty) /// - never worse than the untrimmed behaviour, just not improved. Pure/testable. static func trimToLineBoundary(_ data: Data) -> Data { diff --git a/StrandTests/TestBundleAssemblerTests.swift b/StrandTests/TestBundleAssemblerTests.swift index b979945cfa..c179217921 100644 --- a/StrandTests/TestBundleAssemblerTests.swift +++ b/StrandTests/TestBundleAssemblerTests.swift @@ -117,8 +117,8 @@ final class TestBundleAssemblerTests: XCTestCase { } func testCapSharesBudgetAcrossOuraAndRawCaptureKeepingTails() { - // report.txt kept whole; raw-capture + oura-ibihr together blow the cap → BOTH trimmed to a - // proportional tail, bundle stays under cap, and each keeps its most-recent byte. + // report.txt kept whole; raw-capture + oura-ibihr together blow the cap → BOTH over their fair + // share, so both are trimmed, the bundle stays under cap, and each keeps its most-recent byte. let small = FileExport.BundleEntry(name: "report.txt", data: Data("small".utf8)) let raw = FileExport.BundleEntry(name: "raw-capture.jsonl", data: Data((String(repeating: "r", count: 30 * 1024 * 1024) + "R").utf8)) @@ -135,8 +135,55 @@ final class TestBundleAssemblerTests: XCTestCase { XCTAssertLessThan(cappedIbi.data.count, ibi.data.count) // trimmed XCTAssertEqual(cappedRaw.data.last, raw.data.last) // newest kept XCTAssertEqual(cappedIbi.data.last, ibi.data.last) - // Bigger stream gets the bigger share (raw was 3× ibi). - XCTAssertGreaterThan(cappedRaw.data.count, cappedIbi.data.count) + // Both were over the fair share, so they land on it — within a byte of each other, NOT split 3:1 + // by size the way the old proportional rule would have. + XCTAssertLessThanOrEqual(abs(cappedRaw.data.count - cappedIbi.data.count), 1) + } + + /// The bug this allocation exists to prevent (measured on `noop-master-iOS-v9.3.1-260809-0716.zip`): + /// proportional-to-size handed 53.9 % of the budget to the bulk SpO2 dump and left the raw wire capture + /// — the only sidecar a protocol fact can be re-derived from — with 1.9 %, i.e. 8 min of an 8.4 h night. + /// Max-min fair keeps a modest stream WHOLE and takes the bytes off the stream that is merely bulky. + func testSmallHighValueSidecarSurvivesABulkyNeighbour() { + let report = FileExport.BundleEntry(name: "report.txt", data: Data("small".utf8)) + // Mirrors the real shape: one 40 MB bulk dump beside a 2 MB wire capture, 20 MB of room. + let spo2 = FileExport.BundleEntry(name: "oura-spo2.jsonl", + data: Data(String(repeating: "s\n", count: 20 * 1024 * 1024).utf8)) + let raw = FileExport.BundleEntry(name: "oura-raw.jsonl", + data: Data(String(repeating: "r\n", count: 1024 * 1024).utf8)) + let cap = 20 * 1024 * 1024 + let (capped, truncated) = TestBundleAssembler.capEntries([report, spo2, raw], capBytes: cap) + XCTAssertTrue(truncated) + XCTAssertLessThanOrEqual(capped.reduce(0) { $0 + $1.data.count }, cap) + // The wire capture is under its ~10 MB fair share, so it ships WHOLE — not scaled down to ~9 % of + // the budget the way size-proportional splitting would have left it. + XCTAssertEqual(capped.first { $0.name == "oura-raw.jsonl" }?.data.count, raw.data.count) + // Its surplus rolls forward: the bulk dump gets everything the raw capture did not need. + let cappedSpo2 = capped.first { $0.name == "oura-spo2.jsonl" }! + XCTAssertLessThan(cappedSpo2.data.count, spo2.data.count) + XCTAssertGreaterThan(cappedSpo2.data.count, cap - raw.data.count - 1024) + } + + func testFairAllowancesIsWaterFillingAndNeverBreachesBudget() { + // Classic water-filling: 1 fits whole, 10 takes its share of what is left, 30 takes the rest. + let a = TestBundleAssembler.fairAllowances( + sizes: [("big", 30), ("small", 1), ("mid", 10)], budget: 20) + XCTAssertEqual(a["small"], 1) // whole, under any share + XCTAssertEqual(a["mid"], 9) // (20-1)/2 + XCTAssertEqual(a["big"], 10) // the remainder + XCTAssertEqual(a.values.reduce(0, +), 20) // budget fully used + // Input order must not change the answer. + let b = TestBundleAssembler.fairAllowances( + sizes: [("small", 1), ("mid", 10), ("big", 30)], budget: 20) + XCTAssertEqual(a, b) + // Everything fits: nobody is trimmed. + XCTAssertEqual(TestBundleAssembler.fairAllowances(sizes: [("x", 3), ("y", 4)], budget: 100), + ["x": 3, "y": 4]) + // Degenerate budgets stay in range rather than going negative. + XCTAssertEqual(TestBundleAssembler.fairAllowances(sizes: [("x", 5)], budget: 0), ["x": 0]) + XCTAssertEqual(TestBundleAssembler.fairAllowances(sizes: [], budget: 10), [:]) + // A single trimmable stream still gets the WHOLE remainder — the original behaviour, unchanged. + XCTAssertEqual(TestBundleAssembler.fairAllowances(sizes: [("only", 999)], budget: 42), ["only": 42]) } /// The redaction contract for the Oura sidecars (PR review, ryanbr): the ONLY PII in a raw line is the From 88626db0e921e1f909b6eca48dadf2cba1ffc626 Mon Sep 17 00:00:00 2001 From: Pipiche Date: Sun, 9 Aug 2026 15:40:31 +0200 Subject: [PATCH 4/7] fix(diagnostics): scope the Oura raw sidecar to ONE capture session, never roll mid-drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `OuraRawDump` rolled its sidecar to a `.1` sibling whenever the file passed 25 MB, checked per record. A size threshold can fire MID-DRAIN, and on 2026-08-09 it did: it rolled at 07:06:58 during the morning offload, carrying 32,005 records of that drain and 71 of the night's 75 0x5D records into the `.1` — which the strap-log bundle does not collect. The bundle shipped 3,123 records covering 8 min 43 s of an 8.4 h night, and the verifiers that read the raw sidecar ON PURPOSE (so a decode bug cannot launder itself into the number checking it) could not run. The evidence was recoverable only by pulling the file off the phone by hand. The live file now holds exactly ONE session. It is rolled aside when the object resolves its file for the first time — the START of a session — and never again while that session runs. One OuraRawDump is built per OuraLiveSource, and a source survives BLE reconnects (SourceCoordinator returns early when the active strap is unchanged), so a drain that disconnects and re-engages several times still lands whole in one file: the five reconnects of that same morning would have been a single session. Lowering the threshold instead would make the failure MORE frequent, not less. Only a session boundary removes it, and the corpus stays bounded anyway because a session is bounded: an overnight drain is ~32,000 records, about 4 MB. The multi-day corpus is deliberately given up, because the bundle replaces it — every capture now carries its own COMPLETE raw sidecar, and bundles are what get archived. One previous session is kept as `.1` so a session that ends without an export is not lost. The per-record `stat` goes with it: the file starts empty each session, so a running byte counter is authoritative and costs no filesystem call. That counter enforces a 25 MB SESSION ceiling which STOPS appending and logs once rather than rolling — rolling mid-session is the exact failure being removed, so it must not reappear at a new threshold. A real drain is nowhere near it; it exists so a pathological backlog cannot fill the disk. `directory` is injectable (defaulting to Application Support) purely so the roll is testable: this class had no coverage at all, because it hardcoded its path. Six tests now pin the session contract — a whole session in one file with nothing rolled aside, a new session rolling the previous one aside and starting empty, exactly two generations retained, an empty leftover NOT spending the retained slot, a silent session touching nothing, and the ceiling bounding a session WITHOUT rolling it. Verification: StrandTests 1089/0 (1 skipped); `Strand` macOS and `NOOPiOS` iOS both built locally, which no default CI covers. Apple-only — OuraRawDump has no Android twin, so there is no parity half. Diagnostics only: no BLE writes, no schema change, no analytics, nothing user-facing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01E7i8DiChrWDDbppKpywaSA --- Strand/BLE/OuraRawDump.swift | 91 ++++++++++++++----- StrandTests/OuraRawDumpSessionTests.swift | 106 ++++++++++++++++++++++ 2 files changed, 173 insertions(+), 24 deletions(-) create mode 100644 StrandTests/OuraRawDumpSessionTests.swift diff --git a/Strand/BLE/OuraRawDump.swift b/Strand/BLE/OuraRawDump.swift index 335b515b01..2396708bd9 100644 --- a/Strand/BLE/OuraRawDump.swift +++ b/Strand/BLE/OuraRawDump.swift @@ -18,18 +18,40 @@ import WhoopStore /// decoded sidecars there is NO dedup high-water: a re-served record is still evidence the ring re-sent it, /// and the offline reframer collapses duplicates by (tag, ring-time). /// +/// LIFETIME: the live file holds exactly ONE capture session. It is rolled to a single `.1` sibling when +/// this object resolves its file for the first time — i.e. at the START of a session — and never again +/// while the session runs. One `OuraRawDump` is built per `OuraLiveSource`, and a source survives BLE +/// reconnects (`SourceCoordinator` returns early when the active strap is unchanged), so a drain that +/// disconnects and re-engages several times still lands whole in one file. +/// +/// WHY NOT a size threshold, which is what this used to do: a threshold can fire MID-DRAIN, and on +/// 2026-08-09 it did — 25 MB rolled at 07:06:58 during the morning offload, carrying 32,005 records of +/// that drain and 71 of the night's 75 `0x5D` records into the `.1`, which the strap-log bundle does not +/// collect. The night's raw evidence never left the device (it was recoverable only by pulling the file +/// off the phone by hand). Lowering the threshold makes that MORE frequent, not less; only rolling on a +/// session boundary removes it. The corpus stays bounded because a session is bounded: an overnight +/// drain is ~32,000 records ≈ 4 MB. +/// +/// The multi-day corpus this used to accumulate is deliberately given up, because the bundle replaces it: +/// every capture now carries its own COMPLETE raw sidecar, and the bundles are what get archived. One +/// previous session is kept as `.1` purely so a session that ends without an export is not lost. +/// /// Location: `/OpenWhoop/Diagnostics/oura-raw-.jsonl` — beside the SQLite. final class OuraRawDump { private let deviceId: String private let log: (String) -> Void + private let directory: URL? private var fileURL: URL? private var resolveFailed = false private var announced = false + private var sessionBytes = 0 + private var ceilingHit = false - /// Rotate the sidecar past this size (keeping one previous ".1"), so an always-on research corpus is - /// bounded to ~2× this on disk instead of growing forever. Matches `OuraMotionDump`/`OuraActivityDump` — - /// this file needs it MORE than either: full hex of every notification, and no dedup high-water. - private static let maxBytes = 25 * 1024 * 1024 + /// Hard ceiling on ONE session's file. Never a rotation — a session that somehow reaches this stops + /// appending and says so once, because rolling mid-session is the exact failure this class was changed + /// to remove. Far above a real overnight drain (~4 MB), so in practice it never fires; it exists so a + /// pathological backlog cannot fill the disk. + static let maxSessionBytes = 25 * 1024 * 1024 private static let iso: ISO8601DateFormatter = { let f = ISO8601DateFormatter() @@ -37,28 +59,18 @@ final class OuraRawDump { return f }() - init(deviceId: String, log: @escaping (String) -> Void) { + /// `directory` is injectable so the session-roll behaviour is testable against a temp dir; production + /// passes nil and gets `/OpenWhoop/Diagnostics`. + init(deviceId: String, log: @escaping (String) -> Void, directory: URL? = nil) { self.deviceId = deviceId self.log = log + self.directory = directory } /// Append one raw notification's bytes verbatim (hex-encoded), stamped with wall-clock arrival time. /// No-op on empty input. Best-effort: any file error is logged once and never disrupts the BLE path. func record(bytes: [UInt8]) { - guard !bytes.isEmpty, var url = resolveURL() else { return } - - // Bound the corpus (rotate to a single ".1", dropping the prior one) so an always-on research sidecar - // can't grow unbounded — same rotation as OuraMotionDump. Read the size via a fresh FileManager stat - // rather than URL.resourceValues, whose cache on the reused URL can return a stale small size. - let size = ((try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? Int) ?? 0 - if size > Self.maxBytes { - let old = url.deletingLastPathComponent().appendingPathComponent(url.lastPathComponent + ".1") - try? FileManager.default.removeItem(at: old) - try? FileManager.default.moveItem(at: url, to: old) - fileURL = nil - guard let fresh = resolveURL() else { return } - url = fresh - } + guard !bytes.isEmpty, let url = resolveURL() else { return } let now = Date() let line = OuraRawDumpLine.encode( @@ -66,11 +78,24 @@ final class OuraRawDump { iso: Self.iso.string(from: now), bytes: bytes) guard let data = (line + "\n").data(using: .utf8) else { return } + + // Ceiling check against a running counter, not a stat per record: the file starts EMPTY each + // session (resolveURL rolls it), so the counter is authoritative and costs no filesystem call. + guard sessionBytes + data.count <= Self.maxSessionBytes else { + if !ceilingHit { + ceilingHit = true + log("Oura: raw capture reached its \(Self.maxSessionBytes / (1024 * 1024)) MB session " + + "ceiling - no longer appending (the file is NOT rolled mid-session)") + } + return + } + do { let handle = try FileHandle(forWritingTo: url) defer { try? handle.close() } try handle.seekToEnd() handle.write(data) + sessionBytes += data.count } catch { log("Oura: raw dump write failed - \(error.localizedDescription)") return @@ -82,21 +107,39 @@ final class OuraRawDump { } } - /// Resolve (and create on first use) the sidecar file + its parent directory. Cached; a failure is - /// logged once and latched so we never spam the strap log on a read-only volume. + /// Resolve the sidecar file + its parent directory, ROLLING the previous session's file aside on the + /// first call so this session starts from empty. Cached, so the roll happens exactly once per object — + /// which is what makes it a session boundary rather than a size threshold. A failure is logged once and + /// latched so we never spam the strap log on a read-only volume. private func resolveURL() -> URL? { if let fileURL { return fileURL } if resolveFailed { return nil } do { - let base = try FileManager.default.url(for: .applicationSupportDirectory, in: .userDomainMask, - appropriateFor: nil, create: true) - let dir = base.appendingPathComponent("OpenWhoop/Diagnostics", isDirectory: true) + let dir: URL + if let directory { + dir = directory + } else { + let base = try FileManager.default.url(for: .applicationSupportDirectory, in: .userDomainMask, + appropriateFor: nil, create: true) + dir = base.appendingPathComponent("OpenWhoop/Diagnostics", isDirectory: true) + } try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) let safeId = deviceId.replacingOccurrences(of: "/", with: "_") let url = dir.appendingPathComponent("oura-raw-\(safeId).jsonl") + + // Session roll: keep the previous session as ".1" (dropping the one before it) and begin fresh, + // so the live file is always exactly this capture. An empty leftover is simply reused — rolling + // it would spend the one retained generation on nothing. + let existing = ((try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? Int) ?? 0 + if existing > 0 { + let previous = dir.appendingPathComponent(url.lastPathComponent + ".1") + try? FileManager.default.removeItem(at: previous) + try? FileManager.default.moveItem(at: url, to: previous) + } if !FileManager.default.fileExists(atPath: url.path) { FileManager.default.createFile(atPath: url.path, contents: nil) } + sessionBytes = 0 fileURL = url return url } catch { diff --git a/StrandTests/OuraRawDumpSessionTests.swift b/StrandTests/OuraRawDumpSessionTests.swift new file mode 100644 index 0000000000..59e8efe17b --- /dev/null +++ b/StrandTests/OuraRawDumpSessionTests.swift @@ -0,0 +1,106 @@ +import XCTest +@testable import Strand + +/// The raw Oura sidecar is scoped to ONE capture session (option 1 of the 2026-08-09 sidecar decision). +/// These pin the property that matters: the live file is never rolled part-way through a session, which is +/// the failure that cost a night of raw evidence on 2026-08-09 (a 25 MB threshold fired at 07:06:58, mid +/// drain, and the strap-log bundle does not collect the `.1`). +final class OuraRawDumpSessionTests: XCTestCase { + private var dir: URL! + + override func setUpWithError() throws { + dir = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("oura-raw-tests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + try? FileManager.default.removeItem(at: dir) + } + + private func live() -> URL { dir.appendingPathComponent("oura-raw-ring1.jsonl") } + private func previous() -> URL { dir.appendingPathComponent("oura-raw-ring1.jsonl.1") } + private func lines(_ url: URL) -> [String] { + guard let s = try? String(contentsOf: url, encoding: .utf8) else { return [] } + return s.split(separator: "\n").map(String.init) + } + + func testOneSessionKeepsEveryRecordInTheLiveFile() { + let dump = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + for i in 0..<500 { dump.record(bytes: [0x5D, UInt8(i % 256), 0x01, 0x02]) } + // The whole session is in ONE file, and nothing was rolled aside part-way through. + XCTAssertEqual(lines(live()).count, 500) + XCTAssertFalse(FileManager.default.fileExists(atPath: previous().path)) + } + + func testANewSessionRollsThePreviousOneAsideAndStartsEmpty() { + let first = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + for _ in 0..<10 { first.record(bytes: [0xAA]) } + XCTAssertEqual(lines(live()).count, 10) + + // A new object is a new session (one is built per OuraLiveSource). + let second = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + second.record(bytes: [0xBB]) + + XCTAssertEqual(lines(live()).count, 1, "the live file is THIS session only") + XCTAssertEqual(lines(previous()).count, 10, "the previous session is retained as .1") + XCTAssertTrue(lines(live())[0].contains("bb"), "the live file holds the new session's bytes") + } + + func testOnlyOnePreviousSessionIsRetained() { + for tag in [UInt8(0x11), 0x22, 0x33] { + let dump = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + dump.record(bytes: [tag]) + } + XCTAssertTrue(lines(live())[0].contains("33")) // newest session + XCTAssertTrue(lines(previous())[0].contains("22")) // the one before it + // The oldest is gone: exactly two generations exist, so the corpus stays bounded. + let files = (try? FileManager.default.contentsOfDirectory(atPath: dir.path)) ?? [] + XCTAssertEqual(files.filter { $0.hasPrefix("oura-raw-ring1.jsonl") }.count, 2) + } + + func testAnEmptyLeftoverIsReusedRatherThanSpendingTheRetainedGeneration() { + // Build the state a killed session leaves behind: `.1` holds a REAL session, and the live file was + // created but never written to (0 bytes). Rolling that empty file would spend the single retained + // generation on nothing and discard the real one. + let sessionA = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + sessionA.record(bytes: [0xAA]) + let sessionB = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + sessionB.record(bytes: [0xBB]) // rolls A aside + XCTAssertTrue(lines(previous())[0].contains("aa")) + FileManager.default.createFile(atPath: live().path, contents: Data()) // B left nothing behind + + let sessionC = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + sessionC.record(bytes: [0xCC]) + + XCTAssertEqual(lines(live()).count, 1) + XCTAssertTrue(lines(live())[0].contains("cc")) + XCTAssertTrue(lines(previous())[0].contains("aa"), + "the real previous session survived an empty leftover") + } + + func testASessionThatReceivesNothingTouchesNoFilesAtAll() { + let sessionA = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + sessionA.record(bytes: [0xAA]) + // A source that comes up and never gets a record never resolves, so it cannot roll anything. + let silent = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + silent.record(bytes: []) // empty input is a no-op + XCTAssertEqual(lines(live()).count, 1) + XCTAssertTrue(lines(live())[0].contains("aa")) + XCTAssertFalse(FileManager.default.fileExists(atPath: previous().path)) + } + + func testCeilingStopsAppendingAndNeverRollsMidSession() { + let dump = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + // One record is far under the ceiling, so drive the counter with a payload sized to cross it. + let big = [UInt8](repeating: 0xEE, count: 64 * 1024) + for _ in 0..<(OuraRawDump.maxSessionBytes / (128 * 1024) + 8) { dump.record(bytes: big) } + + let size = (try? FileManager.default.attributesOfItem(atPath: live().path))?[.size] as? Int ?? 0 + XCTAssertLessThanOrEqual(size, OuraRawDump.maxSessionBytes, "the ceiling bounds the session") + XCTAssertGreaterThan(size, 0) + // The critical property: hitting the ceiling must NOT roll the file, or we recreate the 2026-08-09 + // bug at a different threshold. + XCTAssertFalse(FileManager.default.fileExists(atPath: previous().path)) + } +} From 997ea7244a9cdd910e36453e6487f6d2f61080cc Mon Sep 17 00:00:00 2001 From: Pipiche Date: Mon, 10 Aug 2026 09:54:09 +0200 Subject: [PATCH 5/7] fix(diagnostics): keep a RING of raw-sidecar generations, and export the one that holds the capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Session scoping (c9fd883e) fixed the mid-drain roll and then lost a night a different way. Measured on the 2026-08-10 capture: the app relaunched twice after wake (07:45 and 08:17 local), so both retained slots — the live file and its single `.1` — held post-wake sessions of 87 KB and 44 KB, and the night's 4 MB was gone before it could be pulled off the phone. Rotation ate a night every ~5-6 days at random. One-deep session scoping ate it every morning the app restarted twice, which is the normal morning. An iOS app is relaunched whenever the user opens it, so the number of session boundaries between a night and its export is not something OuraRawDump can predict; it can only keep enough of them. Two changes, and the second is what makes the first useful: * OuraRawDump keeps a numbered generation ring (`.1` ... `.8`) instead of one `.1`. A session under `minRolledBytes` (16 KB) is discarded rather than rolled, so a connect that captured essentially nothing cannot evict a real overnight capture, and the retained generations are pruned oldest-first to a 64 MB budget so the ring is bounded in BYTES, not merely in file count. * TestBundleAssembler.normalizedOuraEntryName now maps `.jsonl.` to the same entry name as the live file. Its existing largest-wins rule then ships the generation that actually holds the capture. Without this the export could only ever carry the live file — a 30-second morning session — and keeping generations on disk would help nobody without a USB cable. Largest-wins means a morning export can ship a sidecar from an earlier session than the live one. That is the point, and it is safe to choose blind: every line carries its own `utc`/`iso`, so an analysis scoped to a night either finds that night's records or honestly finds none. Diagnostics only - no BLE writes, no protocol change, nothing scored. Apple-only (Android has no raw sidecar writer), so there is no parity half. Verified: StrandTests 1092/0 (1 skipped), including 4 new OuraRawDump tests (the regression test replays the two post-wake relaunches) and a new normalizer test. `Strand` macOS and `NOOPiOS` iOS both built locally - no default CI compiles either. --- Strand/BLE/OuraRawDump.swift | 124 +++++++++++++++++---- Strand/System/TestBundleAssembler.swift | 28 ++++- StrandTests/OuraRawDumpSessionTests.swift | 113 ++++++++++++++----- StrandTests/TestBundleAssemblerTests.swift | 21 ++++ 4 files changed, 233 insertions(+), 53 deletions(-) diff --git a/Strand/BLE/OuraRawDump.swift b/Strand/BLE/OuraRawDump.swift index 2396708bd9..de9d135a04 100644 --- a/Strand/BLE/OuraRawDump.swift +++ b/Strand/BLE/OuraRawDump.swift @@ -18,23 +18,38 @@ import WhoopStore /// decoded sidecars there is NO dedup high-water: a re-served record is still evidence the ring re-sent it, /// and the offline reframer collapses duplicates by (tag, ring-time). /// -/// LIFETIME: the live file holds exactly ONE capture session. It is rolled to a single `.1` sibling when -/// this object resolves its file for the first time — i.e. at the START of a session — and never again -/// while the session runs. One `OuraRawDump` is built per `OuraLiveSource`, and a source survives BLE -/// reconnects (`SourceCoordinator` returns early when the active strap is unchanged), so a drain that -/// disconnects and re-engages several times still lands whole in one file. +/// LIFETIME: the live file holds exactly ONE capture session. At the START of a session — the first time +/// this object resolves its file — the previous session is rolled into a numbered generation ring +/// (`.1` … `.N`, newest first) and never touched again while the session runs. One `OuraRawDump` is built +/// per `OuraLiveSource`, and a source survives BLE reconnects (`SourceCoordinator` returns early when the +/// active strap is unchanged), so a drain that disconnects and re-engages several times still lands whole +/// in one file. /// /// WHY NOT a size threshold, which is what this used to do: a threshold can fire MID-DRAIN, and on /// 2026-08-09 it did — 25 MB rolled at 07:06:58 during the morning offload, carrying 32,005 records of -/// that drain and 71 of the night's 75 `0x5D` records into the `.1`, which the strap-log bundle does not -/// collect. The night's raw evidence never left the device (it was recoverable only by pulling the file -/// off the phone by hand). Lowering the threshold makes that MORE frequent, not less; only rolling on a -/// session boundary removes it. The corpus stays bounded because a session is bounded: an overnight -/// drain is ~32,000 records ≈ 4 MB. +/// that drain and 71 of the night's 75 `0x5D` records into the `.1`. Lowering the threshold makes that +/// MORE frequent, not less; only rolling on a session boundary removes it. The corpus stays bounded +/// because a session is bounded: an overnight drain is ~32,000 records ≈ 4 MB. /// -/// The multi-day corpus this used to accumulate is deliberately given up, because the bundle replaces it: -/// every capture now carries its own COMPLETE raw sidecar, and the bundles are what get archived. One -/// previous session is kept as `.1` purely so a session that ends without an export is not lost. +/// WHY A RING AND NOT ONE `.1`, which is what the first version of this did: **retaining a single +/// generation is not enough to survive an ordinary morning.** Measured 2026-08-10 — the app relaunched +/// twice after wake (07:45 and 08:17 local), so by the time the capture was pulled off the phone BOTH +/// retained slots held post-wake sessions of 87 KB and 44 KB, and the night's 4 MB was gone. Session +/// scoping fixed the mid-drain roll and then lost the night a different way: rotation ate a night every +/// ~5–6 days at random; one-deep session scoping ate it every morning the app restarted twice, which is +/// the normal morning. An iOS app is relaunched whenever the user opens it, so the number of session +/// boundaries between a night and its export is not something this class can predict — it can only keep +/// enough of them. +/// +/// Two rules keep the ring honest, and both exist because of that measurement: +/// * a session under `minRolledBytes` is DISCARDED rather than rolled, so a connect that captured +/// essentially nothing (the 44 KB one above) cannot evict a real overnight capture; +/// * the retained generations are pruned oldest-first to `maxRetainedBytes`, so the ring is bounded in +/// BYTES and not merely in file count (each generation may be up to `maxSessionBytes`). +/// +/// `TestBundleAssembler` ships the LARGEST generation for this kind, so the export carries the night even +/// when the live file is a fresh 30-second session — which is exactly the state a morning export is taken +/// in. Keeping generations on disk without that change would only help someone with a USB cable. /// /// Location: `/OpenWhoop/Diagnostics/oura-raw-.jsonl` — beside the SQLite. final class OuraRawDump { @@ -53,6 +68,21 @@ final class OuraRawDump { /// pathological backlog cannot fill the disk. static let maxSessionBytes = 25 * 1024 * 1024 + /// How many completed sessions are kept beside the live file, as `.1` (newest) … `.N` (oldest). Eight + /// covers a morning of app relaunches with room to spare, and at a typical ~4 MB overnight session the + /// whole ring is ~32 MB — well inside `maxRetainedBytes`, which is what actually bounds it. + static let retainedGenerations = 8 + + /// A finished session smaller than this is deleted instead of rolled. It is not a judgement about which + /// captures matter — it is the one rule that stops a burst of trivial reconnect sessions from flushing + /// the ring. 16 KB is ~100 records: a connect that drained nothing. Deliberately far below the 44 KB + /// session that destroyed the 2026-08-10 capture, so the rule is a backstop and the ring does the work. + static let minRolledBytes = 16 * 1024 + + /// Byte budget for the retained generations (the live file is excluded — it has its own ceiling). + /// Pruned oldest-first, so a pathological run of large sessions cannot fill the disk. + static let maxRetainedBytes = 64 * 1024 * 1024 + private static let iso: ISO8601DateFormatter = { let f = ISO8601DateFormatter() f.timeZone = TimeZone(identifier: "UTC") @@ -127,14 +157,16 @@ final class OuraRawDump { let safeId = deviceId.replacingOccurrences(of: "/", with: "_") let url = dir.appendingPathComponent("oura-raw-\(safeId).jsonl") - // Session roll: keep the previous session as ".1" (dropping the one before it) and begin fresh, - // so the live file is always exactly this capture. An empty leftover is simply reused — rolling - // it would spend the one retained generation on nothing. - let existing = ((try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? Int) ?? 0 - if existing > 0 { - let previous = dir.appendingPathComponent(url.lastPathComponent + ".1") - try? FileManager.default.removeItem(at: previous) - try? FileManager.default.moveItem(at: url, to: previous) + // Session roll: shift the previous session into the generation ring and begin fresh, so the live + // file is always exactly this capture. A leftover that captured essentially nothing (including an + // empty one) is dropped rather than rolled — spending a generation on it is how the 2026-08-10 + // capture was lost. + let existing = Self.byteSize(url) + if existing >= Self.minRolledBytes { + rollGenerations(in: dir, base: url) + pruneRetained(in: dir, base: url) + } else if existing > 0 { + try? FileManager.default.removeItem(at: url) } if !FileManager.default.fileExists(atPath: url.path) { FileManager.default.createFile(atPath: url.path, contents: nil) @@ -148,4 +180,54 @@ final class OuraRawDump { return nil } } + + /// The generation file for `index` (1 = newest retained session). + static func generationURL(base: URL, index: Int) -> URL { + base.deletingLastPathComponent() + .appendingPathComponent(base.lastPathComponent + ".\(index)") + } + + private static func byteSize(_ url: URL) -> Int { + ((try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? Int) ?? 0 + } + + /// Shift the ring up by one: the oldest generation is dropped, every other moves down a slot, and the + /// finished live file becomes `.1`. Best-effort throughout — a failed move must never break the BLE + /// path, and the worst outcome is one lost generation rather than a lost session. + private func rollGenerations(in dir: URL, base url: URL) { + let oldest = Self.generationURL(base: url, index: Self.retainedGenerations) + try? FileManager.default.removeItem(at: oldest) + var index = Self.retainedGenerations - 1 + while index >= 1 { + let from = Self.generationURL(base: url, index: index) + let to = Self.generationURL(base: url, index: index + 1) + if FileManager.default.fileExists(atPath: from.path) { + try? FileManager.default.removeItem(at: to) + try? FileManager.default.moveItem(at: from, to: to) + } + index -= 1 + } + try? FileManager.default.moveItem(at: url, to: Self.generationURL(base: url, index: 1)) + } + + /// Drop retained generations oldest-first until they fit `maxRetainedBytes`. Counts only the ring: the + /// live file has its own per-session ceiling and is not part of this budget. + private func pruneRetained(in dir: URL, base url: URL) { + var total = 0 + var kept: [URL] = [] + for index in 1...Self.retainedGenerations { + let gen = Self.generationURL(base: url, index: index) + let size = Self.byteSize(gen) + guard size > 0 else { continue } + total += size + kept.append(gen) + } + guard total > Self.maxRetainedBytes else { return } + for gen in kept.reversed() { // oldest slot first + guard total > Self.maxRetainedBytes else { break } + total -= Self.byteSize(gen) + try? FileManager.default.removeItem(at: gen) + } + log("Oura: raw capture ring pruned to its \(Self.maxRetainedBytes / (1024 * 1024)) MB budget") + } } diff --git a/Strand/System/TestBundleAssembler.swift b/Strand/System/TestBundleAssembler.swift index d8f88d5c3f..e7bc3fda67 100644 --- a/Strand/System/TestBundleAssembler.swift +++ b/Strand/System/TestBundleAssembler.swift @@ -375,12 +375,21 @@ enum TestBundleAssembler { /// file is not one of the Oura sidecars in `ouraSidecarKinds`. `oura--.jsonl` → /// `oura-.jsonl`, which (a) keeps the ring id out of the bundle's filenames (redaction scrubs /// content, never names) and (b) lands on the `trimmableNames` set so the cap can trim it. + /// + /// A ROLLED GENERATION (`oura--.jsonl.`, written by `OuraRawDump`'s session ring) maps + /// to the SAME normalized name on purpose, so `ouraDiagnosticEntries`' existing largest-wins rule picks + /// whichever generation actually holds the capture. Before this, the `.jsonl` suffix test rejected every + /// generation and the bundle could only ever ship the live file — which on 2026-08-10 was a 30-second + /// morning session while the night's 4 MB sat in a generation the export never looked at. /// Pure/string-only for unit testing. static func normalizedOuraEntryName(forFile filename: String) -> String? { - for kind in ouraSidecarKinds { - if filename.hasPrefix("oura-\(kind)-"), filename.hasSuffix(".jsonl") { - return "oura-\(kind).jsonl" - } + for kind in ouraSidecarKinds where filename.hasPrefix("oura-\(kind)-") { + guard let range = filename.range(of: ".jsonl") else { continue } + let tail = filename[range.upperBound...] // "" for the live file, ".3" for a generation + guard tail.isEmpty + || (tail.hasPrefix(".") && tail.dropFirst().allSatisfy(\.isNumber) + && tail.count > 1) else { continue } + return "oura-\(kind).jsonl" } return nil } @@ -388,8 +397,15 @@ enum TestBundleAssembler { /// Gather the Oura ring's Tier-B JSONL sidecars as bundle entries, normalized names and RAW bytes (the /// caller redacts + caps). Enumerates the Diagnostics dir rather than reconstructing per-ring filenames, /// so it needs no active-ring id and picks up whatever was captured. If two rings produced the same kind - /// (rare), the entry names would collide; we keep the LARGEST file per normalized name (the fuller - /// capture is the more useful one) so the bundle never carries duplicate names. + /// (rare), or one ring left several rolled generations of a kind, the entry names collide; we keep the + /// LARGEST file per normalized name (the fuller capture is the more useful one) so the bundle never + /// carries duplicate names. + /// + /// ⚠️ Largest-wins means a morning export can legitimately ship a sidecar from an EARLIER session than + /// the live one — that is the point, and it is why the choice is safe to make blind: every line carries + /// its own `utc`/`iso`, so an analysis scoped to a night either finds that night's records or honestly + /// finds none. The alternative (shipping every generation as its own entry) would multiply the entries + /// competing for the export cap and starve the very file this exists to preserve. static func ouraDiagnosticEntries() -> [FileExport.BundleEntry] { guard let dir = ouraDiagnosticsDir(), let files = try? FileManager.default.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil) diff --git a/StrandTests/OuraRawDumpSessionTests.swift b/StrandTests/OuraRawDumpSessionTests.swift index 59e8efe17b..49eb91f53d 100644 --- a/StrandTests/OuraRawDumpSessionTests.swift +++ b/StrandTests/OuraRawDumpSessionTests.swift @@ -1,10 +1,15 @@ import XCTest @testable import Strand -/// The raw Oura sidecar is scoped to ONE capture session (option 1 of the 2026-08-09 sidecar decision). -/// These pin the property that matters: the live file is never rolled part-way through a session, which is -/// the failure that cost a night of raw evidence on 2026-08-09 (a 25 MB threshold fired at 07:06:58, mid -/// drain, and the strap-log bundle does not collect the `.1`). +/// The raw Oura sidecar is scoped to ONE capture session (option 1 of the 2026-08-09 sidecar decision), +/// and completed sessions are kept in a numbered generation ring (the 2026-08-10 follow-up). +/// +/// These pin two properties, each bought with a lost night of raw evidence: +/// * the live file is never rolled part-way through a session — a 25 MB threshold fired mid-drain at +/// 07:06:58 on 2026-08-09 and carried 71 of the night's 75 `0x5D` records out of the export; +/// * a session boundary does not evict a real capture — one-deep retention lost the 2026-08-10 night +/// because the app relaunched twice after wake (07:45, 08:17), filling both slots with sessions of +/// 87 KB and 44 KB. final class OuraRawDumpSessionTests: XCTestCase { private var dir: URL! @@ -19,12 +24,37 @@ final class OuraRawDumpSessionTests: XCTestCase { } private func live() -> URL { dir.appendingPathComponent("oura-raw-ring1.jsonl") } - private func previous() -> URL { dir.appendingPathComponent("oura-raw-ring1.jsonl.1") } + private func previous() -> URL { generation(1) } + private func generation(_ index: Int) -> URL { + dir.appendingPathComponent("oura-raw-ring1.jsonl.\(index)") + } private func lines(_ url: URL) -> [String] { guard let s = try? String(contentsOf: url, encoding: .utf8) else { return [] } return s.split(separator: "\n").map(String.init) } + private func size(_ url: URL) -> Int { + ((try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? Int) ?? 0 + } + + /// A session big enough to be worth a generation slot (>= `minRolledBytes`), tagged so it can be + /// identified after rolling. `repeat` is load-bearing: a fresh `OuraRawDump` has not resolved its file + /// yet, so before the FIRST record the live file still holds the previous session — a leading size + /// check would see it already "full" and write nothing at all. + private func writeSubstantialSession(tag: UInt8) { + let dump = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + let payload = [tag] + [UInt8](repeating: 0x00, count: 512) + repeat { dump.record(bytes: payload) } while size(live()) < OuraRawDump.minRolledBytes + } + + /// The `hex` field of a file's first record. Read the FIELD, never the whole line: the line also carries + /// a unix `utc` and an ISO stamp, and a substring search for a tag like "22" hits those constantly. + private func firstHex(_ url: URL) -> String? { + guard let line = lines(url).first, + let range = line.range(of: "\"hex\":\"") else { return nil } + return line[range.upperBound...].prefix(while: { $0 != "\"" }).description + } + func testOneSessionKeepsEveryRecordInTheLiveFile() { let dump = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) for i in 0..<500 { dump.record(bytes: [0x5D, UInt8(i % 256), 0x01, 0x02]) } @@ -34,40 +64,71 @@ final class OuraRawDumpSessionTests: XCTestCase { } func testANewSessionRollsThePreviousOneAsideAndStartsEmpty() { - let first = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) - for _ in 0..<10 { first.record(bytes: [0xAA]) } - XCTAssertEqual(lines(live()).count, 10) + writeSubstantialSession(tag: 0xAA) + let rolledLineCount = lines(live()).count + XCTAssertGreaterThan(rolledLineCount, 0) // A new object is a new session (one is built per OuraLiveSource). let second = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) second.record(bytes: [0xBB]) XCTAssertEqual(lines(live()).count, 1, "the live file is THIS session only") - XCTAssertEqual(lines(previous()).count, 10, "the previous session is retained as .1") + XCTAssertEqual(lines(previous()).count, rolledLineCount, "the previous session is retained as .1") XCTAssertTrue(lines(live())[0].contains("bb"), "the live file holds the new session's bytes") + XCTAssertEqual(firstHex(previous())?.prefix(2), "aa") } - func testOnlyOnePreviousSessionIsRetained() { - for tag in [UInt8(0x11), 0x22, 0x33] { - let dump = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) - dump.record(bytes: [tag]) - } - XCTAssertTrue(lines(live())[0].contains("33")) // newest session - XCTAssertTrue(lines(previous())[0].contains("22")) // the one before it - // The oldest is gone: exactly two generations exist, so the corpus stays bounded. + /// THE 2026-08-10 REGRESSION TEST. Two ordinary post-wake app relaunches must not be able to reach back + /// and destroy the night: with one-deep retention this is exactly what happened. + func testAnOvernightSessionSurvivesTwoRestartsAfterWake() { + writeSubstantialSession(tag: 0x11) // "the night" + writeSubstantialSession(tag: 0x22) // 07:45 relaunch + writeSubstantialSession(tag: 0x33) // 08:17 relaunch + let export = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + export.record(bytes: [0xFF]) // the session the export is taken during + + XCTAssertEqual(firstHex(live()), "ff") + XCTAssertEqual(firstHex(generation(1))?.prefix(2), "33") + XCTAssertEqual(firstHex(generation(2))?.prefix(2), "22") + XCTAssertEqual(firstHex(generation(3))?.prefix(2), "11", "the night is still on disk") + } + + func testTheGenerationRingIsBoundedAtRetainedGenerations() { + for _ in 0..<(OuraRawDump.retainedGenerations + 4) { writeSubstantialSession(tag: 0x55) } let files = (try? FileManager.default.contentsOfDirectory(atPath: dir.path)) ?? [] - XCTAssertEqual(files.filter { $0.hasPrefix("oura-raw-ring1.jsonl") }.count, 2) + let ours = files.filter { $0.hasPrefix("oura-raw-ring1.jsonl") } + XCTAssertEqual(ours.count, OuraRawDump.retainedGenerations + 1, + "the live file plus exactly \(OuraRawDump.retainedGenerations) generations") + XCTAssertFalse(FileManager.default.fileExists( + atPath: generation(OuraRawDump.retainedGenerations + 1).path)) } - func testAnEmptyLeftoverIsReusedRatherThanSpendingTheRetainedGeneration() { + /// The rule that makes the ring worth having: a connect that captured essentially nothing is DELETED, + /// not rolled, so a burst of trivial reconnect sessions cannot flush a real capture out of the ring. + func testATrivialSessionIsDiscardedRatherThanSpendingAGeneration() { + writeSubstantialSession(tag: 0xAA) // the capture worth keeping + + for _ in 0..<4 { // four near-empty sessions + let tiny = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + tiny.record(bytes: [0xBB]) + } + let final = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) + final.record(bytes: [0xCC]) + + XCTAssertEqual(firstHex(live()), "cc") + XCTAssertEqual(firstHex(previous())?.prefix(2), "aa", + "the real session is still .1 — no trivial session took a slot") + XCTAssertFalse(FileManager.default.fileExists(atPath: generation(2).path)) + } + + func testAnEmptyLeftoverIsReusedRatherThanSpendingARetainedGeneration() { // Build the state a killed session leaves behind: `.1` holds a REAL session, and the live file was - // created but never written to (0 bytes). Rolling that empty file would spend the single retained - // generation on nothing and discard the real one. - let sessionA = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) - sessionA.record(bytes: [0xAA]) + // created but never written to (0 bytes). Rolling that empty file would push the real capture down + // the ring for nothing. + writeSubstantialSession(tag: 0xAA) let sessionB = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) sessionB.record(bytes: [0xBB]) // rolls A aside - XCTAssertTrue(lines(previous())[0].contains("aa")) + XCTAssertEqual(firstHex(previous())?.prefix(2), "aa") FileManager.default.createFile(atPath: live().path, contents: Data()) // B left nothing behind let sessionC = OuraRawDump(deviceId: "ring1", log: { _ in }, directory: dir) @@ -75,8 +136,8 @@ final class OuraRawDumpSessionTests: XCTestCase { XCTAssertEqual(lines(live()).count, 1) XCTAssertTrue(lines(live())[0].contains("cc")) - XCTAssertTrue(lines(previous())[0].contains("aa"), - "the real previous session survived an empty leftover") + XCTAssertEqual(firstHex(previous())?.prefix(2), "aa", + "the real previous session survived an empty leftover") } func testASessionThatReceivesNothingTouchesNoFilesAtAll() { diff --git a/StrandTests/TestBundleAssemblerTests.swift b/StrandTests/TestBundleAssemblerTests.swift index c179217921..d29b5ffe8d 100644 --- a/StrandTests/TestBundleAssemblerTests.swift +++ b/StrandTests/TestBundleAssemblerTests.swift @@ -107,6 +107,27 @@ final class TestBundleAssemblerTests: XCTestCase { XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName(forFile: "oura-raw.txt")) } + /// A ROLLED GENERATION must normalize to the same entry name as the live file, so + /// `ouraDiagnosticEntries`' largest-wins rule can ship the generation that actually holds the capture. + /// Before this, `hasSuffix(".jsonl")` rejected every `.jsonl.` and a morning export could only ever + /// carry the live file — a 30-second session, while the night sat in a generation nothing looked at + /// (measured 2026-08-10). + func testRolledGenerationsNormalizeToTheSameEntryName() { + for suffix in [".jsonl.1", ".jsonl.2", ".jsonl.8", ".jsonl.12"] { + XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-raw-oura-2H3B2405003655" + suffix), "oura-raw.jsonl", suffix) + } + XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-spo2-oura-2H3B2405003655.jsonl.3"), "oura-spo2.jsonl") + // A non-numeric or empty tail is NOT a generation — those are someone else's files, not ours. + XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-raw-oura-2H3B2405003655.jsonl.bak")) + XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-raw-oura-2H3B2405003655.jsonl.")) + XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName( + forFile: "oura-raw-oura-2H3B2405003655.jsonl.1.gz")) + } + func testNormalizedOuraNamesAreAllTrimmable() { // Every normalized sidecar name must be in the cap's trimmable set, else a big night's dump could // blow the 20 MB cap instead of being tail-trimmed. From d3f514b6cbf24f034dbb3b75eefac3897f5b9ea0 Mon Sep 17 00:00:00 2001 From: Pipiche Date: Sat, 5 Sep 2026 15:33:59 +0200 Subject: [PATCH 6/7] fix(diagnostics): merge the Oura sidecar generations instead of shipping the largest The bundle picked, per sidecar kind, whichever generation file was biggest and shipped only that one. Size is not recency: a backfill burst rolls a huge generation that then wins every export until it ages out, so the export can carry a stale window while the night it was taken for sits in a smaller, newer file that is dropped. Measured on the 2026-09-05 export: the shipped `oura-raw.jsonl` spanned 2026-09-04 10:58:22 -> 12:03:47 UTC, the previous day's post-reflash backfill, and the hypnogram ledger then reported a plausible-looking 2026-09-04 row for a bundle exported on 09-05 - a wrong answer, not a missing one. Merge the generations chronologically instead of choosing between them. The entry cap already keeps the tail, so "the newest records survive" falls out of the two rules composed rather than being a third rule to get right. The selection logic is pure, so it lives in WhoopStore where `swift test` covers it (`app-build.yml` is disabled, so nothing in the app target is CI-validated); `TestBundleAssembler` keeps only the file I/O. Shipping every generation as separate entries was the other option and is worse: extra entries compete for the same byte budget under the fair-share allocator and would starve the very file being protected. Verification: WhoopStore `swift test` 503 tests / 0 failures (14 new); StrandTests/TestBundleAssemblerTests 19/19 including three end-to-end cases over real files, one reproducing the 09-05 generation layout; macOS app builds (`xcodebuild -scheme Strand -destination 'platform=macOS'`); doc_comment_lint clean. Not yet verified against a device export - that needs this build on the phone and one morning capture. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YKP96uMkGnnFhrp4VuXnPD --- .../WhoopStore/OuraSidecarGenerations.swift | 117 +++++++++++++++ .../OuraSidecarGenerationsTests.swift | 140 ++++++++++++++++++ Strand/System/TestBundleAssembler.swift | 94 +++++++----- StrandTests/TestBundleAssemblerTests.swift | 81 +++++++++- 4 files changed, 396 insertions(+), 36 deletions(-) create mode 100644 Packages/WhoopStore/Sources/WhoopStore/OuraSidecarGenerations.swift create mode 100644 Packages/WhoopStore/Tests/WhoopStoreTests/OuraSidecarGenerationsTests.swift diff --git a/Packages/WhoopStore/Sources/WhoopStore/OuraSidecarGenerations.swift b/Packages/WhoopStore/Sources/WhoopStore/OuraSidecarGenerations.swift new file mode 100644 index 0000000000..d7376d5446 --- /dev/null +++ b/Packages/WhoopStore/Sources/WhoopStore/OuraSidecarGenerations.swift @@ -0,0 +1,117 @@ +import Foundation + +/// Pure, deterministic selection rule for WHICH Oura diagnostics files a Test Centre export ships. +/// +/// The dumps (`Strand/BLE/Oura*Dump.swift`) keep a per-session generation ring: the live file +/// `oura--.jsonl` is the CURRENT session, and finished sessions roll into +/// `…jsonl.1` (newest retained) … `…jsonl.N` (oldest). Several files therefore map to one bundle +/// entry name, and the assembler must decide what to do about that. +/// +/// WHY THIS EXISTS — the rule it replaces was `LARGEST WINS`, and largest is the wrong criterion. +/// The wake drain flushes the whole night's bank in one burst, so on an ordinary morning the biggest +/// generation is whichever session happened to drain the most — routinely NOT the one holding the +/// night. Measured on `noop-master-iOS-v11.1.1-260905-0714.zip`: the shipped `oura-raw.jsonl` spanned +/// `2026-09-04 10:58:22 → 12:03:47 UTC` — the previous day's post-reinstall backfill, which was +/// simply the fattest file on disk. That morning's own 07:08 drain, the one carrying the night's +/// banked records (including the single `0x5c` of the day and the hypnogram), was in a different +/// generation and never left the device. Over the whole `Sleep Nights` corpus this cost **11 of 31 +/// nights**, which is the single biggest limit on the hypnogram ledger. +/// +/// THE RULE: do not pick at all — **concatenate the generations, oldest → newest**, and let the +/// bundle cap keep the tail. The cap already trims from the FRONT and keeps the most-recent bytes, +/// so merging in chronological order makes "the newest capture survives" a property of the two rules +/// composed, rather than a guess made blind at selection time. The morning drain is by construction +/// the newest data, so it can no longer be the thing that gets dropped. +/// +/// WHY NOT ship every generation as its own entry (the other candidate in the issue draft): each +/// extra entry competes for the same 20 MB cap under max-min fair allocation, so N generations of a +/// bulk sidecar would shrink every other stream's share — starving exactly the file this exists to +/// preserve. Merging costs one entry, keeps the bundle listing stable, and needs no new names. +/// +/// SAFE TO MERGE: generations are disjoint app sessions, and the raw capture deliberately keeps no +/// dedup high-water (a re-served record is itself evidence the ring re-sent it). Every line carries +/// its own `utc`/`iso`, and the offline reframer collapses duplicates by `(tag, ring-time)`, so a +/// concatenation reads exactly like the longer single capture it stands in for. +public enum OuraSidecarGenerations { + + /// Bundle entry name for a sidecar kind — the ring id is dropped, because bundle redaction scrubs + /// file CONTENT and never file NAMES. + public static func entryName(kind: String) -> String { "oura-\(kind).jsonl" } + + /// Classify a diagnostics filename as `(kind, generation)`, or nil when it is not one of `kinds`. + /// + /// `generation` is 0 for the live file and `n` for `…jsonl.n`, matching the dumps' own ring where + /// **1 is the NEWEST retained** session and higher indices are older. So a smaller generation is + /// always the more recent capture, and 0 (live) is the most recent of all — that single ordering + /// fact is what `mergePlan` sorts on. + /// + /// Rejects anything that is not exactly the ring's shape: no suffix at all beyond `.jsonl`, or a + /// dot followed by digits only. `oura-raw.txt`, `oura-raw-.jsonl.bak` and a bare + /// `oura-raw.jsonl` (no ring id) are all nil. + public static func classify(filename: String, kinds: [String]) -> (kind: String, generation: Int)? { + for kind in kinds where filename.hasPrefix("oura-\(kind)-") { + guard let range = filename.range(of: ".jsonl") else { continue } + let tail = filename[range.upperBound...] // "" for the live file, ".3" for a generation + if tail.isEmpty { return (kind, 0) } + guard tail.hasPrefix("."), tail.count > 1 else { continue } + let digits = tail.dropFirst() + guard digits.allSatisfy(\.isNumber), let n = Int(digits) else { continue } + return (kind, n) + } + return nil + } + + /// One bundle entry and the files that make it up, already in concatenation order. + public struct Plan: Equatable { + /// Normalized bundle entry name, e.g. `oura-raw.jsonl`. + public let entryName: String + /// Filenames to concatenate, **OLDEST → NEWEST**. Never empty. + public let files: [String] + public init(entryName: String, files: [String]) { + self.entryName = entryName + self.files = files + } + } + + /// Plan the merge for every sidecar kind present in `files`. + /// + /// Files are grouped by kind, ordered newest-first (live, then generation 1, 2, …), and taken + /// until their cumulative size reaches `ceilingBytes`; the file that crosses the ceiling is + /// INCLUDED (so the boundary never silently drops the run that spans it) and everything older is + /// left on the device. The kept run is then returned oldest → newest, ready to concatenate. + /// + /// `ceilingBytes` is a READ bound, not a budget: the bundle cap is the thing that decides what + /// finally ships, and it can never keep more than the whole cap for one entry — so reading beyond + /// that is pure wasted memory. Pass the bundle's cap. A non-positive ceiling still yields the + /// single newest file per kind, because shipping nothing would be worse than shipping the tail. + /// + /// Deterministic: the result does not depend on the order of `files`, and ties between two files + /// of the same kind and generation (impossible from one ring, possible if two rings wrote the same + /// kind) break on the filename so the bundle listing stays stable. + /// + /// Returns one `Plan` per kind, in `kinds` order, so the bundle listing is deterministic too. + public static func mergePlan(files: [(name: String, bytes: Int)], + kinds: [String], + ceilingBytes: Int) -> [Plan] { + var byKind: [String: [(generation: Int, name: String, bytes: Int)]] = [:] + for file in files { + guard let hit = classify(filename: file.name, kinds: kinds) else { continue } + byKind[hit.kind, default: []].append((hit.generation, file.name, max(0, file.bytes))) + } + return kinds.compactMap { kind -> Plan? in + guard let group = byKind[kind], !group.isEmpty else { return nil } + // Newest first: generation 0 (live), then 1 (newest retained), … N (oldest). + let newestFirst = group.sorted { + $0.generation != $1.generation ? $0.generation < $1.generation : $0.name < $1.name + } + var kept: [String] = [] + var accumulated = 0 + for file in newestFirst { + kept.append(file.name) + accumulated += file.bytes + if accumulated >= ceilingBytes { break } + } + return Plan(entryName: entryName(kind: kind), files: kept.reversed()) + } + } +} diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/OuraSidecarGenerationsTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/OuraSidecarGenerationsTests.swift new file mode 100644 index 0000000000..8eb767f59a --- /dev/null +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/OuraSidecarGenerationsTests.swift @@ -0,0 +1,140 @@ +import XCTest +@testable import WhoopStore + +/// Pins the export's generation-selection rule. These run under `swift test` with no app, no strap and +/// no CoreBluetooth, which matters because the caller (`TestBundleAssembler`) is app-target Swift that +/// no default CI job compiles — this is the only automated cover the rule has. +final class OuraSidecarGenerationsTests: XCTestCase { + + private let kinds = ["raw", "spo2", "motion", "activity"] + + // MARK: - classify + + func testClassifyLiveFileIsGenerationZero() { + let hit = OuraSidecarGenerations.classify(filename: "oura-raw-AABB.jsonl", kinds: kinds) + XCTAssertEqual(hit?.kind, "raw") + XCTAssertEqual(hit?.generation, 0) + } + + func testClassifyRolledGenerations() { + for n in 1...8 { + let hit = OuraSidecarGenerations.classify(filename: "oura-raw-AABB.jsonl.\(n)", kinds: kinds) + XCTAssertEqual(hit?.kind, "raw") + XCTAssertEqual(hit?.generation, n, "generation \(n)") + } + // Multi-digit, so the ring is not silently capped at nine. + XCTAssertEqual(OuraSidecarGenerations.classify(filename: "oura-raw-AABB.jsonl.12", + kinds: kinds)?.generation, 12) + } + + func testClassifyRejectsNonRingShapes() { + // Not a JSONL sidecar, a non-numeric suffix, a trailing dot, and no ring id at all. + for bad in ["oura-raw.txt", "oura-raw-AABB.jsonl.bak", "oura-raw-AABB.jsonl.", + "oura-raw-AABB.jsonl.1a", "raw-capture.jsonl", "whoop.sqlite", "oura-raw.jsonl"] { + XCTAssertNil(OuraSidecarGenerations.classify(filename: bad, kinds: kinds), bad) + } + // A kind we do not collect. + XCTAssertNil(OuraSidecarGenerations.classify(filename: "oura-unknown-AABB.jsonl", kinds: kinds)) + } + + func testEntryNameDropsRingId() { + XCTAssertEqual(OuraSidecarGenerations.entryName(kind: "raw"), "oura-raw.jsonl") + } + + // MARK: - mergePlan ordering (the actual defect) + + /// THE REGRESSION TEST. Under the old largest-wins rule this exact input shipped `gen 2` alone — + /// the fattest file — and the newest capture never left the device. The plan must now END with the + /// live file, because the cap keeps the TAIL. + func testMergePlanEndsWithTheNewestFileEvenWhenAnOlderGenerationIsFarLarger() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-raw-AABB.jsonl", 100), // live: this morning's drain, small + ("oura-raw-AABB.jsonl.1", 500), + ("oura-raw-AABB.jsonl.2", 9_000)], // yesterday's backfill, the old winner + kinds: kinds, ceilingBytes: 1_000_000) + XCTAssertEqual(plan.count, 1) + XCTAssertEqual(plan[0].entryName, "oura-raw.jsonl") + XCTAssertEqual(plan[0].files, ["oura-raw-AABB.jsonl.2", + "oura-raw-AABB.jsonl.1", + "oura-raw-AABB.jsonl"]) + XCTAssertEqual(plan[0].files.last, "oura-raw-AABB.jsonl", "newest must be last") + } + + func testMergePlanIsIndependentOfInputOrder() { + let forward: [(name: String, bytes: Int)] = [("oura-raw-A.jsonl", 1), ("oura-raw-A.jsonl.1", 2), + ("oura-raw-A.jsonl.2", 3)] + let a = OuraSidecarGenerations.mergePlan(files: forward, kinds: kinds, ceilingBytes: 1_000) + let b = OuraSidecarGenerations.mergePlan(files: forward.reversed(), kinds: kinds, ceilingBytes: 1_000) + XCTAssertEqual(a, b) + } + + func testMergePlanOrdersEntriesByKindsNotByDiscovery() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-motion-A.jsonl", 1), ("oura-raw-A.jsonl", 1), ("oura-spo2-A.jsonl", 1)], + kinds: kinds, ceilingBytes: 1_000) + XCTAssertEqual(plan.map(\.entryName), ["oura-raw.jsonl", "oura-spo2.jsonl", "oura-motion.jsonl"]) + } + + func testMergePlanSkipsKindsWithNoFiles() { + let plan = OuraSidecarGenerations.mergePlan(files: [("oura-raw-A.jsonl", 1)], + kinds: kinds, ceilingBytes: 1_000) + XCTAssertEqual(plan.map(\.entryName), ["oura-raw.jsonl"]) + } + + // MARK: - mergePlan ceiling + + /// The ceiling is a READ bound. The file that crosses it is kept, so the run spanning the boundary + /// is never half-read; everything strictly older is left on the device. + func testMergePlanStopsAtCeilingButIncludesTheCrossingFile() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-raw-A.jsonl", 60), // newest + ("oura-raw-A.jsonl.1", 60), // crosses the 100 ceiling -> included + ("oura-raw-A.jsonl.2", 60)], // older -> dropped + kinds: kinds, ceilingBytes: 100) + XCTAssertEqual(plan[0].files, ["oura-raw-A.jsonl.1", "oura-raw-A.jsonl"]) + } + + /// A single file at or over the ceiling reads exactly as it did before this change — one file, the + /// newest — so the memory profile never regresses against largest-wins. + func testMergePlanKeepsOnlyTheNewestWhenItAlreadyFillsTheCeiling() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-raw-A.jsonl", 500), ("oura-raw-A.jsonl.1", 500)], + kinds: kinds, ceilingBytes: 100) + XCTAssertEqual(plan[0].files, ["oura-raw-A.jsonl"]) + } + + /// Shipping nothing would be strictly worse than shipping the newest tail, so a degenerate ceiling + /// still yields one file rather than an empty plan. + func testMergePlanWithNonPositiveCeilingStillShipsTheNewestFile() { + for ceiling in [0, -1] { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-raw-A.jsonl", 10), ("oura-raw-A.jsonl.1", 10)], + kinds: kinds, ceilingBytes: ceiling) + XCTAssertEqual(plan[0].files, ["oura-raw-A.jsonl"], "ceiling \(ceiling)") + } + } + + func testMergePlanNeverReturnsAnEmptyFileList() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-raw-A.jsonl", 0)], kinds: kinds, ceilingBytes: 1_000) + XCTAssertEqual(plan[0].files, ["oura-raw-A.jsonl"]) + } + + func testMergePlanIgnoresUnrelatedFiles() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("raw-capture.jsonl", 10), ("whoop.sqlite", 10), ("oura-raw-A.jsonl", 10)], + kinds: kinds, ceilingBytes: 1_000) + XCTAssertEqual(plan.count, 1) + XCTAssertEqual(plan[0].files, ["oura-raw-A.jsonl"]) + } + + /// Two rings that wrote the same kind collide on generation. The tie must break deterministically + /// rather than by directory order, or the bundle listing changes between exports. + func testMergePlanBreaksSameGenerationTiesOnFilename() { + let plan = OuraSidecarGenerations.mergePlan( + files: [("oura-raw-BBBB.jsonl", 10), ("oura-raw-AAAA.jsonl", 10)], + kinds: kinds, ceilingBytes: 1_000) + // Newest-first sorts A before B; reversed for concatenation, B lands last. + XCTAssertEqual(plan[0].files, ["oura-raw-BBBB.jsonl", "oura-raw-AAAA.jsonl"]) + } +} diff --git a/Strand/System/TestBundleAssembler.swift b/Strand/System/TestBundleAssembler.swift index e7bc3fda67..e10719a41a 100644 --- a/Strand/System/TestBundleAssembler.swift +++ b/Strand/System/TestBundleAssembler.swift @@ -1,5 +1,6 @@ import Foundation import StrandAnalytics +import WhoopStore /// Assembles the Test Centre export bundle: gathers report.txt, meta.json, raw-capture and last-crash, /// runs the redaction pass over EVERY file, applies the 20 MB cap, and hands the entries to @@ -14,6 +15,11 @@ enum TestBundleAssembler { /// The redaction stamp written into meta.json so a maintainer knows the whole-bundle scrub ran. static let redactionVersion = "v2" + /// The bundle's hard byte cap (spec section 5.4; under GitHub's 25 MB attachment limit). Shared by + /// `capEntries` (what finally ships) and `ouraDiagnosticEntries` (how much it is worth reading), so the + /// read ceiling can never drift above the budget it feeds. + static let defaultCapBytes = 20 * 1024 * 1024 + /// The Oura Diagnostics-dir sidecar kinds the bundle attaches — the SINGLE source of truth /// `normalizedOuraEntryName`/`ouraDiagnosticEntries`/`trimmableNames`/`ouraSidecarNames` all derive /// from, so a new dump writer (`Strand/BLE/Oura*Dump.swift`) needs exactly one line added here to be @@ -91,7 +97,7 @@ enum TestBundleAssembler { /// that motivated it. With a single trimmable file this is still identical to the original /// raw-capture-only behaviour: its share is the whole remainder. static func capEntries(_ entries: [FileExport.BundleEntry], - capBytes: Int = 20 * 1024 * 1024) -> (entries: [FileExport.BundleEntry], truncated: Bool) { + capBytes: Int = defaultCapBytes) -> (entries: [FileExport.BundleEntry], truncated: Bool) { let total = entries.reduce(0) { $0 + $1.data.count } guard total > capBytes else { return (entries, false) } // Reserve the non-trimmable files (kept whole), then share the remainder across the trimmable ones. @@ -377,48 +383,66 @@ enum TestBundleAssembler { /// content, never names) and (b) lands on the `trimmableNames` set so the cap can trim it. /// /// A ROLLED GENERATION (`oura--.jsonl.`, written by `OuraRawDump`'s session ring) maps - /// to the SAME normalized name on purpose, so `ouraDiagnosticEntries`' existing largest-wins rule picks - /// whichever generation actually holds the capture. Before this, the `.jsonl` suffix test rejected every - /// generation and the bundle could only ever ship the live file — which on 2026-08-10 was a 30-second - /// morning session while the night's 4 MB sat in a generation the export never looked at. - /// Pure/string-only for unit testing. + /// to the SAME normalized name on purpose — `ouraDiagnosticEntries` merges the generations behind that + /// one name rather than choosing between them. Before generations were recognised at all, the `.jsonl` + /// suffix test rejected every one of them and the bundle could only ever ship the live file — which on + /// 2026-08-10 was a 30-second morning session while the night's 4 MB sat in a generation the export + /// never looked at. + /// + /// Thin wrapper over `WhoopStore.OuraSidecarGenerations`, where the rule is pure and unit-tested under + /// `swift test`; this file is app-target Swift that no default CI job compiles. static func normalizedOuraEntryName(forFile filename: String) -> String? { - for kind in ouraSidecarKinds where filename.hasPrefix("oura-\(kind)-") { - guard let range = filename.range(of: ".jsonl") else { continue } - let tail = filename[range.upperBound...] // "" for the live file, ".3" for a generation - guard tail.isEmpty - || (tail.hasPrefix(".") && tail.dropFirst().allSatisfy(\.isNumber) - && tail.count > 1) else { continue } - return "oura-\(kind).jsonl" - } - return nil + guard let hit = OuraSidecarGenerations.classify(filename: filename, kinds: ouraSidecarKinds) + else { return nil } + return OuraSidecarGenerations.entryName(kind: hit.kind) } /// Gather the Oura ring's Tier-B JSONL sidecars as bundle entries, normalized names and RAW bytes (the /// caller redacts + caps). Enumerates the Diagnostics dir rather than reconstructing per-ring filenames, - /// so it needs no active-ring id and picks up whatever was captured. If two rings produced the same kind - /// (rare), or one ring left several rolled generations of a kind, the entry names collide; we keep the - /// LARGEST file per normalized name (the fuller capture is the more useful one) so the bundle never - /// carries duplicate names. + /// so it needs no active-ring id and picks up whatever was captured. + /// + /// Several files map to one entry name — the live session plus its rolled generations, and (rarely) two + /// rings that wrote the same kind. We **concatenate them oldest → newest** rather than choosing one, so + /// that the cap's existing keep-the-tail trim decides what ships. `OuraSidecarGenerations.mergePlan` + /// holds that rule, pure and unit-tested; see its doc comment for the measured export that motivated it. /// - /// ⚠️ Largest-wins means a morning export can legitimately ship a sidecar from an EARLIER session than - /// the live one — that is the point, and it is why the choice is safe to make blind: every line carries - /// its own `utc`/`iso`, so an analysis scoped to a night either finds that night's records or honestly - /// finds none. The alternative (shipping every generation as its own entry) would multiply the entries - /// competing for the export cap and starve the very file this exists to preserve. - static func ouraDiagnosticEntries() -> [FileExport.BundleEntry] { - guard let dir = ouraDiagnosticsDir(), - let files = try? FileManager.default.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil) + /// ⚠️ THIS REPLACES `largest-wins`, which was not a buggy pick but the wrong criterion: the wake drain + /// flushes the night's whole bank at once, so the biggest generation is whichever session drained most + /// — routinely not the one holding the night. It cost 11 of 31 nights in the `Sleep Nights` corpus. + /// + /// Reads are bounded by `capBytes`: an entry can never keep more than the whole bundle cap, so pulling + /// older generations past that point would be wasted memory. A kind whose newest file already fills the + /// cap therefore reads exactly one file, as it did before. + /// + /// `directory` is injectable so the merge is testable against a temp dir laid out like a real ring; + /// production passes nil and gets `/OpenWhoop/Diagnostics`. + static func ouraDiagnosticEntries(capBytes: Int = defaultCapBytes, + directory: URL? = nil) -> [FileExport.BundleEntry] { + guard let dir = directory ?? ouraDiagnosticsDir(), + let files = try? FileManager.default.contentsOfDirectory( + at: dir, includingPropertiesForKeys: [.fileSizeKey]) else { return [] } - var byName: [String: FileExport.BundleEntry] = [:] - for url in files.sorted(by: { $0.lastPathComponent < $1.lastPathComponent }) { - guard let name = normalizedOuraEntryName(forFile: url.lastPathComponent), - let entry = fileEntry(at: url, name: name) else { continue } - if let existing = byName[name], existing.data.count >= entry.data.count { continue } - byName[name] = entry + let sizes: [(name: String, bytes: Int)] = files.map { url in + // Size from the directory entry, so planning never reads a file it will not ship. + let bytes = (try? url.resourceValues(forKeys: [.fileSizeKey]))?.fileSize ?? 0 + return (url.lastPathComponent, bytes) + } + let plans = OuraSidecarGenerations.mergePlan(files: sizes, kinds: ouraSidecarKinds, + ceilingBytes: capBytes) + return plans.compactMap { plan in + var merged = Data() + for filename in plan.files { + guard let part = try? Data(contentsOf: dir.appendingPathComponent(filename)), + !part.isEmpty else { continue } + // Every sidecar is newline-delimited JSONL. A file whose last write did not end in a + // newline would otherwise splice its final record onto the next generation's first one, + // producing a line that parses as neither. Cheap to guarantee, silent to get wrong. + if let last = merged.last, last != 0x0A { merged.append(0x0A) } + merged.append(part) + } + guard !merged.isEmpty else { return nil } + return FileExport.BundleEntry(name: plan.entryName, data: merged) } - // Stable order (ouraSidecarKinds order) so the bundle listing is deterministic. - return ouraSidecarKinds.map { "oura-\($0).jsonl" }.compactMap { byName[$0] } } /// The on-disk path a crash report would live at, when a crash handler is wired. There is no producer diff --git a/StrandTests/TestBundleAssemblerTests.swift b/StrandTests/TestBundleAssemblerTests.swift index d29b5ffe8d..7376c6cbb5 100644 --- a/StrandTests/TestBundleAssemblerTests.swift +++ b/StrandTests/TestBundleAssemblerTests.swift @@ -108,7 +108,8 @@ final class TestBundleAssemblerTests: XCTestCase { } /// A ROLLED GENERATION must normalize to the same entry name as the live file, so - /// `ouraDiagnosticEntries`' largest-wins rule can ship the generation that actually holds the capture. + /// `ouraDiagnosticEntries` can MERGE the generations behind that one name (it used to choose the + /// largest between them, which is why a morning export kept shipping the wrong session). /// Before this, `hasSuffix(".jsonl")` rejected every `.jsonl.` and a morning export could only ever /// carry the live file — a 30-second session, while the night sat in a generation nothing looked at /// (measured 2026-08-10). @@ -288,4 +289,82 @@ final class TestBundleAssemblerTests: XCTestCase { [FileExport.BundleEntry(name: "report.txt", data: Data(line.utf8))]).first!.data, encoding: .utf8)! XCTAssertTrue(out.contains("\"deviceId\":\"my-whoop\""), "a non-sidecar logical deviceId must be left readable") } + + // MARK: - generation merge (the capture-coverage defect) + + /// END-TO-END over real files on disk, reproducing the layout that broke the export on + /// 2026-09-05: the live file holds the morning drain (small), while a rolled generation holds the + /// previous day's post-reinstall backfill (large). Under the old largest-wins rule the bundle + /// shipped the BACKFILL and the night's own drain never left the device — the shipped + /// `oura-raw.jsonl` spanned `2026-09-04 10:58 → 12:03 UTC` against a night that ended that + /// morning. The merged entry must now contain BOTH, with the morning drain LAST so the cap's + /// keep-the-tail trim can never be the thing that drops it. + func testOuraDiagnosticEntriesMergesGenerationsNewestLast() throws { + let dir = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("oura-gen-merge-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: dir) } + + let morning = "{\"utc\":1788627494,\"iso\":\"2026-09-05T07:08:14Z\",\"hex\":\"5c08\"}\n" + let backfill = String(repeating: "{\"utc\":1788519502,\"iso\":\"2026-09-04T10:58:22Z\"}\n", + count: 50) + try morning.write(to: dir.appendingPathComponent("oura-raw-RING1.jsonl"), + atomically: true, encoding: .utf8) + try backfill.write(to: dir.appendingPathComponent("oura-raw-RING1.jsonl.1"), + atomically: true, encoding: .utf8) + + let entries = TestBundleAssembler.ouraDiagnosticEntries(directory: dir) + XCTAssertEqual(entries.map(\.name), ["oura-raw.jsonl"], "one entry, ring id dropped") + let text = try XCTUnwrap(String(data: entries[0].data, encoding: .utf8)) + XCTAssertTrue(text.contains("2026-09-04T10:58:22Z"), "older generation must still be present") + XCTAssertTrue(text.hasSuffix(morning), "the newest capture must be LAST — the cap keeps the tail") + XCTAssertLessThan(try XCTUnwrap(text.range(of: "2026-09-04T10:58:22Z")).lowerBound, + try XCTUnwrap(text.range(of: "2026-09-05T07:08:14Z")).lowerBound, + "chronological order") + } + + /// A generation whose last write did not end in a newline must not splice its final record onto + /// the next generation's first one — that would produce a line parsing as neither. + func testOuraDiagnosticEntriesInsertsANewlineBetweenGenerations() throws { + let dir = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("oura-gen-nl-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: dir) } + + try "{\"a\":1}".write(to: dir.appendingPathComponent("oura-raw-R.jsonl.1"), // NO trailing newline + atomically: true, encoding: .utf8) + try "{\"b\":2}\n".write(to: dir.appendingPathComponent("oura-raw-R.jsonl"), + atomically: true, encoding: .utf8) + + let entries = TestBundleAssembler.ouraDiagnosticEntries(directory: dir) + let text = try XCTUnwrap(String(data: entries[0].data, encoding: .utf8)) + XCTAssertEqual(text, "{\"a\":1}\n{\"b\":2}\n") + XCTAssertEqual(text.split(separator: "\n").count, 2, "two whole lines, nothing spliced") + } + + /// Every merged line must survive the redact + cap pass the bundle actually applies, and the + /// newest record must be the one that survives a cap far smaller than the merged size. + func testMergedSidecarKeepsTheNewestRecordsUnderTheCap() throws { + let dir = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("oura-gen-cap-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: dir) } + + try String(repeating: "{\"old\":1}\n", count: 4000) + .write(to: dir.appendingPathComponent("oura-raw-R.jsonl.1"), atomically: true, encoding: .utf8) + try String(repeating: "{\"new\":1}\n", count: 100) + .write(to: dir.appendingPathComponent("oura-raw-R.jsonl"), atomically: true, encoding: .utf8) + + let merged = TestBundleAssembler.ouraDiagnosticEntries(directory: dir) + let (capped, truncated) = TestBundleAssembler.capEntries(merged, capBytes: 2_000) + XCTAssertTrue(truncated) + let text = try XCTUnwrap(String(data: capped[0].data, encoding: .utf8)) + XCTAssertTrue(text.contains("{\"new\":1}"), "the newest records must survive the cap") + // Every kept line must be whole — the front trim snaps to a line boundary, so the partial + // record the raw byte-count tail would otherwise start on is dropped. + for line in text.split(separator: "\n") { + XCTAssertTrue(line.hasPrefix("{") && line.hasSuffix("}"), "no partial line: \(line)") + } + } + } From cf1528345abbc8b1e97784bb017ceeba04088f97 Mon Sep 17 00:00:00 2001 From: Pipiche Date: Sat, 12 Sep 2026 21:57:19 +0200 Subject: [PATCH 7/7] test(diagnostics): model the bulky sidecar as cva-ppg, not the spo2 dump this branch does not carry The four preceding commits were cherry-picked from integration/oura-full, where an SpO2 raw-sample research sidecar exists. It was deliberately dropped from the upstream split (fix/oura-spo2-drop-dc-raw / #2152 carries only the persist-time filter), so on this branch "spo2" is not a registered sidecar kind: the fixture name "oura-spo2.jsonl" fell through normalizedOuraEntryName (nil) and was treated as untrimmable by capEntries, failing two tests for a reason unrelated to what they assert. Swap the fixture to the CVA-PPG dump, which this branch does register and which is the bulky sidecar in practice. The rules under test (generation normalisation, max-min fair capping, merge-plan ordering) are kind-agnostic - nothing about the assertions changes. Co-Authored-By: Claude Opus 5 --- .../OuraSidecarGenerationsTests.swift | 6 +++--- StrandTests/TestBundleAssemblerTests.swift | 16 +++++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/OuraSidecarGenerationsTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/OuraSidecarGenerationsTests.swift index 8eb767f59a..7ea6ff4727 100644 --- a/Packages/WhoopStore/Tests/WhoopStoreTests/OuraSidecarGenerationsTests.swift +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/OuraSidecarGenerationsTests.swift @@ -6,7 +6,7 @@ import XCTest /// no default CI job compiles — this is the only automated cover the rule has. final class OuraSidecarGenerationsTests: XCTestCase { - private let kinds = ["raw", "spo2", "motion", "activity"] + private let kinds = ["raw", "cva-ppg", "motion", "activity"] // MARK: - classify @@ -70,9 +70,9 @@ final class OuraSidecarGenerationsTests: XCTestCase { func testMergePlanOrdersEntriesByKindsNotByDiscovery() { let plan = OuraSidecarGenerations.mergePlan( - files: [("oura-motion-A.jsonl", 1), ("oura-raw-A.jsonl", 1), ("oura-spo2-A.jsonl", 1)], + files: [("oura-motion-A.jsonl", 1), ("oura-raw-A.jsonl", 1), ("oura-cva-ppg-A.jsonl", 1)], kinds: kinds, ceilingBytes: 1_000) - XCTAssertEqual(plan.map(\.entryName), ["oura-raw.jsonl", "oura-spo2.jsonl", "oura-motion.jsonl"]) + XCTAssertEqual(plan.map(\.entryName), ["oura-raw.jsonl", "oura-cva-ppg.jsonl", "oura-motion.jsonl"]) } func testMergePlanSkipsKindsWithNoFiles() { diff --git a/StrandTests/TestBundleAssemblerTests.swift b/StrandTests/TestBundleAssemblerTests.swift index 7376c6cbb5..90c97cfe45 100644 --- a/StrandTests/TestBundleAssemblerTests.swift +++ b/StrandTests/TestBundleAssemblerTests.swift @@ -119,7 +119,7 @@ final class TestBundleAssemblerTests: XCTestCase { forFile: "oura-raw-oura-2H3B2405003655" + suffix), "oura-raw.jsonl", suffix) } XCTAssertEqual(TestBundleAssembler.normalizedOuraEntryName( - forFile: "oura-spo2-oura-2H3B2405003655.jsonl.3"), "oura-spo2.jsonl") + forFile: "oura-cva-ppg-oura-2H3B2405003655.jsonl.3"), "oura-cva-ppg.jsonl") // A non-numeric or empty tail is NOT a generation — those are someone else's files, not ours. XCTAssertNil(TestBundleAssembler.normalizedOuraEntryName( forFile: "oura-raw-oura-2H3B2405003655.jsonl.bak")) @@ -163,27 +163,29 @@ final class TestBundleAssemblerTests: XCTestCase { } /// The bug this allocation exists to prevent (measured on `noop-master-iOS-v9.3.1-260809-0716.zip`): - /// proportional-to-size handed 53.9 % of the budget to the bulk SpO2 dump and left the raw wire capture + /// proportional-to-size handed 53.9 % of the budget to a bulk sidecar dump and left the raw wire capture /// — the only sidecar a protocol fact can be re-derived from — with 1.9 %, i.e. 8 min of an 8.4 h night. /// Max-min fair keeps a modest stream WHOLE and takes the bytes off the stream that is merely bulky. + /// (The bulky neighbour is modelled here as the CVA-PPG dump — the measured case was a SpO2 research + /// sidecar that this branch does not carry; the allocation rule is kind-agnostic.) func testSmallHighValueSidecarSurvivesABulkyNeighbour() { let report = FileExport.BundleEntry(name: "report.txt", data: Data("small".utf8)) // Mirrors the real shape: one 40 MB bulk dump beside a 2 MB wire capture, 20 MB of room. - let spo2 = FileExport.BundleEntry(name: "oura-spo2.jsonl", + let bulk = FileExport.BundleEntry(name: "oura-cva-ppg.jsonl", data: Data(String(repeating: "s\n", count: 20 * 1024 * 1024).utf8)) let raw = FileExport.BundleEntry(name: "oura-raw.jsonl", data: Data(String(repeating: "r\n", count: 1024 * 1024).utf8)) let cap = 20 * 1024 * 1024 - let (capped, truncated) = TestBundleAssembler.capEntries([report, spo2, raw], capBytes: cap) + let (capped, truncated) = TestBundleAssembler.capEntries([report, bulk, raw], capBytes: cap) XCTAssertTrue(truncated) XCTAssertLessThanOrEqual(capped.reduce(0) { $0 + $1.data.count }, cap) // The wire capture is under its ~10 MB fair share, so it ships WHOLE — not scaled down to ~9 % of // the budget the way size-proportional splitting would have left it. XCTAssertEqual(capped.first { $0.name == "oura-raw.jsonl" }?.data.count, raw.data.count) // Its surplus rolls forward: the bulk dump gets everything the raw capture did not need. - let cappedSpo2 = capped.first { $0.name == "oura-spo2.jsonl" }! - XCTAssertLessThan(cappedSpo2.data.count, spo2.data.count) - XCTAssertGreaterThan(cappedSpo2.data.count, cap - raw.data.count - 1024) + let cappedBulk = capped.first { $0.name == "oura-cva-ppg.jsonl" }! + XCTAssertLessThan(cappedBulk.data.count, bulk.data.count) + XCTAssertGreaterThan(cappedBulk.data.count, cap - raw.data.count - 1024) } func testFairAllowancesIsWaterFillingAndNeverBreachesBudget() {