test(protocol): extend the golden decoder oracle past frame decode — and fix the four u32 narrowings it found

Packages/WhoopProtocol/Sources/WhoopProtocol/HistoricalStreams.swift

@@ -122,7 +122,16 @@ public func extractHistoricalStreams(_ parsed: [ParsedFrame],
                                      // The pure package can't read prefs, so the app-layer caller (Backfiller /
                                      // archive replay) reads PuffinExperiment.ppgHrSubLagInterpEnabled and passes
                                      // it. Default false = byte-identical to today. Mirrors the Android arg.
-                                     subLagInterp: Bool = false) -> Streams {
+                                     subLagInterp: Bool = false,
+                                     // TEST SEAM (#547 gate's "now"): overrides the live-clock upper bound
+                                     // resolved just below. nil everywhere in production — every caller keeps
+                                     // the live clock and today's behaviour is byte-identical. It exists so a
+                                     // fixture can pin a record whose own timestamp is in the future relative
+                                     // to the machine running the test, which is the only way to cover the
+                                     // post-2038 (bit-31-set) unix domain at all. Android's
+                                     // `extractHistoricalStreams` already had this parameter; this is Swift
+                                     // catching up, so the two oracle harnesses can drive the gate identically.
+                                     wallNow wallNowOverride: Int? = nil) -> Streams {
     func wall(_ deviceTs: Int?) -> Int? {
         guard let d = deviceTs else { return nil }
         return wallClockRef + (d - deviceClockRef)
@@ -144,7 +153,7 @@ public func extractHistoricalStreams(_ parsed: [ParsedFrame],
     // nor a paused live clock wrongly rejects a real record. The MIN_PLAUSIBLE_UNIX floor is unconditional
     // and still catches the far-past garbage in every caller. Genuine future garbage (pikapik's records
     // dated beyond now, the field's year-2081 overshoot) is still > now + FUTURE_MARGIN → dropped. (#547)
-    let wallNow = max(wallClockRef, Int(Date().timeIntervalSince1970))
+    let wallNow = wallNowOverride ?? max(wallClockRef, Int(Date().timeIntervalSince1970))
     // PRIMARY FIX (#547): a record's own decoded ts must be near "now". A bad-clock strap emits records
     // whose unix is scattered garbage (far-past, a bogus 2027, even future dates); trusted verbatim, one
     // polluted block was re-attributed to every day and a future row surfaced as "last night". Returns

Packages/WhoopProtocol/Tests/WhoopProtocolTests/DecoderOracleTests.swift

@@ -13,12 +13,80 @@ import XCTest
 /// The fixture was seeded from existing in-repo test vectors (Whoop4HistoricalV24HardwareTests,
 /// Whoop4HistoricalV25Tests, Whoop5HistoricalTests and their Kotlin twins), so every expected value is
 /// already independently grounded , this test only proves the two decoders agree on it.
+///
+/// The oracle pins THREE layers, each with a twin test in `DecoderOracleTest.kt` (#647, #775):
+///  1. `frames`         , decoded VALUES. A per-platform fixture-hex test cannot catch a 32-vs-64-bit
+///                        or signedness divergence: the wire bytes are identical on both platforms and
+///                        each suite asserts its own answer. Comparing the decoded NUMBERS against one
+///                        shared expectation is what catches it (PR #848: a u32 with bit 31 set read
+///                        3232194973 on Swift and -1062772323 on Kotlin).
+///  2. `stream_batches` , the ASSEMBLED shape. Frame decode agreeing does not imply assembly agrees ,
+///                        PR #848 also shipped a Kotlin `StreamBatch.isEmpty` that omitted a stream
+///                        Swift's `Streams.isEmpty` included, and `insert` early-returns on empty, so
+///                        that batch banked nothing on Android alone.
+///  3. `coverage`       , the self-defence manifest, so the oracle cannot silently stop covering
+///                        something. Deleting an `expect` key, a frame or a batch fails a test.
 final class DecoderOracleTests: XCTestCase {
 
     // Mirrors the JSON shape. `expect` is a heterogeneous map decoded leniently below.
     private struct Oracle: Decodable {
         let tolerance: Double
+        let coverage: Coverage
         let frames: [Frame]
+        let streamBatches: [StreamBatchCase]
+        private enum CodingKeys: String, CodingKey {
+            case tolerance, coverage, frames
+            case streamBatches = "stream_batches"
+        }
+    }
+    /// The self-defence manifest , see the `coverage._comment` block in the JSON.
+    private struct Coverage: Decodable {
+        /// Pinned decoder key -> how many fixture frames assert it. The COUNT, not just the name: a set
+        /// alone would let one frame quietly drop a key another frame still carries.
+        let fields: [String: Int]
+        let derivedFields: [String]
+        let frames: [String]
+        let batches: [String]
+        let streams: [String]
+        let nonStreamLists: [String]
+        private enum CodingKeys: String, CodingKey {
+            case fields, frames, batches, streams
+            case derivedFields = "derived_fields"
+            case nonStreamLists = "non_stream_lists"
+        }
+    }
+    /// One pinned stream-assembly case: named fixture frames in, one pinned `Streams` shape out.
+    private struct StreamBatchCase: Decodable {
+        let name: String
+        let family: String
+        let frames: [String]
+        let deviceClockRef: Int
+        let wallClockRef: Int
+        let sessionOldestUnix: Int?
+        let sessionNewestUnix: Int?
+        /// Optional #547 future-bound override. Absent → the production default (the live clock).
+        /// A batch whose records are post-2038 MUST set it, or the live clock rejects them for an
+        /// unrelated reason and the batch asserts nothing. Kotlin passes the same value.
+        let wallNow: Int?
+        let expect: BatchExpect
+        private enum CodingKeys: String, CodingKey {
+            case name, family, frames, expect
+            case deviceClockRef = "device_clock_ref"
+            case wallClockRef = "wall_clock_ref"
+            case sessionOldestUnix = "session_oldest_unix"
+            case sessionNewestUnix = "session_newest_unix"
+            case wallNow = "wall_now"
+        }
+    }
+    private struct BatchExpect: Decodable {
+        let isEmpty: Bool
+        let droppedImplausible: Int
+        let counts: [String: Int]
+        private enum CodingKeys: String, CodingKey {
+            case counts
+            case isEmpty = "is_empty"
+            case droppedImplausible = "dropped_implausible"
+        }
     }
     private struct Frame: Decodable {
         let name: String
@@ -106,6 +174,167 @@ final class DecoderOracleTests: XCTestCase {
         }
     }
 
+    // MARK: - Layer 2: stream assembly
+
+    /// Per-stream row counts + the emptiness verdict for each pinned batch. Frame decode agreeing does
+    /// not imply assembly agrees: `extractHistoricalStreams` gates each stream separately (bpm 0 writes
+    /// no HR row, skin temp has a thermal gate, v25 carries gravity but no HR), and the emptiness verdict
+    /// is what `insert` early-returns on. Every stream is pinned INCLUDING the zeros, so a stream that
+    /// materialises on one platform only is a failure rather than an unasserted extra.
+    func testOracleStreamBatchesAssembleToExpectedShape() throws {
+        let oracle = try loadOracle()
+        XCTAssertGreaterThan(oracle.streamBatches.count, 0, "no oracle stream batches loaded")
+        var byName: [String: Frame] = [:]
+        for f in oracle.frames { byName[f.name] = f }
+
+        for batch in oracle.streamBatches {
+            let family: DeviceFamily = batch.family == "whoop5" ? .whoop5 : .whoop4
+            let parsed: [ParsedFrame] = try batch.frames.map { name in
+                let f = try XCTUnwrap(byName[name], "\(batch.name): unknown fixture frame '\(name)'")
+                return parseFrame(hexToBytes(f.hex), family: family)
+            }
+            let s = extractHistoricalStreams(parsed, deviceClockRef: batch.deviceClockRef,
+                                             wallClockRef: batch.wallClockRef,
+                                             sessionOldestUnix: batch.sessionOldestUnix,
+                                             sessionNewestUnix: batch.sessionNewestUnix,
+                                             wallNow: batch.wallNow)
+            let got = Self.streamCounts(s)
+            for (stream, want) in batch.expect.counts {
+                let have = try XCTUnwrap(got[stream], "\(batch.name): unknown stream '\(stream)'")
+                XCTAssertEqual(have, want, "\(batch.name).\(stream) row count")
+            }
+            XCTAssertEqual(s.isEmpty, batch.expect.isEmpty, "\(batch.name): emptiness verdict")
+            XCTAssertEqual(s.droppedImplausible, batch.expect.droppedImplausible,
+                           "\(batch.name): #547 dropped-record count")
+            // The verdict must also AGREE with the counts, so a platform whose isEmpty forgets a stream
+            // is caught even on a batch that wasn't written to target that stream.
+            XCTAssertEqual(s.isEmpty, got.values.allSatisfy { $0 == 0 },
+                           "\(batch.name): isEmpty disagrees with the per-stream counts")
+        }
+    }
+
+    /// Row count per stream, keyed by the oracle's (and `Streams.CodingKeys`') wire names.
+    private static func streamCounts(_ s: Streams) -> [String: Int] {
+        [
+            "hr": s.hr.count, "rr": s.rr.count, "spo2": s.spo2.count, "skin_temp": s.skinTemp.count,
+            "resp": s.resp.count, "gravity": s.gravity.count, "steps": s.steps.count,
+            "sleep_state": s.sleepState.count, "ppg_hr": s.ppgHr.count,
+            "ppg_waveform": s.ppgWaveform.count, "v18_aux": s.v18Aux.count,
+            "events": s.events.count, "battery": s.battery.count,
+        ]
+    }
+
+    /// EVERY stream on its own makes the batch non-empty. This is the exhaustive form of the #848 bug:
+    /// Kotlin's `StreamBatch.isEmpty` omitted one stream, and because `insert` early-returns on an empty
+    /// batch, a batch carrying ONLY that stream banked nothing on Android. A batch fixture can only catch
+    /// that for the streams it happens to populate; constructing a one-stream batch per stream catches it
+    /// for all of them. The cases are keyed by the oracle's stream names and cross-checked against the
+    /// manifest, so a stream added to `Streams` without a case here fails `testDeclaredStreams…` below.
+    func testEmptinessVerdictCoversEveryStream() throws {
+        let oracle = try loadOracle()
+        let oneOf: [String: Streams] = [
+            "hr": Streams(hr: [HRSample(ts: 1, bpm: 60)]),
+            "rr": Streams(rr: [RRInterval(ts: 1, rrMs: 900)]),
+            "spo2": Streams(spo2: [SpO2Sample(ts: 1, red: 1, ir: 1)]),
+            "skin_temp": Streams(skinTemp: [SkinTempSample(ts: 1, raw: 3000)]),
+            "resp": Streams(resp: [RespSample(ts: 1, raw: 3000)]),
+            "gravity": Streams(gravity: [GravitySample(ts: 1, x: 0, y: 0, z: 1)]),
+            "steps": Streams(steps: [StepSample(ts: 1, counter: 1)]),
+            "sleep_state": Streams(sleepState: [SleepStateSample(ts: 1, state: 2)]),
+            "ppg_hr": Streams(ppgHr: [PpgHrSample(ts: 1, bpm: 60, conf: 0.5)]),
+            "ppg_waveform": Streams(ppgWaveform: [PpgWaveformSample(ts: 1, samples: [1, 2])]),
+            // Carries a real slot value rather than a bare `ts`: `V18AuxSample.isEmpty` is "every slot is
+            // nil", so an all-nil sample would still make the ARRAY non-empty and pass this check while
+            // saying nothing about a row that carries data.
+            "v18_aux": Streams(v18Aux: [V18AuxSample(ts: 1, recordIndex: 1)]),
+            "events": Streams(events: [WhoopEvent(ts: 1, kind: "BOOT", payload: [:])]),
+            "battery": Streams(battery: [BatterySample(ts: 1, soc: 50, mv: 3900)]),
+        ]
+        XCTAssertEqual(Set(oneOf.keys), Set(oracle.coverage.streams),
+                       "one-stream cases and the oracle's stream manifest disagree")
+        XCTAssertTrue(Streams().isEmpty, "a Streams with no rows must be empty")
+        for (stream, s) in oneOf {
+            XCTAssertFalse(s.isEmpty, "Streams carrying only '\(stream)' must NOT be empty , "
+                           + "isEmpty gates the insert, so a stream missing from the verdict is silent data loss")
+            XCTAssertEqual(Self.streamCounts(s).values.reduce(0, +), 1,
+                           "'\(stream)' case must populate exactly one row in exactly one stream")
+        }
+    }
+
+    // MARK: - Layer 3: the oracle defends itself
+
+    /// The manifest and the fixtures must agree EXACTLY, in both directions. Without this, deleting an
+    /// `expect` key, a whole frame or a whole batch passes both suites and coverage silently shrinks.
+    /// Precedent: PR #848 needed a test that every storage slot's decoder key exists in a real decode,
+    /// because a rename had broken an extractor and nothing failed.
+    func testOracleCoverageManifestMatchesFixtures() throws {
+        let oracle = try loadOracle()
+        let cov = oracle.coverage
+
+        XCTAssertEqual(oracle.frames.map(\.name), cov.frames,
+                       "coverage.frames must list every fixture frame, in order")
+        XCTAssertEqual(Set(oracle.streamBatches.map(\.name)), Set(cov.batches),
+                       "coverage.batches must list every stream batch")
+
+        // Field -> how many frames assert it. Counting (not just naming) is what makes EVERY individual
+        // assertion load-bearing: dropping `heart_rate` from one frame leaves the key set unchanged.
+        var asserted: [String: Int] = [:]
+        for f in oracle.frames { for key in f.expect.keys { asserted[key, default: 0] += 1 } }
+        XCTAssertEqual(asserted, cov.fields,
+                       "coverage.fields must name every 'expect' key and how many frames assert it")
+        XCTAssertTrue(Set(cov.derivedFields).isSubset(of: Set(cov.fields.keys)),
+                      "coverage.derived_fields must be a subset of coverage.fields")
+
+        // Every pinned batch pins EVERY stream, zeros included , otherwise "absent on one platform"
+        // and "not asserted" are indistinguishable.
+        for batch in oracle.streamBatches {
+            XCTAssertEqual(Set(batch.expect.counts.keys), Set(cov.streams),
+                           "\(batch.name): counts must pin every stream in coverage.streams")
+            XCTAssertEqual(batch.expect.isEmpty, batch.expect.counts.values.allSatisfy { $0 == 0 },
+                           "\(batch.name): pinned is_empty contradicts the pinned counts")
+        }
+
+        // Every non-derived pinned field must appear in at least one REAL decode. A field that no
+        // decoder emits any more would otherwise sit in the manifest looking covered.
+        let derived = Set(cov.derivedFields)
+        var seen = Set<String>()
+        for f in oracle.frames {
+            let family: DeviceFamily = f.family == "whoop5" ? .whoop5 : .whoop4
+            let parsed = parseFrame(hexToBytes(f.hex), family: family).parsed
+            seen.formUnion(parsed.keys)
+        }
+        for field in cov.fields.keys where !derived.contains(field) {
+            XCTAssertTrue(seen.contains(field),
+                          "coverage.fields lists '\(field)' but no fixture decode emits that key")
+        }
+    }
+
+    /// The oracle's stream manifest must equal the array-typed fields `Streams` actually declares. This
+    /// is the direction the manifest alone cannot cover: a NEW stream added to `Streams` (and to the
+    /// emptiness verdict) but never given oracle counts would otherwise be invisible to every check
+    /// above. Diagnostics that are arrays but deliberately excluded from the verdict are listed in
+    /// `coverage.non_stream_lists`. The Kotlin twin reflects `StreamBatch` the same way.
+    func testDeclaredStreamsMatchOracleManifest() throws {
+        let oracle = try loadOracle()
+        let declared = Mirror(reflecting: Streams()).children.compactMap { child -> String? in
+            guard let label = child.label, child.value as? [Any] != nil else { return nil }
+            return Self.snakeCased(label)
+        }
+        XCTAssertEqual(Set(declared),
+                       Set(oracle.coverage.streams).union(oracle.coverage.nonStreamLists),
+                       "Streams declares array fields the oracle does not account for (or vice versa) , "
+                       + "add the new stream to coverage.streams and to every batch's counts")
+    }
+
+    /// `sleepState` -> `sleep_state`. Digits never take a separator, so `spo2` stays `spo2`.
+    private static func snakeCased(_ s: String) -> String {
+        var out = ""
+        for ch in s {
+            if ch.isUppercase { out.append("_"); out.append(Character(ch.lowercased())) } else { out.append(ch) }
+        }
+        return out
+    }
+
     /// The Swift and Android copies of the oracle MUST be byte-identical, so neither platform can edit
     /// its fixture without the other. Compares the bundled Swift resource against the Android source
     /// copy (located relative to this test file). Skips gracefully if the Android tree isn't present