diff --git a/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift b/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift index 12655a2244..3280b3b4d4 100644 --- a/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift +++ b/Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift @@ -37,7 +37,8 @@ public enum StandardHRContact: String, Equatable, Codable, Sendable { /// The sensor channel or transport that produced an R-R interval. /// /// WHOOP 5 exposes one beat train over multiple transports; codes 5–7 distinguish those observations. -/// WHOOP 4 standard-BLE rows and legacy data keep nil; its type-47 historical rows use code 8. An Oura ring has more than one optical channel: +/// WHOOP 4 type-40 realtime and standard-BLE rows use codes 9 and 10; legacy data keeps nil. +/// Its type-47 historical rows use code 8. An Oura ring has more than one optical channel: /// the green-quality tag (0x80) and the SpO2 tag (0x6E) both /// decode to R-R and both were stored, so the table held roughly TWO complete copies of every night — /// not duplicate rows to de-duplicate, but the SAME heartbeats measured twice. Labelling the channel is @@ -73,6 +74,10 @@ public enum RRSourceChannel: Int, Equatable, Codable, Sendable, CaseIterable { /// WHOOP 4.0 type-47 historical R-R intervals. The 4.0 standard BLE feed has no source label, /// so this distinguishes the clock-anchored offload from the overlapping live feed at read time. case whoop4Historical = 8 + /// WHOOP 4.0 type-40 live R-R intervals, timestamped from the strap's realtime record clock. + case whoop4Realtime = 9 + /// WHOOP 4.0 standard BLE 0x2A37 R-R intervals, timestamped on notification receipt. + case whoop4Standard = 10 public var isWhoop5Transport: Bool { (5...7).contains(rawValue) } @@ -825,8 +830,13 @@ private func toWall(_ deviceTs: Int?, _ deviceClockRef: Int, _ wallClockRef: Int /// carries an HR byte but streams alongside type-40 during raw collection, so routing both /// would double-count HR for the same instants. Frames whose integrity verdict is negative are /// skipped — no row is derived from a frame that is not intact. +/// +/// Kotlin twin: `extractStreams`. Adding `family` changed the arity on both sides at once, which +/// unpaired them in the ledger even though they moved together; the reference says they are one +/// contract so the next parameter does not have to rediscover that. public func extractStreams(_ parsed: [ParsedFrame], - deviceClockRef: Int, wallClockRef: Int) -> Streams { + deviceClockRef: Int, wallClockRef: Int, + family: DeviceFamily? = nil) -> Streams { var out = Streams() for r in parsed { // `ok` is now the FULL verdict — header checksum, payload CRC32 and structural length — so it @@ -844,6 +854,7 @@ public func extractStreams(_ parsed: [ParsedFrame], // Unlike Python, drop RR rows when timestamp is absent (a ts-less RR row is unstorable). if let ts = ts, let rrs = p["rr_intervals"]?.intArrayValue { let source = p["rr_source_channel"]?.intValue.flatMap(RRSourceChannel.init(rawValue:)) + ?? (family == .whoop4 ? .whoop4Realtime : nil) for rr in rrs { out.rr.append(RRInterval(ts: ts, rrMs: rr, srcChannel: source)) } } case "EVENT": diff --git a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/StreamsTests.swift b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/StreamsTests.swift index 97d932921c..866cfd9988 100644 --- a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/StreamsTests.swift +++ b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/StreamsTests.swift @@ -41,6 +41,18 @@ final class StreamsTests: XCTestCase { XCTAssertTrue(s.rr.isEmpty) } + func testRealtimeRRSourceIsTaggedOnlyWhenFamilyIsKnown() { + let frame = ParsedFrame(ok: true, typeName: "REALTIME_DATA", seq: nil, cmdName: nil, + crcOK: true, lenBytes: 0, rawHex: "", fields: [], + parsed: ["timestamp": .int(deviceClockRef), + "rr_intervals": .intArray([800])]) + let tagged = extractStreams([frame], deviceClockRef: deviceClockRef, + wallClockRef: wallClockRef, family: .whoop4) + XCTAssertEqual(tagged.rr.map(\.srcChannel), [.whoop4Realtime]) + let legacy = extractStreams([frame], deviceClockRef: deviceClockRef, wallClockRef: wallClockRef) + XCTAssertNil(legacy.rr.first?.srcChannel) + } + func testEventTimestampIsNotOffset() { let s = extractStreams(parsedFrames([ev]), deviceClockRef: deviceClockRef, wallClockRef: wallClockRef) diff --git a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5RRTests.swift b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5RRTests.swift index 42c8eb9dfc..2479d1b80c 100644 --- a/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5RRTests.swift +++ b/Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5RRTests.swift @@ -50,7 +50,7 @@ final class Whoop5RRTests: XCTestCase { } func testTransportCodes() { - XCTAssertEqual(RRSourceChannel.allCases.map(\.rawValue), Array(1...8)) + XCTAssertEqual(RRSourceChannel.allCases.map(\.rawValue), Array(1...10)) } /// The table below is the parity contract for the "this night cannot be scored" explanation. The diff --git a/Packages/WhoopStore/Sources/WhoopStore/Reads.swift b/Packages/WhoopStore/Sources/WhoopStore/Reads.swift index 689b503b8d..378ac6c844 100644 --- a/Packages/WhoopStore/Sources/WhoopStore/Reads.swift +++ b/Packages/WhoopStore/Sources/WhoopStore/Reads.swift @@ -231,7 +231,7 @@ extension WhoopStore { } let historyCount: Int = row["w5"] let registry: String = row["registry"] - return "v4|h\(hc):\(hm)|" + tails.joined(separator: "|") + return "v5|h\(hc):\(hm)|" + tails.joined(separator: "|") + "|w5\(historyCount)|w7\(row["w7"] as Int)|tagged\(row["w5tagged"] as Int)" + "|w4history\(row["w4history"] as Int)|registry\(registry)" } @@ -311,7 +311,7 @@ extension WhoopStore { let historyCount: Int = row["w5"] let registry: String = row["registry"] let strictRR = try Self.isWhoop5RRSource(db: db, deviceId: deviceId) - return "s3|" + parts.joined(separator: "|") + "|w5\(historyCount)|w7\(row["w7"] as Int)" + return "s4|" + parts.joined(separator: "|") + "|w5\(historyCount)|w7\(row["w7"] as Int)" + "|w4h\(row["w4h"] as Int)|ownerTagged\(row["w5owner"] as Int)|registry\(registry)|rr5=\(strictRR)" } } @@ -466,8 +466,9 @@ extension WhoopStore { // One transport for the complete requested interval. Legacy WHOOP 5 rows mix units and // origins, so they remain stored but cannot be converted or spliced into a scored beat train. // This subquery uses the SAME time/suspect predicates as the outer read, before LIMIT. - // A WHOOP 4 takes the history branch below: its type-47 offload is labelled, its standard-BLE - // feed is not, and the two overlap, so one or the other is scored for the whole interval. + // A WHOOP 4 has labelled type-47 history, type-40 realtime, standard-BLE, and legacy + // unlabelled rows. Choose one source per UTC hour in provenance order, so overlapping live + // transports are never merged and partial history takes precedence only where it exists. // Every other source takes the Oura branch: one beat channel for the interval, the fuller one // (see the doc comment on `rrIntervals`), with NULL and non-Oura codes passing untouched. let strictWhoop4History: Bool @@ -484,29 +485,61 @@ extension WhoopStore { srcChannel = (SELECT MIN(srcChannel) FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t AND srcChannel IN \(Self.scorableWhoop5Channels) AND (tsSuspect IS NULL OR tsSuspect <> 1)) - """ : (strictWhoop4History ? """ - ((EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t - AND srcChannel = :whoop4Historical AND (tsSuspect IS NULL OR tsSuspect <> 1)) - AND srcChannel = :whoop4Historical) - OR (NOT EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t - AND srcChannel = :whoop4Historical AND (tsSuspect IS NULL OR tsSuspect <> 1)) - AND srcChannel IS NULL)) """ : """ (srcChannel IS NULL OR srcChannel NOT IN \(Self.scorableOuraChannels) OR (srcChannel = 1) = ( SELECT SUM(srcChannel = 1) > SUM(srcChannel <> 1) FROM rrInterval WHERE deviceId = :d AND ts >= :f AND ts <= :t AND srcChannel IN \(Self.scorableOuraChannels) AND (tsSuspect IS NULL OR tsSuspect <> 1))) - """) - let rows = try Row.fetchAll(db, sql: """ - SELECT ts, rrMs, srcChannel, ord, seq FROM rrInterval - WHERE deviceId = :d AND ts >= :f AND ts <= :t - AND (srcChannel IS NULL OR srcChannel <> :rrx) - AND \(sourcePredicate) - AND (tsSuspect IS NULL OR tsSuspect <> 1) -- #1073: exclude future-stamped beats - ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :lim - """, arguments: ["d": deviceId, "f": from, "t": to, - "rrx": RRSourceChannel.spo2Ibi.rawValue, - "whoop4Historical": RRSourceChannel.whoop4Historical.rawValue, "lim": limit]) + """ + // The WHOOP 4 source decision is per UTC hour. Materialize one choice per hour, then join + // that small result to the requested beats. Counting the same hour from every beat made + // wide HRV reads quadratic in the number of rows per hour and could stall metric screens. + let sql: String + if strictWhoop4History { + sql = """ + WITH whoop4HourChoice AS MATERIALIZED ( + SELECT ts / 3600 AS hour, + MIN(CASE WHEN srcChannel = :whoop4Historical THEN 1 + WHEN srcChannel = :whoop4Realtime THEN 2 + WHEN srcChannel = :whoop4Standard THEN 3 + WHEN srcChannel IS NULL THEN 4 END) AS sourceChoice + FROM rrInterval + WHERE deviceId = :d + AND ts >= (:f / 3600) * 3600 + AND ts < ((:t / 3600) + 1) * 3600 + AND (tsSuspect IS NULL OR tsSuspect <> 1) + AND (srcChannel IS NULL OR srcChannel IN + (:whoop4Historical, :whoop4Realtime, :whoop4Standard)) + GROUP BY ts / 3600 + ) + SELECT r.ts, r.rrMs, r.srcChannel, r.ord, r.seq FROM rrInterval r + JOIN whoop4HourChoice h ON h.hour = r.ts / 3600 + WHERE r.deviceId = :d AND r.ts >= :f AND r.ts <= :t + AND (r.srcChannel IS NULL OR r.srcChannel <> :rrx) + AND ((r.srcChannel = :whoop4Historical AND h.sourceChoice = 1) + OR (r.srcChannel = :whoop4Realtime AND h.sourceChoice = 2) + OR (r.srcChannel = :whoop4Standard AND h.sourceChoice = 3) + OR (r.srcChannel IS NULL AND h.sourceChoice = 4)) + AND (r.tsSuspect IS NULL OR r.tsSuspect <> 1) + ORDER BY r.ts ASC, r.ord ASC, r.rrMs ASC, r.seq ASC LIMIT :lim + """ + } else { + sql = """ + SELECT ts, rrMs, srcChannel, ord, seq FROM rrInterval + WHERE deviceId = :d AND ts >= :f AND ts <= :t + AND (srcChannel IS NULL OR srcChannel <> :rrx) + AND \(sourcePredicate) + AND (tsSuspect IS NULL OR tsSuspect <> 1) -- #1073: exclude future-stamped beats + ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :lim + """ + } + let rows = try Row.fetchAll(db, sql: sql, + arguments: ["d": deviceId, "f": from, "t": to, + "rrx": RRSourceChannel.spo2Ibi.rawValue, + "whoop4Historical": RRSourceChannel.whoop4Historical.rawValue, + "whoop4Realtime": RRSourceChannel.whoop4Realtime.rawValue, + "whoop4Standard": RRSourceChannel.whoop4Standard.rawValue, + "lim": limit]) .map { row in RRInterval(ts: row["ts"], rrMs: row["rrMs"], srcChannel: (row["srcChannel"] as Int?).flatMap(RRSourceChannel.init(rawValue:)), diff --git a/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift b/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift index fb88680f3e..78657a1a15 100644 --- a/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift +++ b/Packages/WhoopStore/Tests/WhoopStoreTests/RrSourceChannelTests.swift @@ -18,12 +18,16 @@ import OuraProtocol final class RrSourceChannelTests: XCTestCase { private let ts = 1_750_000_000 - func testWhoop4HistoricalSourceWinsAndPromotesLegacyLiveRows() async throws { - let store = try await WhoopStore.inMemory() - try await store.upsertDevice(id: "strap", mac: nil, name: nil) + private func setRegistry(_ store: WhoopStore) throws { try DeviceRegistryStore(dbQueue: store.registryWriter).add(PairedDevice( id: "strap", brand: "WHOOP", model: "4.0", sourceKind: .liveBLE, capabilities: [.hr, .hrv], status: .active, addedAt: ts, lastSeenAt: ts)) + } + + func testWhoop4HistoricalSourceWinsAndPromotesLegacyLiveRows() async throws { + let store = try await WhoopStore.inMemory() + try await store.upsertDevice(id: "strap", mac: nil, name: nil) + try setRegistry(store) _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 800), RRInterval(ts: ts + 1, rrMs: 810)]), deviceId: "strap") @@ -39,9 +43,7 @@ final class RrSourceChannelTests: XCTestCase { func testWhoop4FallsBackToUnlabelledRowsWhenNoHistoricalDataExists() async throws { let store = try await WhoopStore.inMemory() try await store.upsertDevice(id: "strap", mac: nil, name: nil) - try DeviceRegistryStore(dbQueue: store.registryWriter).add(PairedDevice( - id: "strap", brand: "WHOOP", model: "4.0", sourceKind: .liveBLE, - capabilities: [.hr, .hrv], status: .active, addedAt: ts, lastSeenAt: ts)) + try setRegistry(store) _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 800)]), deviceId: "strap") let selected = try await store.rrIntervals(deviceId: "strap", from: ts, to: ts, limit: 100) @@ -49,12 +51,10 @@ final class RrSourceChannelTests: XCTestCase { XCTAssertNil(selected.first?.srcChannel) } - func testWhoop4UsesOneHistoricalTransportForTheWholeWindow() async throws { + func testWhoop4HistoricalSourceHasPriorityWithinItsHour() async throws { let store = try await WhoopStore.inMemory() try await store.upsertDevice(id: "strap", mac: nil, name: nil) - try DeviceRegistryStore(dbQueue: store.registryWriter).add(PairedDevice( - id: "strap", brand: "WHOOP", model: "4.0", sourceKind: .liveBLE, - capabilities: [.hr, .hrv], status: .active, addedAt: ts, lastSeenAt: ts)) + try setRegistry(store) _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 800), RRInterval(ts: ts + 1, rrMs: 810)]), deviceId: "strap") _ = try await store.insert(Streams(rr: [RRInterval(ts: ts, rrMs: 805, srcChannel: .whoop4Historical)]), @@ -65,6 +65,46 @@ final class RrSourceChannelTests: XCTestCase { XCTAssertEqual(selected.map(\.srcChannel), [.whoop4Historical]) } + func testWhoop4RealtimeSourceWinsOverStandardAndLegacyRows() async throws { + let store = try await WhoopStore.inMemory() + try await store.upsertDevice(id: "strap", mac: nil, name: nil) + try setRegistry(store) + _ = try await store.insert(Streams(rr: [ + RRInterval(ts: ts, rrMs: 800), + RRInterval(ts: ts, rrMs: 810, srcChannel: .whoop4Standard), + RRInterval(ts: ts, rrMs: 820, srcChannel: .whoop4Realtime), + ]), deviceId: "strap") + + let selected = try await store.rrIntervals(deviceId: "strap", from: ts, to: ts, limit: 100) + XCTAssertEqual(selected.map(\.rrMs), [820]) + XCTAssertEqual(selected.map(\.srcChannel), [.whoop4Realtime]) + } + + func testWhoop4PartialHistoryKeepsUnlabelledRowsFromOtherHours() async throws { + let store = try await WhoopStore.inMemory() + try await store.upsertDevice(id: "strap", mac: nil, name: nil) + try setRegistry(store) + + let base = (ts / 3600) * 3600 + _ = try await store.insert(Streams(rr: [ + RRInterval(ts: base + 10, rrMs: 800), + RRInterval(ts: base + 3600 + 10, rrMs: 820), + RRInterval(ts: base + 3600 + 11, rrMs: 830), + RRInterval(ts: base + 3600 + 12, rrMs: 840), + ]), deviceId: "strap") + _ = try await store.insert(Streams(rr: [ + RRInterval(ts: base + 10, rrMs: 805, srcChannel: .whoop4Historical), + RRInterval(ts: base + 11, rrMs: 815, srcChannel: .whoop4Historical), + RRInterval(ts: base + 3600 + 10, rrMs: 825, srcChannel: .whoop4Historical), + ]), deviceId: "strap") + + let selected = try await store.rrIntervals(deviceId: "strap", from: base, + to: base + 7200, limit: 100) + XCTAssertEqual(selected.map(\.rrMs), [805, 815, 825]) + XCTAssertEqual(selected.map(\.srcChannel), + [.whoop4Historical, .whoop4Historical, .whoop4Historical]) + } + // MARK: - The label survives the mapping /// A 0x6E record and a 0x80 record covering the same interval must produce rows with DISTINCT diff --git a/Strand/Collect/Collector.swift b/Strand/Collect/Collector.swift index 35e9fde3fd..07fd73f3df 100644 --- a/Strand/Collect/Collector.swift +++ b/Strand/Collect/Collector.swift @@ -231,7 +231,8 @@ final class Collector { let frames = batch.map(\.frame) // still needed for the raw-capture outbox let parsed = batch.map(\.parsed) // #47: the seam already decoded these — don't re-parse - let streams = extractStreams(parsed, deviceClockRef: ref.device, wallClockRef: ref.wall) + let streams = extractStreams(parsed, deviceClockRef: ref.device, wallClockRef: ref.wall, + family: family) // #1118: the SECOND live transport. `flushStandardHR` stamps a beat at the second it arrived over // 0x2A37; this one stamps it from the strap's own record clock. The same beat reaching both lands // on two different seconds, which no same-second de-dup can collapse — the signature every @@ -291,7 +292,8 @@ final class Collector { let acceptedHR = (30...220).contains(hr) ? 1 : 0 let acceptedRR = rr.filter { (250...3000).contains($0) } if acceptedHR == 1 { stdHR.append(HRSample(ts: ts, bpm: hr)) } - let source: RRSourceChannel? = family == .whoop5 ? .whoop5Standard : nil + let source: RRSourceChannel? = family == .whoop5 ? .whoop5Standard + : (family == .whoop4 ? .whoop4Standard : nil) stdRR.append(contentsOf: acceptedRR.map { RRInterval(ts: ts, rrMs: $0, srcChannel: source) }) // Only the CHANGES. Advanced here rather than at flush because the event travels in the buffer // until it persists: a failed insert re-inserts it at the front, so nothing has to be unwound. diff --git a/StrandTests/CollectorStandardHRContactTests.swift b/StrandTests/CollectorStandardHRContactTests.swift index e8746365fd..0a39a01991 100644 --- a/StrandTests/CollectorStandardHRContactTests.swift +++ b/StrandTests/CollectorStandardHRContactTests.swift @@ -118,7 +118,11 @@ final class CollectorStandardHRContactTests: XCTestCase { await collector.flushStandardHR() XCTAssertTrue(store.inserted.isEmpty) await collector.flushStandardHR() - XCTAssertEqual(store.inserted.flatMap(\.rr).map(\.srcChannel), [.whoop5Standard, nil, nil]) + // A WHOOP 4 standard-BLE beat is labelled now (`whoop4Standard`) where it used to be left + // unlabelled: that NULL is what let a partial type-47 offload hide live intervals, so the + // assertion moves to the new shape. Only a beat with NO family stays nil. + XCTAssertEqual(store.inserted.flatMap(\.rr).map(\.srcChannel), + [.whoop5Standard, .whoop4Standard, nil]) XCTAssertEqual(store.inserted.flatMap(\.rr).map(\.rrMs), [1000, 1001, 1002]) } } diff --git a/Tools/parity_twin_map.json b/Tools/parity_twin_map.json index 88ab06b73c..6abf5498b8 100644 --- a/Tools/parity_twin_map.json +++ b/Tools/parity_twin_map.json @@ -19,15 +19,15 @@ }, "authority": { "files": {"count": 504, "sha256": "fc60ea929a4de0859ebb2fefb0302248f79c712a9f537f6050b6e02a64f485e8"}, - "functions": {"count": 4497, "sha256": "acc9b4ea67a5b44cc1b3497715d0c42bf39d378a703d71e8541a4f3d40131b81"}, + "functions": {"count": 4497, "sha256": "af2aef301c4482689bc5c4704be1f2e0c0688538511d0c8247953a350cf1f70c"}, "properties": {"count": 465, "sha256": "64fcd18c85bf33a4738d154d45ec86967fdc974e78a448d16e9504d9b13609eb"}, "constants": {"count": 1962, "sha256": "be21ed554bc51ed8d3faffce82c131b9d025f2252a8f7c5281c06736a480270a"}, "file_pairs": {"count": 75, "sha256": "d581f60241ec523c595a7e5ff123f7987caee501d8be4ebf6df5ba105e26fe1f"}, - "function_pairs": {"count": 202, "sha256": "face39eac8619a041db0d34d811065ef6d2a83a3b8f0e17b679c27ecd37e7c4a"}, + "function_pairs": {"count": 203, "sha256": "b5647a01ad1b00477ac9dcbb0a7260acf49badf9164f3d7562a0bf4b905d1b8e"}, "property_pairs": {"count": 150, "sha256": "be7168d309c8a9cf5f3f25438c028cc60ab64202bb6e9c09a6c553e3699ad0e2"}, "constant_pairs": {"count": 682, "sha256": "95b64a46a21dec593537c83be57084d2904e2de779a5672d8bdd4cf9da7fdee8"}, "unpaired_files": {"count": 376, "sha256": "654d951b8137dbb2266b83734fa1e3f1cb1648f15c1aab3d015cdef7e79deb75"}, - "unpaired_functions": {"count": 4106, "sha256": "be727bd03462055bec05379807b15686087f43df0ef4706773046fb22ff1e9b1"}, + "unpaired_functions": {"count": 4104, "sha256": "4a7e8aeaf1544374de442570f6eba5a04af48d93459a62c5c9d87311f57b6dbc"}, "unpaired_properties": {"count": 165, "sha256": "fe609dd25384cda6de1616d36753fed68431b1a1f9a73b6fd664c7ff35089955"}, "unpaired_constants": {"count": 598, "sha256": "3d84e1d95ee3a93aad1681443605a641284d894f5e5014669abf7668eb1eb49d"} } diff --git a/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt b/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt index 081e1a44b0..c7c6107cee 100644 --- a/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt +++ b/android/app/src/main/java/com/noop/ble/WhoopBleClient.kt @@ -10266,7 +10266,8 @@ class WhoopBleClient( .filter { it.ok && it.typeName == "REALTIME_DATA" } .mapNotNull { (it.parsed["timestamp"] as? Number)?.toInt() } .maxOrNull() ?: now - val streams: Streams = extractStreams(parsed, deviceClockRef = newestRealtimeTs, wallClockRef = now) + val streams: Streams = extractStreams(parsed, deviceClockRef = newestRealtimeTs, + wallClockRef = now, family = connectedFamily) val batch = StreamPersistence.toBatch(streams) // #1118: the SECOND live transport. The standard 0x2A37 path above stamps a beat at the second // it arrived; this one stamps it from the strap's own record clock. The same beat reaching both @@ -10334,8 +10335,11 @@ class WhoopBleClient( family: DeviceFamily) { val shouldFlush = synchronized(collectorLock) { if (hr in 30..220) stdHr.add(HrRow(ts, hr)) - val source = if (family == DeviceFamily.WHOOP5) - com.noop.protocol.RrSourceChannel.WHOOP5_STANDARD else null + val source = when (family) { + DeviceFamily.WHOOP5 -> com.noop.protocol.RrSourceChannel.WHOOP5_STANDARD + DeviceFamily.WHOOP4 -> com.noop.protocol.RrSourceChannel.WHOOP4_STANDARD + else -> null + } for (r in rr) if (r in 250..3000) stdRr.add(RrRow(ts, r, source)) stdContact.add(StandardHrMapping.contactEvent(ts, contact)) standardHrBufferReachedFlushThreshold(stdHr.size, stdRr.size, stdContact.size) diff --git a/android/app/src/main/java/com/noop/data/WhoopDao.kt b/android/app/src/main/java/com/noop/data/WhoopDao.kt index 5a1132f8ad..4582e39c76 100644 --- a/android/app/src/main/java/com/noop/data/WhoopDao.kt +++ b/android/app/src/main/java/com/noop/data/WhoopDao.kt @@ -12,7 +12,7 @@ import kotlinx.coroutines.flow.Flow /** Kept as one compile-time constant so Room and the plain-JVM SQLite regression test execute the exact * same statement. Swift's twin lives in WhoopStore.analysisFingerprint(). */ internal const val ANALYSIS_FINGERPRINT_SQL = - "SELECT 'v4|' || " + + "SELECT 'v5|' || " + "'h' || (SELECT COUNT(*) FROM hrSample) || ':' || (SELECT COALESCE(MAX(ts), 0) FROM hrSample) || '|' || " + "'p' || (SELECT COALESCE(MAX(rowid), 0) FROM ppgHrSample) || '|' || " + "'r' || (SELECT COALESCE(MAX(rowid), 0) FROM rrInterval) || '|' || " + @@ -58,7 +58,7 @@ internal const val ANALYSIS_FINGERPRINT_SQL = * same statement; unlike [ANALYSIS_FINGERPRINT_SQL] it carries :deviceId/:from/:to binds, so a plain-JVM * SQLite harness could not run it verbatim. Room's KSP verification is what checks it. */ internal const val DAY_STREAM_FINGERPRINT_SQL = - "SELECT 's3|' || " + + "SELECT 's4|' || " + "'p' || (SELECT COUNT(*) FROM ppgHrSample WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to) || " + "':' || (SELECT COALESCE(MAX(ts), 0) FROM ppgHrSample WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to) || '|' || " + "'r' || (SELECT COUNT(*) FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + @@ -135,15 +135,35 @@ internal const val WHOOP5_RR_INTERVALS_SQL = "AND (tsSuspect IS NULL OR tsSuspect <> 1)) " + "ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :limit" +/** + * One R-R source per UTC hour for a WHOOP 4, in provenance order: type-47 history, then type-40 + * realtime, then standard BLE, then legacy unlabelled rows. Overlapping transports are never merged, + * and history takes precedence only in the hours it actually covers, so a partial offload no longer + * hides live intervals across the whole interval (#1118, #2335). + * + * Deliberately NOT `AS MATERIALIZED`, unlike the Swift twin. That hint needs SQLite 3.35 and `minSdk` + * is 26, so it is a syntax error on most supported devices. The cost is that SQLite may re-evaluate + * the hour choice per row, which is the quadratic read the Swift side names; do not "fix" the + * asymmetry by adding the keyword. + */ internal const val WHOOP4_RR_INTERVALS_SQL = - "SELECT * FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + - "AND (srcChannel IS NULL OR srcChannel <> 2) " + + "WITH whoop4HourChoice AS (" + + "SELECT ts / 3600 AS hour, " + + "MIN(CASE WHEN srcChannel = 8 THEN 1 WHEN srcChannel = 9 THEN 2 " + + "WHEN srcChannel = 10 THEN 3 WHEN srcChannel IS NULL THEN 4 END) AS sourceChoice " + + "FROM rrInterval WHERE deviceId = :deviceId " + + "AND ts >= (:from / 3600) * 3600 AND ts < ((:to / 3600) + 1) * 3600 " + "AND (tsSuspect IS NULL OR tsSuspect <> 1) " + - "AND ((EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + - "AND srcChannel = 8 AND (tsSuspect IS NULL OR tsSuspect <> 1)) AND srcChannel = 8) " + - "OR (NOT EXISTS(SELECT 1 FROM rrInterval WHERE deviceId = :deviceId AND ts >= :from AND ts <= :to " + - "AND srcChannel = 8 AND (tsSuspect IS NULL OR tsSuspect <> 1)) AND srcChannel IS NULL)) " + - "ORDER BY ts ASC, ord ASC, rrMs ASC, seq ASC LIMIT :limit" + "AND (srcChannel IS NULL OR srcChannel IN (8, 9, 10)) GROUP BY ts / 3600) " + + "SELECT r.* FROM rrInterval r JOIN whoop4HourChoice h ON h.hour = r.ts / 3600 " + + "WHERE r.deviceId = :deviceId AND r.ts >= :from AND r.ts <= :to " + + "AND (r.srcChannel IS NULL OR r.srcChannel <> 2) " + + "AND ((r.srcChannel = 8 AND h.sourceChoice = 1) " + + "OR (r.srcChannel = 9 AND h.sourceChoice = 2) " + + "OR (r.srcChannel = 10 AND h.sourceChoice = 3) " + + "OR (r.srcChannel IS NULL AND h.sourceChoice = 4)) " + + "AND (r.tsSuspect IS NULL OR r.tsSuspect <> 1) " + + "ORDER BY r.ts ASC, r.ord ASC, r.rrMs ASC, r.seq ASC LIMIT :limit" /** The earliest beat a device has banked AT ALL, labelled or not, or null when it has none. The lower * bound on the "cannot be scored" explanation: it separates history this strap actually recorded from diff --git a/android/app/src/main/java/com/noop/protocol/Streams.kt b/android/app/src/main/java/com/noop/protocol/Streams.kt index 5698d9f5e0..c6bdd73db0 100644 --- a/android/app/src/main/java/com/noop/protocol/Streams.kt +++ b/android/app/src/main/java/com/noop/protocol/Streams.kt @@ -81,6 +81,10 @@ enum class RrSourceChannel(val code: Int) { WHOOP5_STANDARD(7), /** WHOOP 4.0 type-47 historical stream, distinguished from its unlabelled live BLE feed. */ WHOOP4_HISTORICAL(8), + /** WHOOP 4.0 type-40 realtime RR, timestamped from its record clock. */ + WHOOP4_REALTIME(9), + /** WHOOP 4.0 standard BLE 0x2A37 RR, timestamped on notification receipt. */ + WHOOP4_STANDARD(10), ; val isWhoop5Transport: Boolean get() = code in 5..7 @@ -323,8 +327,12 @@ private fun toWall(deviceTs: Int?, deviceClockRef: Int, wallClockRef: Int): Int? * HR for the same instants. Frames whose full integrity verdict is negative are skipped — that is * the header checksum, the payload CRC32 and the structural length together, not the payload CRC * alone. + * + * Swift twin: `extractStreams`. Adding `family` changed the arity on both sides at once, which + * unpaired them in the ledger even though they moved together. */ -fun extractStreams(parsed: List, deviceClockRef: Int, wallClockRef: Int): Streams { +fun extractStreams(parsed: List, deviceClockRef: Int, wallClockRef: Int, + family: DeviceFamily? = null): Streams { val out = Streams() for (r in parsed) { if (!r.ok) continue @@ -337,6 +345,7 @@ fun extractStreams(parsed: List, deviceClockRef: Int, wallClockRef: // Drop RR rows when timestamp is absent (a ts-less RR row is unstorable). p.intArrayOrNull("rr_intervals")?.let { rrs -> val source = RrSourceChannel.fromCode(p.intOrNull("rr_source_channel")) + ?: if (family == DeviceFamily.WHOOP4) RrSourceChannel.WHOOP4_REALTIME else null for (rr in rrs) out.rr.add(RrInterval(ts, rr, source)) } } diff --git a/android/app/src/test/java/com/noop/data/HrFingerprintTest.kt b/android/app/src/test/java/com/noop/data/HrFingerprintTest.kt index 487e3cc5c2..4579083b19 100644 --- a/android/app/src/test/java/com/noop/data/HrFingerprintTest.kt +++ b/android/app/src/test/java/com/noop/data/HrFingerprintTest.kt @@ -49,7 +49,7 @@ class HrFingerprintTest { @Test fun analysisFingerprintSqlRunsAndMovesForGravityOnlyCommit() { DriverManager.getConnection("jdbc:sqlite::memory:").use { db -> createFingerprintTables(db) - assertEquals("v4|h0:0|p0|r0|x0|g0|s0|e0|o0|t0|z0|w50|w70|tagged0|w4history0|registry", fingerprint(db)) + assertEquals("v5|h0:0|p0|r0|x0|g0|s0|e0|o0|t0|z0|w50|w70|tagged0|w4history0|registry", fingerprint(db)) db.createStatement().use { sql -> sql.executeUpdate("INSERT INTO hrSample(ts) VALUES (1000)") @@ -57,7 +57,7 @@ class HrFingerprintTest { sql.executeUpdate("INSERT INTO rrInterval DEFAULT VALUES") sql.executeUpdate("INSERT INTO gravitySample DEFAULT VALUES") } - assertEquals("v4|h1:1000|p0|r2|x0|g1|s0|e0|o0|t0|z0|w50|w70|tagged0|w4history0|registry", fingerprint(db)) + assertEquals("v5|h1:1000|p0|r2|x0|g1|s0|e0|o0|t0|z0|w50|w70|tagged0|w4history0|registry", fingerprint(db)) } } diff --git a/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt b/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt index f9f9aaa1d2..963e96bfba 100644 --- a/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt +++ b/android/app/src/test/java/com/noop/data/Whoop5RRSqliteTest.kt @@ -173,6 +173,28 @@ class Whoop5RRSqliteTest { registry("5.0 MG") } + @Test fun whoop4PartialHistoryKeepsUnlabelledRowsFromOtherHours() { + val base = 1_750_000_000L / 3600 * 3600 + fun insert(ts: Long, rr: Int, channel: Int?) { + statement("INSERT INTO rrInterval(deviceId,ts,rrMs,seq,synced,ord,srcChannel,tsSuspect) " + + "VALUES(:d,:t,:r,0,0,0,:c,NULL)", + mapOf("d" to id, "t" to ts, "r" to rr, "c" to channel)).use { it.executeUpdate() } + } + insert(base + 10, 800, null) + insert(base + 3600 + 10, 820, null) + insert(base + 3600 + 11, 830, null) + insert(base + 3600 + 12, 840, null) + insert(base + 10, 805, 8) + insert(base + 11, 815, 8) + insert(base + 3600 + 10, 825, 8) + + val selected = query( + WHOOP4_RR_INTERVALS_SQL, + mapOf("deviceId" to id, "from" to base, "to" to base + 7200, "limit" to 100), + ) { it.getInt("rrMs") to it.getInt("srcChannel") } + assertEquals(listOf(805 to 8, 815 to 8, 825 to 8), selected) + } + @After fun close() { db.close() } private fun sql(sql: String) { db.createStatement().use { it.execute(sql) } } private fun statement(sql: String, values: Map) = run { @@ -682,7 +704,7 @@ class Whoop5RRSqliteTest { assertEquals(listOf(8, 8), rows.map { it.srcChannel }) } - @Test fun whoop4PrefersOneHistoricalTransportForTheWholeWindow() = runBlocking { + @Test fun whoop4HistoricalSourceHasPriorityWithinItsHour() = runBlocking { registry("4.0") repo.insert(StreamBatch(rr = listOf(RrRow(100, 800), RrRow(101, 810))), id) repo.insert(StreamBatch(rr = listOf(RrRow(100, 805, RrSourceChannel.WHOOP4_HISTORICAL))), id) @@ -690,6 +712,18 @@ class Whoop5RRSqliteTest { assertEquals(listOf(805), rows.map { it.rrMs }) } + @Test fun whoop4RealtimeSourceWinsOverStandardAndLegacyRows() = runBlocking { + registry("4.0") + repo.insert(StreamBatch(rr = listOf( + RrRow(100, 800), + RrRow(100, 810, RrSourceChannel.WHOOP4_STANDARD), + RrRow(100, 820, RrSourceChannel.WHOOP4_REALTIME), + )), id) + val rows = repo.rrIntervalsForDevice(id, 100, 100, 100) + assertEquals(listOf(820), rows.map { it.rrMs }) + assertEquals(listOf(9), rows.map { it.srcChannel }) + } + @Test fun whoop4FallsBackToUnlabelledRowsWhenNoHistoryExists() = runBlocking { registry("4.0") repo.insert(StreamBatch(rr = listOf(RrRow(100, 800))), id) diff --git a/android/app/src/test/java/com/noop/protocol/FramingTest.kt b/android/app/src/test/java/com/noop/protocol/FramingTest.kt index 4c11d40b66..2a834e3fef 100644 --- a/android/app/src/test/java/com/noop/protocol/FramingTest.kt +++ b/android/app/src/test/java/com/noop/protocol/FramingTest.kt @@ -422,6 +422,10 @@ class FramingTest { assertEquals(2, streams.rr.size) assertEquals(RrInterval(ts = 1700000000, rrMs = 850), streams.rr[0]) assertEquals(RrInterval(ts = 1700000000, rrMs = 870), streams.rr[1]) + val whoop4 = extractStreams(listOf(parsed), deviceClockRef = 1700000000, + wallClockRef = 1700000000, family = DeviceFamily.WHOOP4) + assertEquals(listOf(RrSourceChannel.WHOOP4_REALTIME, RrSourceChannel.WHOOP4_REALTIME), + whoop4.rr.map { it.srcChannel }) } // MARK: - WHOOP 5.0/MG REALTIME_DATA (+4) + family-aware reassembly diff --git a/android/app/src/test/java/com/noop/protocol/Whoop5RRTest.kt b/android/app/src/test/java/com/noop/protocol/Whoop5RRTest.kt index e716b759b0..6ee1f0e47e 100644 --- a/android/app/src/test/java/com/noop/protocol/Whoop5RRTest.kt +++ b/android/app/src/test/java/com/noop/protocol/Whoop5RRTest.kt @@ -88,7 +88,7 @@ class Whoop5RRTest { } @Test fun transportCodes() { - assertEquals((1..8).toList(), RrSourceChannel.entries.map { it.code }) + assertEquals((1..10).toList(), RrSourceChannel.entries.map { it.code }) } /**